Upload memory sweep moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS +1 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt +14 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.pickle +3 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.json +79 -0
- moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.pickle +3 -0
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
SUCCESS
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
cuda_graph: on
|
| 2 |
+
seq_length: 2048
|
| 3 |
+
capacity: 1.00
|
| 4 |
+
expert_dispatch_overlap: off
|
| 5 |
+
expert_dispatch_recompute: off
|
| 6 |
+
global_batch_size: 32
|
| 7 |
+
train_iters: 10
|
| 8 |
+
master_port: 6000
|
| 9 |
+
expected_pickles: 80
|
| 10 |
+
actual_pickles: 80
|
| 11 |
+
status: SUCCESS
|
| 12 |
+
duration_s: 230
|
| 13 |
+
log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off.log
|
| 14 |
+
result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66871475200,
|
| 3 |
+
"active_bytes_current_delta": 31640019968,
|
| 4 |
+
"active_bytes_peak": 79683648512,
|
| 5 |
+
"active_bytes_peak_delta": 44217312256,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66871475200,
|
| 8 |
+
"active_bytes_peak": 79683648512,
|
| 9 |
+
"allocated_bytes_current": 66871475200,
|
| 10 |
+
"allocated_bytes_peak": 79683648512,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 4857414656,
|
| 13 |
+
"inactive_split_bytes_peak": 5749855232,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 94378131456,
|
| 21 |
+
"reserved_bytes_peak": 94378131456,
|
| 22 |
+
"segment_count_current": 518,
|
| 23 |
+
"segment_count_peak": 518,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:12.483106Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66871475200,
|
| 28 |
+
"allocated_bytes_current_delta": 31640019968,
|
| 29 |
+
"allocated_bytes_peak": 79683648512,
|
| 30 |
+
"allocated_bytes_peak_delta": 44217312256,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 35231455232,
|
| 33 |
+
"active_bytes_peak": 35466336256,
|
| 34 |
+
"allocated_bytes_current": 35231455232,
|
| 35 |
+
"allocated_bytes_peak": 35466336256,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 134916096,
|
| 38 |
+
"inactive_split_bytes_peak": 160631808,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 39246102528,
|
| 46 |
+
"reserved_bytes_peak": 39246102528,
|
| 47 |
+
"segment_count_current": 72,
|
| 48 |
+
"segment_count_peak": 72,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:39:04.476196Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4857414656,
|
| 55 |
+
"inactive_split_bytes_current_delta": 4722498560,
|
| 56 |
+
"inactive_split_bytes_peak": 5749855232,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 5589223424,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 94378131456,
|
| 68 |
+
"reserved_bytes_current_delta": 55132028928,
|
| 69 |
+
"reserved_bytes_peak": 94378131456,
|
| 70 |
+
"reserved_bytes_peak_delta": 55132028928,
|
| 71 |
+
"segment_count_current": 518,
|
| 72 |
+
"segment_count_current_delta": 446,
|
| 73 |
+
"segment_count_peak": 518,
|
| 74 |
+
"segment_count_peak_delta": 446,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:12.592236Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:13eb403a964d4aaa0e195ef80d1de987cbc685f88b0a65095ec8c26eff635061
|
| 3 |
+
size 5383895
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66871475200,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 103322287104,
|
| 5 |
+
"active_bytes_peak_delta": 23638638592,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66871475200,
|
| 8 |
+
"active_bytes_peak": 103322287104,
|
| 9 |
+
"allocated_bytes_current": 66871475200,
|
| 10 |
+
"allocated_bytes_peak": 103322287104,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 3961930752,
|
| 13 |
+
"inactive_split_bytes_peak": 5749855232,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 104406712320,
|
| 21 |
+
"reserved_bytes_peak": 104406712320,
|
| 22 |
+
"segment_count_current": 591,
|
| 23 |
+
"segment_count_peak": 591,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:13.440234Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66871475200,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103322287104,
|
| 30 |
+
"allocated_bytes_peak_delta": 23638638592,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66871475200,
|
| 33 |
+
"active_bytes_peak": 79683648512,
|
| 34 |
+
"allocated_bytes_current": 66871475200,
|
| 35 |
+
"allocated_bytes_peak": 79683648512,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 4857414656,
|
| 38 |
+
"inactive_split_bytes_peak": 5749855232,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 94378131456,
|
| 46 |
+
"reserved_bytes_peak": 94378131456,
|
| 47 |
+
"segment_count_current": 518,
|
| 48 |
+
"segment_count_peak": 518,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:12.611327Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 3961930752,
|
| 55 |
+
"inactive_split_bytes_current_delta": -895483904,
|
| 56 |
+
"inactive_split_bytes_peak": 5749855232,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 104406712320,
|
| 68 |
+
"reserved_bytes_current_delta": 10028580864,
|
| 69 |
+
"reserved_bytes_peak": 104406712320,
|
| 70 |
+
"reserved_bytes_peak_delta": 10028580864,
|
| 71 |
+
"segment_count_current": 591,
|
| 72 |
+
"segment_count_current_delta": 73,
|
| 73 |
+
"segment_count_peak": 591,
|
| 74 |
+
"segment_count_peak_delta": 73,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:13.538384Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7770105a42e873f83f3788b3924e707562035434d844fe87fa2cfc5ae405aaee
|
| 3 |
+
size 5583228
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66871475200,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 103322287104,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66871475200,
|
| 8 |
+
"active_bytes_peak": 103322287104,
|
| 9 |
+
"allocated_bytes_current": 66871475200,
|
| 10 |
+
"allocated_bytes_peak": 103322287104,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 4131800064,
|
| 13 |
+
"inactive_split_bytes_peak": 5749855232,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 104406712320,
|
| 21 |
+
"reserved_bytes_peak": 104406712320,
|
| 22 |
+
"segment_count_current": 591,
|
| 23 |
+
"segment_count_peak": 591,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:14.352612Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66871475200,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103322287104,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66871475200,
|
| 33 |
+
"active_bytes_peak": 103322287104,
|
| 34 |
+
"allocated_bytes_current": 66871475200,
|
| 35 |
+
"allocated_bytes_peak": 103322287104,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 3961930752,
|
| 38 |
+
"inactive_split_bytes_peak": 5749855232,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 104406712320,
|
| 46 |
+
"reserved_bytes_peak": 104406712320,
|
| 47 |
+
"segment_count_current": 591,
|
| 48 |
+
"segment_count_peak": 591,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:13.549825Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4131800064,
|
| 55 |
+
"inactive_split_bytes_current_delta": 169869312,
|
| 56 |
+
"inactive_split_bytes_peak": 5749855232,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 104406712320,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 104406712320,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 591,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 591,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:14.444958Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:900b45d88f1d794f2575cd36e05bc551a33c2bd3d82fbc54ed027bbfdf34e584
|
| 3 |
+
size 5584389
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66888518656,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 121491349504,
|
| 5 |
+
"active_bytes_peak_delta": 18169062400,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66888518656,
|
| 8 |
+
"active_bytes_peak": 121491349504,
|
| 9 |
+
"allocated_bytes_current": 66888518656,
|
| 10 |
+
"allocated_bytes_peak": 103336542208,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 4584518656,
|
| 13 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 132521132032,
|
| 21 |
+
"reserved_bytes_peak": 132521132032,
|
| 22 |
+
"segment_count_current": 798,
|
| 23 |
+
"segment_count_peak": 798,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:17.033382Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66888518656,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 103336542208,
|
| 30 |
+
"allocated_bytes_peak_delta": 14255104,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66871475200,
|
| 33 |
+
"active_bytes_peak": 103322287104,
|
| 34 |
+
"allocated_bytes_current": 66871475200,
|
| 35 |
+
"allocated_bytes_peak": 103322287104,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 4131800064,
|
| 38 |
+
"inactive_split_bytes_peak": 5749855232,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 104406712320,
|
| 46 |
+
"reserved_bytes_peak": 104406712320,
|
| 47 |
+
"segment_count_current": 591,
|
| 48 |
+
"segment_count_peak": 591,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:14.455675Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4584518656,
|
| 55 |
+
"inactive_split_bytes_current_delta": 452718592,
|
| 56 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 1470686208,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132521132032,
|
| 68 |
+
"reserved_bytes_current_delta": 28114419712,
|
| 69 |
+
"reserved_bytes_peak": 132521132032,
|
| 70 |
+
"reserved_bytes_peak_delta": 28114419712,
|
| 71 |
+
"segment_count_current": 798,
|
| 72 |
+
"segment_count_current_delta": 207,
|
| 73 |
+
"segment_count_peak": 798,
|
| 74 |
+
"segment_count_peak_delta": 207,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:17.150803Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b3353b418b19312db946e05d6a17b3e9d5b702632e9c28904515cff97fe2cce6
|
| 3 |
+
size 6274771
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66888518656,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121491349504,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66888518656,
|
| 8 |
+
"active_bytes_peak": 121491349504,
|
| 9 |
+
"allocated_bytes_current": 66888518656,
|
| 10 |
+
"allocated_bytes_peak": 103336542208,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 4584518656,
|
| 13 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 132521132032,
|
| 21 |
+
"reserved_bytes_peak": 132521132032,
|
| 22 |
+
"segment_count_current": 798,
|
| 23 |
+
"segment_count_peak": 798,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:17.864648Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66888518656,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103336542208,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66888518656,
|
| 33 |
+
"active_bytes_peak": 121491349504,
|
| 34 |
+
"allocated_bytes_current": 66888518656,
|
| 35 |
+
"allocated_bytes_peak": 103336542208,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 4584518656,
|
| 38 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 132521132032,
|
| 46 |
+
"reserved_bytes_peak": 132521132032,
|
| 47 |
+
"segment_count_current": 798,
|
| 48 |
+
"segment_count_peak": 798,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:17.164045Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4584518656,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132521132032,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 132521132032,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 798,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 798,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:17.872215Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6435a311776ea8d99a13db28debafd039dbda3f4b899a139c6a412d0070e9fa7
|
| 3 |
+
size 266048
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66888518656,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121491349504,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66888518656,
|
| 8 |
+
"active_bytes_peak": 121491349504,
|
| 9 |
+
"allocated_bytes_current": 66888518656,
|
| 10 |
+
"allocated_bytes_peak": 103336542208,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 4584518656,
|
| 13 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 132521132032,
|
| 21 |
+
"reserved_bytes_peak": 132521132032,
|
| 22 |
+
"segment_count_current": 798,
|
| 23 |
+
"segment_count_peak": 798,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:18.565796Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66888518656,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103336542208,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66888518656,
|
| 33 |
+
"active_bytes_peak": 121491349504,
|
| 34 |
+
"allocated_bytes_current": 66888518656,
|
| 35 |
+
"allocated_bytes_peak": 103336542208,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 4584518656,
|
| 38 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 132521132032,
|
| 46 |
+
"reserved_bytes_peak": 132521132032,
|
| 47 |
+
"segment_count_current": 798,
|
| 48 |
+
"segment_count_peak": 798,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:17.873174Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4584518656,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132521132032,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 132521132032,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 798,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 798,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:18.572888Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8e12dcf81fed3acc951c00dde72513e207be4b99b8f85eac0f1e26ccc122d9c8
|
| 3 |
+
size 266049
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66888518656,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121491349504,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66888518656,
|
| 8 |
+
"active_bytes_peak": 121491349504,
|
| 9 |
+
"allocated_bytes_current": 66888518656,
|
| 10 |
+
"allocated_bytes_peak": 103336542208,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 4584518656,
|
| 13 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 14 |
+
"iteration": 6,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 132521132032,
|
| 21 |
+
"reserved_bytes_peak": 132521132032,
|
| 22 |
+
"segment_count_current": 798,
|
| 23 |
+
"segment_count_peak": 798,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:19.267583Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66888518656,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103336542208,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66888518656,
|
| 33 |
+
"active_bytes_peak": 121491349504,
|
| 34 |
+
"allocated_bytes_current": 66888518656,
|
| 35 |
+
"allocated_bytes_peak": 103336542208,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 4584518656,
|
| 38 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 39 |
+
"iteration": 6,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 132521132032,
|
| 46 |
+
"reserved_bytes_peak": 132521132032,
|
| 47 |
+
"segment_count_current": 798,
|
| 48 |
+
"segment_count_peak": 798,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:18.573818Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4584518656,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 6,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132521132032,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 132521132032,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 798,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 798,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:19.275794Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6d3325263acc1c15b76e149c7e8d846bdff5c56966c03cbdc5967e9edf662aaf
|
| 3 |
+
size 266049
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66888518656,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121491349504,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66888518656,
|
| 8 |
+
"active_bytes_peak": 121491349504,
|
| 9 |
+
"allocated_bytes_current": 66888518656,
|
| 10 |
+
"allocated_bytes_peak": 103336542208,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 4584518656,
|
| 13 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 14 |
+
"iteration": 7,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 132521132032,
|
| 21 |
+
"reserved_bytes_peak": 132521132032,
|
| 22 |
+
"segment_count_current": 798,
|
| 23 |
+
"segment_count_peak": 798,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:19.968607Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66888518656,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103336542208,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66888518656,
|
| 33 |
+
"active_bytes_peak": 121491349504,
|
| 34 |
+
"allocated_bytes_current": 66888518656,
|
| 35 |
+
"allocated_bytes_peak": 103336542208,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 4584518656,
|
| 38 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 39 |
+
"iteration": 7,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 132521132032,
|
| 46 |
+
"reserved_bytes_peak": 132521132032,
|
| 47 |
+
"segment_count_current": 798,
|
| 48 |
+
"segment_count_peak": 798,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:19.276766Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4584518656,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 7,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132521132032,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 132521132032,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 798,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 798,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:19.975851Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:058c23d59cf99ce1d34d3e898fccaa8bd00d8cea459c66f29cbde75e279054b4
|
| 3 |
+
size 266049
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66888518656,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121491349504,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66888518656,
|
| 8 |
+
"active_bytes_peak": 121491349504,
|
| 9 |
+
"allocated_bytes_current": 66888518656,
|
| 10 |
+
"allocated_bytes_peak": 103336542208,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 4584518656,
|
| 13 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 14 |
+
"iteration": 8,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 132521132032,
|
| 21 |
+
"reserved_bytes_peak": 132521132032,
|
| 22 |
+
"segment_count_current": 798,
|
| 23 |
+
"segment_count_peak": 798,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:20.672461Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66888518656,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103336542208,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66888518656,
|
| 33 |
+
"active_bytes_peak": 121491349504,
|
| 34 |
+
"allocated_bytes_current": 66888518656,
|
| 35 |
+
"allocated_bytes_peak": 103336542208,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 4584518656,
|
| 38 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 39 |
+
"iteration": 8,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 132521132032,
|
| 46 |
+
"reserved_bytes_peak": 132521132032,
|
| 47 |
+
"segment_count_current": 798,
|
| 48 |
+
"segment_count_peak": 798,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:19.976811Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4584518656,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 8,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132521132032,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 132521132032,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 798,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 798,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:20.679449Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:497816d6d6e28441383edc174a3ef28054bf0ac55df61c8863520df789c7791f
|
| 3 |
+
size 266049
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66854884352,
|
| 3 |
+
"active_bytes_current_delta": -33634304,
|
| 4 |
+
"active_bytes_peak": 121491349504,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66854884352,
|
| 8 |
+
"active_bytes_peak": 121491349504,
|
| 9 |
+
"allocated_bytes_current": 66854884352,
|
| 10 |
+
"allocated_bytes_peak": 103336542208,
|
| 11 |
+
"device": 0,
|
| 12 |
+
"inactive_split_bytes_current": 2523098112,
|
| 13 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 14 |
+
"iteration": 9,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 0,
|
| 20 |
+
"reserved_bytes_current": 132544200704,
|
| 21 |
+
"reserved_bytes_peak": 132544200704,
|
| 22 |
+
"segment_count_current": 800,
|
| 23 |
+
"segment_count_peak": 800,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:41:13.363803Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66854884352,
|
| 28 |
+
"allocated_bytes_current_delta": -33634304,
|
| 29 |
+
"allocated_bytes_peak": 103336542208,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66888518656,
|
| 33 |
+
"active_bytes_peak": 121491349504,
|
| 34 |
+
"allocated_bytes_current": 66888518656,
|
| 35 |
+
"allocated_bytes_peak": 103336542208,
|
| 36 |
+
"device": 0,
|
| 37 |
+
"inactive_split_bytes_current": 4584518656,
|
| 38 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 39 |
+
"iteration": 9,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 0,
|
| 45 |
+
"reserved_bytes_current": 132521132032,
|
| 46 |
+
"reserved_bytes_peak": 132521132032,
|
| 47 |
+
"segment_count_current": 798,
|
| 48 |
+
"segment_count_peak": 798,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:20.680382Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 0,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2523098112,
|
| 55 |
+
"inactive_split_bytes_current_delta": -2061420544,
|
| 56 |
+
"inactive_split_bytes_peak": 7220541440,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 9,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 0,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132544200704,
|
| 68 |
+
"reserved_bytes_current_delta": 23068672,
|
| 69 |
+
"reserved_bytes_peak": 132544200704,
|
| 70 |
+
"reserved_bytes_peak_delta": 23068672,
|
| 71 |
+
"segment_count_current": 800,
|
| 72 |
+
"segment_count_current_delta": 2,
|
| 73 |
+
"segment_count_peak": 800,
|
| 74 |
+
"segment_count_peak_delta": 2,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:41:13.448813Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:691713541afb401c4594240700d7746937afee4116911dc1465e490b845e29e1
|
| 3 |
+
size 4204922
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66871475200,
|
| 3 |
+
"active_bytes_current_delta": 31640019968,
|
| 4 |
+
"active_bytes_peak": 79690286080,
|
| 5 |
+
"active_bytes_peak_delta": 44223949824,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66871475200,
|
| 8 |
+
"active_bytes_peak": 79690286080,
|
| 9 |
+
"allocated_bytes_current": 66871475200,
|
| 10 |
+
"allocated_bytes_peak": 79690286080,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 4328932352,
|
| 13 |
+
"inactive_split_bytes_peak": 5229261824,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 94376034304,
|
| 21 |
+
"reserved_bytes_peak": 94376034304,
|
| 22 |
+
"segment_count_current": 520,
|
| 23 |
+
"segment_count_peak": 520,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:12.483218Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66871475200,
|
| 28 |
+
"allocated_bytes_current_delta": 31640019968,
|
| 29 |
+
"allocated_bytes_peak": 79690286080,
|
| 30 |
+
"allocated_bytes_peak_delta": 44223949824,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 35231455232,
|
| 33 |
+
"active_bytes_peak": 35466336256,
|
| 34 |
+
"allocated_bytes_current": 35231455232,
|
| 35 |
+
"allocated_bytes_peak": 35466336256,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 132818944,
|
| 38 |
+
"inactive_split_bytes_peak": 160386048,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 39244005376,
|
| 46 |
+
"reserved_bytes_peak": 39244005376,
|
| 47 |
+
"segment_count_current": 71,
|
| 48 |
+
"segment_count_peak": 71,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:39:04.476570Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4328932352,
|
| 55 |
+
"inactive_split_bytes_current_delta": 4196113408,
|
| 56 |
+
"inactive_split_bytes_peak": 5229261824,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 5068875776,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 94376034304,
|
| 68 |
+
"reserved_bytes_current_delta": 55132028928,
|
| 69 |
+
"reserved_bytes_peak": 94376034304,
|
| 70 |
+
"reserved_bytes_peak_delta": 55132028928,
|
| 71 |
+
"segment_count_current": 520,
|
| 72 |
+
"segment_count_current_delta": 449,
|
| 73 |
+
"segment_count_peak": 520,
|
| 74 |
+
"segment_count_peak_delta": 449,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:12.600020Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:81b303ab50c373505e17e5a08d61daf215d49515fb8d51512f5579ea9dc08012
|
| 3 |
+
size 5379054
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66871475200,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 103326691328,
|
| 5 |
+
"active_bytes_peak_delta": 23636405248,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66871475200,
|
| 8 |
+
"active_bytes_peak": 103326691328,
|
| 9 |
+
"allocated_bytes_current": 66871475200,
|
| 10 |
+
"allocated_bytes_peak": 103326691328,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 3620094976,
|
| 13 |
+
"inactive_split_bytes_peak": 5229261824,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 104421392384,
|
| 21 |
+
"reserved_bytes_peak": 104421392384,
|
| 22 |
+
"segment_count_current": 590,
|
| 23 |
+
"segment_count_peak": 590,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:13.440274Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66871475200,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103326691328,
|
| 30 |
+
"allocated_bytes_peak_delta": 23636405248,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66871475200,
|
| 33 |
+
"active_bytes_peak": 79690286080,
|
| 34 |
+
"allocated_bytes_current": 66871475200,
|
| 35 |
+
"allocated_bytes_peak": 79690286080,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 4328932352,
|
| 38 |
+
"inactive_split_bytes_peak": 5229261824,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 94376034304,
|
| 46 |
+
"reserved_bytes_peak": 94376034304,
|
| 47 |
+
"segment_count_current": 520,
|
| 48 |
+
"segment_count_peak": 520,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:12.617749Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 3620094976,
|
| 55 |
+
"inactive_split_bytes_current_delta": -708837376,
|
| 56 |
+
"inactive_split_bytes_peak": 5229261824,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 104421392384,
|
| 68 |
+
"reserved_bytes_current_delta": 10045358080,
|
| 69 |
+
"reserved_bytes_peak": 104421392384,
|
| 70 |
+
"reserved_bytes_peak_delta": 10045358080,
|
| 71 |
+
"segment_count_current": 590,
|
| 72 |
+
"segment_count_current_delta": 70,
|
| 73 |
+
"segment_count_peak": 590,
|
| 74 |
+
"segment_count_peak_delta": 70,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:13.545141Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6c9035cf43d8fc94c39de8f7a9fc59b0c2de1afe3ff5e2e3ec9392788bc29574
|
| 3 |
+
size 5578891
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66871475200,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 103326691328,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66871475200,
|
| 8 |
+
"active_bytes_peak": 103326691328,
|
| 9 |
+
"allocated_bytes_current": 66871475200,
|
| 10 |
+
"allocated_bytes_peak": 103326691328,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 3712369664,
|
| 13 |
+
"inactive_split_bytes_peak": 5229261824,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 104421392384,
|
| 21 |
+
"reserved_bytes_peak": 104421392384,
|
| 22 |
+
"segment_count_current": 590,
|
| 23 |
+
"segment_count_peak": 590,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:14.352684Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66871475200,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103326691328,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66871475200,
|
| 33 |
+
"active_bytes_peak": 103326691328,
|
| 34 |
+
"allocated_bytes_current": 66871475200,
|
| 35 |
+
"allocated_bytes_peak": 103326691328,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 3620094976,
|
| 38 |
+
"inactive_split_bytes_peak": 5229261824,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 104421392384,
|
| 46 |
+
"reserved_bytes_peak": 104421392384,
|
| 47 |
+
"segment_count_current": 590,
|
| 48 |
+
"segment_count_peak": 590,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:13.558680Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 3712369664,
|
| 55 |
+
"inactive_split_bytes_current_delta": 92274688,
|
| 56 |
+
"inactive_split_bytes_peak": 5229261824,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 104421392384,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 104421392384,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 590,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 590,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:14.456557Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:919b3d739ef62345a6435a45506cc04af73b5428803af66af59c963371607c48
|
| 3 |
+
size 5581958
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66888518656,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 121477455872,
|
| 5 |
+
"active_bytes_peak_delta": 18150764544,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66888518656,
|
| 8 |
+
"active_bytes_peak": 121477455872,
|
| 9 |
+
"allocated_bytes_current": 66888518656,
|
| 10 |
+
"allocated_bytes_peak": 103326691328,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 4248974336,
|
| 13 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 132101701632,
|
| 21 |
+
"reserved_bytes_peak": 132101701632,
|
| 22 |
+
"segment_count_current": 790,
|
| 23 |
+
"segment_count_peak": 790,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:17.033418Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66888518656,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 103326691328,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66871475200,
|
| 33 |
+
"active_bytes_peak": 103326691328,
|
| 34 |
+
"allocated_bytes_current": 66871475200,
|
| 35 |
+
"allocated_bytes_peak": 103326691328,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 3712369664,
|
| 38 |
+
"inactive_split_bytes_peak": 5229261824,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 104421392384,
|
| 46 |
+
"reserved_bytes_peak": 104421392384,
|
| 47 |
+
"segment_count_current": 590,
|
| 48 |
+
"segment_count_peak": 590,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:14.469703Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4248974336,
|
| 55 |
+
"inactive_split_bytes_current_delta": 536604672,
|
| 56 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 2452887552,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132101701632,
|
| 68 |
+
"reserved_bytes_current_delta": 27680309248,
|
| 69 |
+
"reserved_bytes_peak": 132101701632,
|
| 70 |
+
"reserved_bytes_peak_delta": 27680309248,
|
| 71 |
+
"segment_count_current": 790,
|
| 72 |
+
"segment_count_current_delta": 200,
|
| 73 |
+
"segment_count_peak": 790,
|
| 74 |
+
"segment_count_peak_delta": 200,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:17.153033Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:26ccef36f0e9d2cec4a938fee739eb69e8ff1132ac1575aab7473826237fbc75
|
| 3 |
+
size 5852185
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66888518656,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121477455872,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66888518656,
|
| 8 |
+
"active_bytes_peak": 121477455872,
|
| 9 |
+
"allocated_bytes_current": 66888518656,
|
| 10 |
+
"allocated_bytes_peak": 103326691328,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 4248974336,
|
| 13 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 132101701632,
|
| 21 |
+
"reserved_bytes_peak": 132101701632,
|
| 22 |
+
"segment_count_current": 790,
|
| 23 |
+
"segment_count_peak": 790,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:17.864704Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66888518656,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103326691328,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66888518656,
|
| 33 |
+
"active_bytes_peak": 121477455872,
|
| 34 |
+
"allocated_bytes_current": 66888518656,
|
| 35 |
+
"allocated_bytes_peak": 103326691328,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 4248974336,
|
| 38 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 132101701632,
|
| 46 |
+
"reserved_bytes_peak": 132101701632,
|
| 47 |
+
"segment_count_current": 790,
|
| 48 |
+
"segment_count_peak": 790,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:17.168178Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4248974336,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132101701632,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 132101701632,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 790,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 790,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:17.872594Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:621a7e04293676d762c0de3bc2668b3b5ae6f5ebe02fb676dcf05f92e211d2dc
|
| 3 |
+
size 243291
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66888518656,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121477455872,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66888518656,
|
| 8 |
+
"active_bytes_peak": 121477455872,
|
| 9 |
+
"allocated_bytes_current": 66888518656,
|
| 10 |
+
"allocated_bytes_peak": 103326691328,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 4248974336,
|
| 13 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 132101701632,
|
| 21 |
+
"reserved_bytes_peak": 132101701632,
|
| 22 |
+
"segment_count_current": 790,
|
| 23 |
+
"segment_count_peak": 790,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:18.566062Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66888518656,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103326691328,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66888518656,
|
| 33 |
+
"active_bytes_peak": 121477455872,
|
| 34 |
+
"allocated_bytes_current": 66888518656,
|
| 35 |
+
"allocated_bytes_peak": 103326691328,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 4248974336,
|
| 38 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 132101701632,
|
| 46 |
+
"reserved_bytes_peak": 132101701632,
|
| 47 |
+
"segment_count_current": 790,
|
| 48 |
+
"segment_count_peak": 790,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:17.873601Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4248974336,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132101701632,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 132101701632,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 790,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 790,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:18.574762Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:203517b91270fdd338a0320f887d94b09fe296284c61cb7a3d3d215f631adea6
|
| 3 |
+
size 243291
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66888518656,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121477455872,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66888518656,
|
| 8 |
+
"active_bytes_peak": 121477455872,
|
| 9 |
+
"allocated_bytes_current": 66888518656,
|
| 10 |
+
"allocated_bytes_peak": 103326691328,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 4248974336,
|
| 13 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 14 |
+
"iteration": 6,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 132101701632,
|
| 21 |
+
"reserved_bytes_peak": 132101701632,
|
| 22 |
+
"segment_count_current": 790,
|
| 23 |
+
"segment_count_peak": 790,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:19.267574Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66888518656,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103326691328,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66888518656,
|
| 33 |
+
"active_bytes_peak": 121477455872,
|
| 34 |
+
"allocated_bytes_current": 66888518656,
|
| 35 |
+
"allocated_bytes_peak": 103326691328,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 4248974336,
|
| 38 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 39 |
+
"iteration": 6,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 132101701632,
|
| 46 |
+
"reserved_bytes_peak": 132101701632,
|
| 47 |
+
"segment_count_current": 790,
|
| 48 |
+
"segment_count_peak": 790,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:18.575728Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4248974336,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 6,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132101701632,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 132101701632,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 790,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 790,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:19.274628Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:84daf2896871a2726a78c16d3263a77aa76eb557618b149c658242774e14beba
|
| 3 |
+
size 243291
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66888518656,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121477455872,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66888518656,
|
| 8 |
+
"active_bytes_peak": 121477455872,
|
| 9 |
+
"allocated_bytes_current": 66888518656,
|
| 10 |
+
"allocated_bytes_peak": 103326691328,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 4248974336,
|
| 13 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 14 |
+
"iteration": 7,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 132101701632,
|
| 21 |
+
"reserved_bytes_peak": 132101701632,
|
| 22 |
+
"segment_count_current": 790,
|
| 23 |
+
"segment_count_peak": 790,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:19.968643Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66888518656,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103326691328,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66888518656,
|
| 33 |
+
"active_bytes_peak": 121477455872,
|
| 34 |
+
"allocated_bytes_current": 66888518656,
|
| 35 |
+
"allocated_bytes_peak": 103326691328,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 4248974336,
|
| 38 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 39 |
+
"iteration": 7,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 132101701632,
|
| 46 |
+
"reserved_bytes_peak": 132101701632,
|
| 47 |
+
"segment_count_current": 790,
|
| 48 |
+
"segment_count_peak": 790,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:19.275567Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4248974336,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 7,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132101701632,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 132101701632,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 790,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 790,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:19.975826Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:491e864bb6d6cebfc7d4d0186fc45e79cd23addbdc1793bbe2e6978bf57b54c5
|
| 3 |
+
size 243291
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66888518656,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121477455872,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66888518656,
|
| 8 |
+
"active_bytes_peak": 121477455872,
|
| 9 |
+
"allocated_bytes_current": 66888518656,
|
| 10 |
+
"allocated_bytes_peak": 103326691328,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 4248974336,
|
| 13 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 14 |
+
"iteration": 8,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 132101701632,
|
| 21 |
+
"reserved_bytes_peak": 132101701632,
|
| 22 |
+
"segment_count_current": 790,
|
| 23 |
+
"segment_count_peak": 790,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:20.672505Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66888518656,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103326691328,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66888518656,
|
| 33 |
+
"active_bytes_peak": 121477455872,
|
| 34 |
+
"allocated_bytes_current": 66888518656,
|
| 35 |
+
"allocated_bytes_peak": 103326691328,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 4248974336,
|
| 38 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 39 |
+
"iteration": 8,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 132101701632,
|
| 46 |
+
"reserved_bytes_peak": 132101701632,
|
| 47 |
+
"segment_count_current": 790,
|
| 48 |
+
"segment_count_peak": 790,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:19.976764Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4248974336,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 8,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132101701632,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 132101701632,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 790,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 790,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:20.679586Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:eafba8982e23688b1a834c2913971f061d8f8840902cd0bbc65619f83b29868c
|
| 3 |
+
size 243291
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66854884352,
|
| 3 |
+
"active_bytes_current_delta": -33634304,
|
| 4 |
+
"active_bytes_peak": 121477455872,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66854884352,
|
| 8 |
+
"active_bytes_peak": 121477455872,
|
| 9 |
+
"allocated_bytes_current": 66854884352,
|
| 10 |
+
"allocated_bytes_peak": 103326691328,
|
| 11 |
+
"device": 1,
|
| 12 |
+
"inactive_split_bytes_current": 2296605696,
|
| 13 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 14 |
+
"iteration": 9,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 1,
|
| 20 |
+
"reserved_bytes_current": 132124770304,
|
| 21 |
+
"reserved_bytes_peak": 132124770304,
|
| 22 |
+
"segment_count_current": 792,
|
| 23 |
+
"segment_count_peak": 792,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:41:13.363810Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66854884352,
|
| 28 |
+
"allocated_bytes_current_delta": -33634304,
|
| 29 |
+
"allocated_bytes_peak": 103326691328,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66888518656,
|
| 33 |
+
"active_bytes_peak": 121477455872,
|
| 34 |
+
"allocated_bytes_current": 66888518656,
|
| 35 |
+
"allocated_bytes_peak": 103326691328,
|
| 36 |
+
"device": 1,
|
| 37 |
+
"inactive_split_bytes_current": 4248974336,
|
| 38 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 39 |
+
"iteration": 9,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 1,
|
| 45 |
+
"reserved_bytes_current": 132101701632,
|
| 46 |
+
"reserved_bytes_peak": 132101701632,
|
| 47 |
+
"segment_count_current": 790,
|
| 48 |
+
"segment_count_peak": 790,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:20.680576Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 1,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 2296605696,
|
| 55 |
+
"inactive_split_bytes_current_delta": -1952368640,
|
| 56 |
+
"inactive_split_bytes_peak": 7682149376,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 9,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 1,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132124770304,
|
| 68 |
+
"reserved_bytes_current_delta": 23068672,
|
| 69 |
+
"reserved_bytes_peak": 132124770304,
|
| 70 |
+
"reserved_bytes_peak_delta": 23068672,
|
| 71 |
+
"segment_count_current": 792,
|
| 72 |
+
"segment_count_current_delta": 2,
|
| 73 |
+
"segment_count_peak": 792,
|
| 74 |
+
"segment_count_peak_delta": 2,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:41:13.451555Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e2e99df3ac0599cd00e64a13265bc0d4fe58af3467079b99f62d72165aa9eab7
|
| 3 |
+
size 4121826
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66871475200,
|
| 3 |
+
"active_bytes_current_delta": 31640019968,
|
| 4 |
+
"active_bytes_peak": 79690286080,
|
| 5 |
+
"active_bytes_peak_delta": 44223949824,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66871475200,
|
| 8 |
+
"active_bytes_peak": 79690286080,
|
| 9 |
+
"allocated_bytes_current": 66871475200,
|
| 10 |
+
"allocated_bytes_peak": 79690286080,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 4328932352,
|
| 13 |
+
"inactive_split_bytes_peak": 5464060928,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 93906272256,
|
| 21 |
+
"reserved_bytes_peak": 93906272256,
|
| 22 |
+
"segment_count_current": 517,
|
| 23 |
+
"segment_count_peak": 517,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:12.483142Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66871475200,
|
| 28 |
+
"allocated_bytes_current_delta": 31640019968,
|
| 29 |
+
"allocated_bytes_peak": 79690286080,
|
| 30 |
+
"allocated_bytes_peak_delta": 44223949824,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 35231455232,
|
| 33 |
+
"active_bytes_peak": 35466336256,
|
| 34 |
+
"allocated_bytes_current": 35231455232,
|
| 35 |
+
"allocated_bytes_peak": 35466336256,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 132818944,
|
| 38 |
+
"inactive_split_bytes_peak": 160386048,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 39244005376,
|
| 46 |
+
"reserved_bytes_peak": 39244005376,
|
| 47 |
+
"segment_count_current": 71,
|
| 48 |
+
"segment_count_peak": 71,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:39:04.476596Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4328932352,
|
| 55 |
+
"inactive_split_bytes_current_delta": 4196113408,
|
| 56 |
+
"inactive_split_bytes_peak": 5464060928,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 5303674880,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 93906272256,
|
| 68 |
+
"reserved_bytes_current_delta": 54662266880,
|
| 69 |
+
"reserved_bytes_peak": 93906272256,
|
| 70 |
+
"reserved_bytes_peak_delta": 54662266880,
|
| 71 |
+
"segment_count_current": 517,
|
| 72 |
+
"segment_count_current_delta": 446,
|
| 73 |
+
"segment_count_peak": 517,
|
| 74 |
+
"segment_count_peak_delta": 446,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:12.593828Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:eba85767f668dba66ab14017ea6b62d73b66eea84f68ff3cc013c9ba27273d4d
|
| 3 |
+
size 5378522
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66871475200,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 103326691328,
|
| 5 |
+
"active_bytes_peak_delta": 23636405248,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66871475200,
|
| 8 |
+
"active_bytes_peak": 103326691328,
|
| 9 |
+
"allocated_bytes_current": 66871475200,
|
| 10 |
+
"allocated_bytes_peak": 103326691328,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 4033233920,
|
| 13 |
+
"inactive_split_bytes_peak": 5464060928,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 104421392384,
|
| 21 |
+
"reserved_bytes_peak": 104421392384,
|
| 22 |
+
"segment_count_current": 589,
|
| 23 |
+
"segment_count_peak": 589,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:13.440247Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66871475200,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103326691328,
|
| 30 |
+
"allocated_bytes_peak_delta": 23636405248,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66871475200,
|
| 33 |
+
"active_bytes_peak": 79690286080,
|
| 34 |
+
"allocated_bytes_current": 66871475200,
|
| 35 |
+
"allocated_bytes_peak": 79690286080,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 4328932352,
|
| 38 |
+
"inactive_split_bytes_peak": 5464060928,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 93906272256,
|
| 46 |
+
"reserved_bytes_peak": 93906272256,
|
| 47 |
+
"segment_count_current": 517,
|
| 48 |
+
"segment_count_peak": 517,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:12.611672Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4033233920,
|
| 55 |
+
"inactive_split_bytes_current_delta": -295698432,
|
| 56 |
+
"inactive_split_bytes_peak": 5464060928,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 104421392384,
|
| 68 |
+
"reserved_bytes_current_delta": 10515120128,
|
| 69 |
+
"reserved_bytes_peak": 104421392384,
|
| 70 |
+
"reserved_bytes_peak_delta": 10515120128,
|
| 71 |
+
"segment_count_current": 589,
|
| 72 |
+
"segment_count_current_delta": 72,
|
| 73 |
+
"segment_count_peak": 589,
|
| 74 |
+
"segment_count_peak_delta": 72,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:13.545258Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a2c95ab6ab4b590631e5c6111813d6e9e421b61faaca37e1d100e8d2b6ed24d9
|
| 3 |
+
size 5574765
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66871475200,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 103329097728,
|
| 5 |
+
"active_bytes_peak_delta": 2406400,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66871475200,
|
| 8 |
+
"active_bytes_peak": 103329097728,
|
| 9 |
+
"allocated_bytes_current": 66871475200,
|
| 10 |
+
"allocated_bytes_peak": 103329097728,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 3735438336,
|
| 13 |
+
"inactive_split_bytes_peak": 5464060928,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 104421392384,
|
| 21 |
+
"reserved_bytes_peak": 104421392384,
|
| 22 |
+
"segment_count_current": 589,
|
| 23 |
+
"segment_count_peak": 589,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:14.352653Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66871475200,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 103329097728,
|
| 30 |
+
"allocated_bytes_peak_delta": 2406400,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66871475200,
|
| 33 |
+
"active_bytes_peak": 103326691328,
|
| 34 |
+
"allocated_bytes_current": 66871475200,
|
| 35 |
+
"allocated_bytes_peak": 103326691328,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 4033233920,
|
| 38 |
+
"inactive_split_bytes_peak": 5464060928,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 104421392384,
|
| 46 |
+
"reserved_bytes_peak": 104421392384,
|
| 47 |
+
"segment_count_current": 589,
|
| 48 |
+
"segment_count_peak": 589,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:13.558605Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 3735438336,
|
| 55 |
+
"inactive_split_bytes_current_delta": -297795584,
|
| 56 |
+
"inactive_split_bytes_peak": 5464060928,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 104421392384,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 104421392384,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 589,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 589,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:14.453082Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:84cb620613bf3d7a35b8afc61b5ec87a3df4a82cd8a767258abbd7428bca44df
|
| 3 |
+
size 5578752
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 66888518656,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 121477455872,
|
| 5 |
+
"active_bytes_peak_delta": 18148358144,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 66888518656,
|
| 8 |
+
"active_bytes_peak": 121477455872,
|
| 9 |
+
"allocated_bytes_current": 66888518656,
|
| 10 |
+
"allocated_bytes_peak": 103329097728,
|
| 11 |
+
"device": 2,
|
| 12 |
+
"inactive_split_bytes_current": 4125242368,
|
| 13 |
+
"inactive_split_bytes_peak": 7711509504,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 2,
|
| 20 |
+
"reserved_bytes_current": 132124770304,
|
| 21 |
+
"reserved_bytes_peak": 132124770304,
|
| 22 |
+
"segment_count_current": 788,
|
| 23 |
+
"segment_count_peak": 788,
|
| 24 |
+
"timestamp_utc": "2026-08-01T11:40:17.033393Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 66888518656,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 103329097728,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 66871475200,
|
| 33 |
+
"active_bytes_peak": 103329097728,
|
| 34 |
+
"allocated_bytes_current": 66871475200,
|
| 35 |
+
"allocated_bytes_peak": 103329097728,
|
| 36 |
+
"device": 2,
|
| 37 |
+
"inactive_split_bytes_current": 3735438336,
|
| 38 |
+
"inactive_split_bytes_peak": 5464060928,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 2,
|
| 45 |
+
"reserved_bytes_current": 104421392384,
|
| 46 |
+
"reserved_bytes_peak": 104421392384,
|
| 47 |
+
"segment_count_current": 589,
|
| 48 |
+
"segment_count_peak": 589,
|
| 49 |
+
"timestamp_utc": "2026-08-01T11:40:14.465898Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 2,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4125242368,
|
| 55 |
+
"inactive_split_bytes_current_delta": 389804032,
|
| 56 |
+
"inactive_split_bytes_peak": 7711509504,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 2247448576,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 2,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 132124770304,
|
| 68 |
+
"reserved_bytes_current_delta": 27703377920,
|
| 69 |
+
"reserved_bytes_peak": 132124770304,
|
| 70 |
+
"reserved_bytes_peak_delta": 27703377920,
|
| 71 |
+
"segment_count_current": 788,
|
| 72 |
+
"segment_count_current_delta": 199,
|
| 73 |
+
"segment_count_peak": 788,
|
| 74 |
+
"segment_count_peak_delta": 199,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-01T11:40:17.153026Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:57d6fe28e9cba49c9df5310e3ed2d64bf48bf43c5ae103a180cadecbce7c445c
|
| 3 |
+
size 5851567
|