AbhishekGhosh1998 commited on
Commit
d006ef3
·
verified ·
1 Parent(s): 78412d4

Upload memory sweep moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS +1 -0
  2. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt +14 -0
  3. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.json +79 -0
  4. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.pickle +3 -0
  5. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.json +79 -0
  6. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.pickle +3 -0
  7. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.json +79 -0
  8. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.pickle +3 -0
  9. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.json +79 -0
  10. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.pickle +3 -0
  11. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.json +79 -0
  12. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.pickle +3 -0
  13. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.json +79 -0
  14. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.pickle +3 -0
  15. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.json +79 -0
  16. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.pickle +3 -0
  17. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.json +79 -0
  18. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.pickle +3 -0
  19. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.json +79 -0
  20. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.pickle +3 -0
  21. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.json +79 -0
  22. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.pickle +3 -0
  23. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.json +79 -0
  24. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.pickle +3 -0
  25. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.json +79 -0
  26. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.pickle +3 -0
  27. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.json +79 -0
  28. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.pickle +3 -0
  29. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.json +79 -0
  30. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.pickle +3 -0
  31. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.json +79 -0
  32. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.pickle +3 -0
  33. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.json +79 -0
  34. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.pickle +3 -0
  35. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.json +79 -0
  36. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.pickle +3 -0
  37. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.json +79 -0
  38. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.pickle +3 -0
  39. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.json +79 -0
  40. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.pickle +3 -0
  41. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.json +79 -0
  42. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.pickle +3 -0
  43. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.json +79 -0
  44. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.pickle +3 -0
  45. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.json +79 -0
  46. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.pickle +3 -0
  47. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.json +79 -0
  48. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.pickle +3 -0
  49. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.json +79 -0
  50. moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.pickle +3 -0
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS ADDED
@@ -0,0 +1 @@
 
 
1
+ SUCCESS
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ cuda_graph: on
2
+ seq_length: 2048
3
+ capacity: 1.00
4
+ expert_dispatch_overlap: off
5
+ expert_dispatch_recompute: off
6
+ global_batch_size: 32
7
+ train_iters: 10
8
+ master_port: 6000
9
+ expected_pickles: 80
10
+ actual_pickles: 80
11
+ status: SUCCESS
12
+ duration_s: 230
13
+ log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off.log
14
+ result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66871475200,
3
+ "active_bytes_current_delta": 31640019968,
4
+ "active_bytes_peak": 79683648512,
5
+ "active_bytes_peak_delta": 44217312256,
6
+ "after": {
7
+ "active_bytes_current": 66871475200,
8
+ "active_bytes_peak": 79683648512,
9
+ "allocated_bytes_current": 66871475200,
10
+ "allocated_bytes_peak": 79683648512,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 4857414656,
13
+ "inactive_split_bytes_peak": 5749855232,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 94378131456,
21
+ "reserved_bytes_peak": 94378131456,
22
+ "segment_count_current": 518,
23
+ "segment_count_peak": 518,
24
+ "timestamp_utc": "2026-08-01T11:40:12.483106Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66871475200,
28
+ "allocated_bytes_current_delta": 31640019968,
29
+ "allocated_bytes_peak": 79683648512,
30
+ "allocated_bytes_peak_delta": 44217312256,
31
+ "before": {
32
+ "active_bytes_current": 35231455232,
33
+ "active_bytes_peak": 35466336256,
34
+ "allocated_bytes_current": 35231455232,
35
+ "allocated_bytes_peak": 35466336256,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 134916096,
38
+ "inactive_split_bytes_peak": 160631808,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 39246102528,
46
+ "reserved_bytes_peak": 39246102528,
47
+ "segment_count_current": 72,
48
+ "segment_count_peak": 72,
49
+ "timestamp_utc": "2026-08-01T11:39:04.476196Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4857414656,
55
+ "inactive_split_bytes_current_delta": 4722498560,
56
+ "inactive_split_bytes_peak": 5749855232,
57
+ "inactive_split_bytes_peak_delta": 5589223424,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 94378131456,
68
+ "reserved_bytes_current_delta": 55132028928,
69
+ "reserved_bytes_peak": 94378131456,
70
+ "reserved_bytes_peak_delta": 55132028928,
71
+ "segment_count_current": 518,
72
+ "segment_count_current_delta": 446,
73
+ "segment_count_peak": 518,
74
+ "segment_count_peak_delta": 446,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:12.592236Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13eb403a964d4aaa0e195ef80d1de987cbc685f88b0a65095ec8c26eff635061
3
+ size 5383895
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66871475200,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 103322287104,
5
+ "active_bytes_peak_delta": 23638638592,
6
+ "after": {
7
+ "active_bytes_current": 66871475200,
8
+ "active_bytes_peak": 103322287104,
9
+ "allocated_bytes_current": 66871475200,
10
+ "allocated_bytes_peak": 103322287104,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 3961930752,
13
+ "inactive_split_bytes_peak": 5749855232,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 104406712320,
21
+ "reserved_bytes_peak": 104406712320,
22
+ "segment_count_current": 591,
23
+ "segment_count_peak": 591,
24
+ "timestamp_utc": "2026-08-01T11:40:13.440234Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66871475200,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103322287104,
30
+ "allocated_bytes_peak_delta": 23638638592,
31
+ "before": {
32
+ "active_bytes_current": 66871475200,
33
+ "active_bytes_peak": 79683648512,
34
+ "allocated_bytes_current": 66871475200,
35
+ "allocated_bytes_peak": 79683648512,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 4857414656,
38
+ "inactive_split_bytes_peak": 5749855232,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 94378131456,
46
+ "reserved_bytes_peak": 94378131456,
47
+ "segment_count_current": 518,
48
+ "segment_count_peak": 518,
49
+ "timestamp_utc": "2026-08-01T11:40:12.611327Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 3961930752,
55
+ "inactive_split_bytes_current_delta": -895483904,
56
+ "inactive_split_bytes_peak": 5749855232,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 104406712320,
68
+ "reserved_bytes_current_delta": 10028580864,
69
+ "reserved_bytes_peak": 104406712320,
70
+ "reserved_bytes_peak_delta": 10028580864,
71
+ "segment_count_current": 591,
72
+ "segment_count_current_delta": 73,
73
+ "segment_count_peak": 591,
74
+ "segment_count_peak_delta": 73,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:13.538384Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7770105a42e873f83f3788b3924e707562035434d844fe87fa2cfc5ae405aaee
3
+ size 5583228
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66871475200,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 103322287104,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66871475200,
8
+ "active_bytes_peak": 103322287104,
9
+ "allocated_bytes_current": 66871475200,
10
+ "allocated_bytes_peak": 103322287104,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 4131800064,
13
+ "inactive_split_bytes_peak": 5749855232,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 104406712320,
21
+ "reserved_bytes_peak": 104406712320,
22
+ "segment_count_current": 591,
23
+ "segment_count_peak": 591,
24
+ "timestamp_utc": "2026-08-01T11:40:14.352612Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66871475200,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103322287104,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66871475200,
33
+ "active_bytes_peak": 103322287104,
34
+ "allocated_bytes_current": 66871475200,
35
+ "allocated_bytes_peak": 103322287104,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 3961930752,
38
+ "inactive_split_bytes_peak": 5749855232,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 104406712320,
46
+ "reserved_bytes_peak": 104406712320,
47
+ "segment_count_current": 591,
48
+ "segment_count_peak": 591,
49
+ "timestamp_utc": "2026-08-01T11:40:13.549825Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4131800064,
55
+ "inactive_split_bytes_current_delta": 169869312,
56
+ "inactive_split_bytes_peak": 5749855232,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 104406712320,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 104406712320,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 591,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 591,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:14.444958Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:900b45d88f1d794f2575cd36e05bc551a33c2bd3d82fbc54ed027bbfdf34e584
3
+ size 5584389
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66888518656,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 121491349504,
5
+ "active_bytes_peak_delta": 18169062400,
6
+ "after": {
7
+ "active_bytes_current": 66888518656,
8
+ "active_bytes_peak": 121491349504,
9
+ "allocated_bytes_current": 66888518656,
10
+ "allocated_bytes_peak": 103336542208,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 4584518656,
13
+ "inactive_split_bytes_peak": 7220541440,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 132521132032,
21
+ "reserved_bytes_peak": 132521132032,
22
+ "segment_count_current": 798,
23
+ "segment_count_peak": 798,
24
+ "timestamp_utc": "2026-08-01T11:40:17.033382Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66888518656,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 103336542208,
30
+ "allocated_bytes_peak_delta": 14255104,
31
+ "before": {
32
+ "active_bytes_current": 66871475200,
33
+ "active_bytes_peak": 103322287104,
34
+ "allocated_bytes_current": 66871475200,
35
+ "allocated_bytes_peak": 103322287104,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 4131800064,
38
+ "inactive_split_bytes_peak": 5749855232,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 104406712320,
46
+ "reserved_bytes_peak": 104406712320,
47
+ "segment_count_current": 591,
48
+ "segment_count_peak": 591,
49
+ "timestamp_utc": "2026-08-01T11:40:14.455675Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4584518656,
55
+ "inactive_split_bytes_current_delta": 452718592,
56
+ "inactive_split_bytes_peak": 7220541440,
57
+ "inactive_split_bytes_peak_delta": 1470686208,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132521132032,
68
+ "reserved_bytes_current_delta": 28114419712,
69
+ "reserved_bytes_peak": 132521132032,
70
+ "reserved_bytes_peak_delta": 28114419712,
71
+ "segment_count_current": 798,
72
+ "segment_count_current_delta": 207,
73
+ "segment_count_peak": 798,
74
+ "segment_count_peak_delta": 207,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:17.150803Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3353b418b19312db946e05d6a17b3e9d5b702632e9c28904515cff97fe2cce6
3
+ size 6274771
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66888518656,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121491349504,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66888518656,
8
+ "active_bytes_peak": 121491349504,
9
+ "allocated_bytes_current": 66888518656,
10
+ "allocated_bytes_peak": 103336542208,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 4584518656,
13
+ "inactive_split_bytes_peak": 7220541440,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 132521132032,
21
+ "reserved_bytes_peak": 132521132032,
22
+ "segment_count_current": 798,
23
+ "segment_count_peak": 798,
24
+ "timestamp_utc": "2026-08-01T11:40:17.864648Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66888518656,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103336542208,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66888518656,
33
+ "active_bytes_peak": 121491349504,
34
+ "allocated_bytes_current": 66888518656,
35
+ "allocated_bytes_peak": 103336542208,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 4584518656,
38
+ "inactive_split_bytes_peak": 7220541440,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 132521132032,
46
+ "reserved_bytes_peak": 132521132032,
47
+ "segment_count_current": 798,
48
+ "segment_count_peak": 798,
49
+ "timestamp_utc": "2026-08-01T11:40:17.164045Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4584518656,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 7220541440,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132521132032,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 132521132032,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 798,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 798,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:17.872215Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6435a311776ea8d99a13db28debafd039dbda3f4b899a139c6a412d0070e9fa7
3
+ size 266048
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66888518656,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121491349504,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66888518656,
8
+ "active_bytes_peak": 121491349504,
9
+ "allocated_bytes_current": 66888518656,
10
+ "allocated_bytes_peak": 103336542208,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 4584518656,
13
+ "inactive_split_bytes_peak": 7220541440,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 132521132032,
21
+ "reserved_bytes_peak": 132521132032,
22
+ "segment_count_current": 798,
23
+ "segment_count_peak": 798,
24
+ "timestamp_utc": "2026-08-01T11:40:18.565796Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66888518656,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103336542208,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66888518656,
33
+ "active_bytes_peak": 121491349504,
34
+ "allocated_bytes_current": 66888518656,
35
+ "allocated_bytes_peak": 103336542208,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 4584518656,
38
+ "inactive_split_bytes_peak": 7220541440,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 132521132032,
46
+ "reserved_bytes_peak": 132521132032,
47
+ "segment_count_current": 798,
48
+ "segment_count_peak": 798,
49
+ "timestamp_utc": "2026-08-01T11:40:17.873174Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4584518656,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 7220541440,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132521132032,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 132521132032,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 798,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 798,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:18.572888Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8e12dcf81fed3acc951c00dde72513e207be4b99b8f85eac0f1e26ccc122d9c8
3
+ size 266049
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66888518656,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121491349504,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66888518656,
8
+ "active_bytes_peak": 121491349504,
9
+ "allocated_bytes_current": 66888518656,
10
+ "allocated_bytes_peak": 103336542208,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 4584518656,
13
+ "inactive_split_bytes_peak": 7220541440,
14
+ "iteration": 6,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 132521132032,
21
+ "reserved_bytes_peak": 132521132032,
22
+ "segment_count_current": 798,
23
+ "segment_count_peak": 798,
24
+ "timestamp_utc": "2026-08-01T11:40:19.267583Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66888518656,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103336542208,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66888518656,
33
+ "active_bytes_peak": 121491349504,
34
+ "allocated_bytes_current": 66888518656,
35
+ "allocated_bytes_peak": 103336542208,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 4584518656,
38
+ "inactive_split_bytes_peak": 7220541440,
39
+ "iteration": 6,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 132521132032,
46
+ "reserved_bytes_peak": 132521132032,
47
+ "segment_count_current": 798,
48
+ "segment_count_peak": 798,
49
+ "timestamp_utc": "2026-08-01T11:40:18.573818Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4584518656,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 7220541440,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 6,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132521132032,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 132521132032,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 798,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 798,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:19.275794Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d3325263acc1c15b76e149c7e8d846bdff5c56966c03cbdc5967e9edf662aaf
3
+ size 266049
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66888518656,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121491349504,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66888518656,
8
+ "active_bytes_peak": 121491349504,
9
+ "allocated_bytes_current": 66888518656,
10
+ "allocated_bytes_peak": 103336542208,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 4584518656,
13
+ "inactive_split_bytes_peak": 7220541440,
14
+ "iteration": 7,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 132521132032,
21
+ "reserved_bytes_peak": 132521132032,
22
+ "segment_count_current": 798,
23
+ "segment_count_peak": 798,
24
+ "timestamp_utc": "2026-08-01T11:40:19.968607Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66888518656,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103336542208,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66888518656,
33
+ "active_bytes_peak": 121491349504,
34
+ "allocated_bytes_current": 66888518656,
35
+ "allocated_bytes_peak": 103336542208,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 4584518656,
38
+ "inactive_split_bytes_peak": 7220541440,
39
+ "iteration": 7,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 132521132032,
46
+ "reserved_bytes_peak": 132521132032,
47
+ "segment_count_current": 798,
48
+ "segment_count_peak": 798,
49
+ "timestamp_utc": "2026-08-01T11:40:19.276766Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4584518656,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 7220541440,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 7,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132521132032,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 132521132032,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 798,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 798,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:19.975851Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:058c23d59cf99ce1d34d3e898fccaa8bd00d8cea459c66f29cbde75e279054b4
3
+ size 266049
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66888518656,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121491349504,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66888518656,
8
+ "active_bytes_peak": 121491349504,
9
+ "allocated_bytes_current": 66888518656,
10
+ "allocated_bytes_peak": 103336542208,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 4584518656,
13
+ "inactive_split_bytes_peak": 7220541440,
14
+ "iteration": 8,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 132521132032,
21
+ "reserved_bytes_peak": 132521132032,
22
+ "segment_count_current": 798,
23
+ "segment_count_peak": 798,
24
+ "timestamp_utc": "2026-08-01T11:40:20.672461Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66888518656,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103336542208,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66888518656,
33
+ "active_bytes_peak": 121491349504,
34
+ "allocated_bytes_current": 66888518656,
35
+ "allocated_bytes_peak": 103336542208,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 4584518656,
38
+ "inactive_split_bytes_peak": 7220541440,
39
+ "iteration": 8,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 132521132032,
46
+ "reserved_bytes_peak": 132521132032,
47
+ "segment_count_current": 798,
48
+ "segment_count_peak": 798,
49
+ "timestamp_utc": "2026-08-01T11:40:19.976811Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4584518656,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 7220541440,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 8,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132521132032,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 132521132032,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 798,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 798,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:20.679449Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:497816d6d6e28441383edc174a3ef28054bf0ac55df61c8863520df789c7791f
3
+ size 266049
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66854884352,
3
+ "active_bytes_current_delta": -33634304,
4
+ "active_bytes_peak": 121491349504,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66854884352,
8
+ "active_bytes_peak": 121491349504,
9
+ "allocated_bytes_current": 66854884352,
10
+ "allocated_bytes_peak": 103336542208,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 2523098112,
13
+ "inactive_split_bytes_peak": 7220541440,
14
+ "iteration": 9,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 132544200704,
21
+ "reserved_bytes_peak": 132544200704,
22
+ "segment_count_current": 800,
23
+ "segment_count_peak": 800,
24
+ "timestamp_utc": "2026-08-01T11:41:13.363803Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66854884352,
28
+ "allocated_bytes_current_delta": -33634304,
29
+ "allocated_bytes_peak": 103336542208,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66888518656,
33
+ "active_bytes_peak": 121491349504,
34
+ "allocated_bytes_current": 66888518656,
35
+ "allocated_bytes_peak": 103336542208,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 4584518656,
38
+ "inactive_split_bytes_peak": 7220541440,
39
+ "iteration": 9,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 132521132032,
46
+ "reserved_bytes_peak": 132521132032,
47
+ "segment_count_current": 798,
48
+ "segment_count_peak": 798,
49
+ "timestamp_utc": "2026-08-01T11:40:20.680382Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2523098112,
55
+ "inactive_split_bytes_current_delta": -2061420544,
56
+ "inactive_split_bytes_peak": 7220541440,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 9,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132544200704,
68
+ "reserved_bytes_current_delta": 23068672,
69
+ "reserved_bytes_peak": 132544200704,
70
+ "reserved_bytes_peak_delta": 23068672,
71
+ "segment_count_current": 800,
72
+ "segment_count_current_delta": 2,
73
+ "segment_count_peak": 800,
74
+ "segment_count_peak_delta": 2,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:41:13.448813Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:691713541afb401c4594240700d7746937afee4116911dc1465e490b845e29e1
3
+ size 4204922
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66871475200,
3
+ "active_bytes_current_delta": 31640019968,
4
+ "active_bytes_peak": 79690286080,
5
+ "active_bytes_peak_delta": 44223949824,
6
+ "after": {
7
+ "active_bytes_current": 66871475200,
8
+ "active_bytes_peak": 79690286080,
9
+ "allocated_bytes_current": 66871475200,
10
+ "allocated_bytes_peak": 79690286080,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 4328932352,
13
+ "inactive_split_bytes_peak": 5229261824,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 94376034304,
21
+ "reserved_bytes_peak": 94376034304,
22
+ "segment_count_current": 520,
23
+ "segment_count_peak": 520,
24
+ "timestamp_utc": "2026-08-01T11:40:12.483218Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66871475200,
28
+ "allocated_bytes_current_delta": 31640019968,
29
+ "allocated_bytes_peak": 79690286080,
30
+ "allocated_bytes_peak_delta": 44223949824,
31
+ "before": {
32
+ "active_bytes_current": 35231455232,
33
+ "active_bytes_peak": 35466336256,
34
+ "allocated_bytes_current": 35231455232,
35
+ "allocated_bytes_peak": 35466336256,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 132818944,
38
+ "inactive_split_bytes_peak": 160386048,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 39244005376,
46
+ "reserved_bytes_peak": 39244005376,
47
+ "segment_count_current": 71,
48
+ "segment_count_peak": 71,
49
+ "timestamp_utc": "2026-08-01T11:39:04.476570Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4328932352,
55
+ "inactive_split_bytes_current_delta": 4196113408,
56
+ "inactive_split_bytes_peak": 5229261824,
57
+ "inactive_split_bytes_peak_delta": 5068875776,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 94376034304,
68
+ "reserved_bytes_current_delta": 55132028928,
69
+ "reserved_bytes_peak": 94376034304,
70
+ "reserved_bytes_peak_delta": 55132028928,
71
+ "segment_count_current": 520,
72
+ "segment_count_current_delta": 449,
73
+ "segment_count_peak": 520,
74
+ "segment_count_peak_delta": 449,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:12.600020Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:81b303ab50c373505e17e5a08d61daf215d49515fb8d51512f5579ea9dc08012
3
+ size 5379054
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66871475200,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 103326691328,
5
+ "active_bytes_peak_delta": 23636405248,
6
+ "after": {
7
+ "active_bytes_current": 66871475200,
8
+ "active_bytes_peak": 103326691328,
9
+ "allocated_bytes_current": 66871475200,
10
+ "allocated_bytes_peak": 103326691328,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 3620094976,
13
+ "inactive_split_bytes_peak": 5229261824,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 104421392384,
21
+ "reserved_bytes_peak": 104421392384,
22
+ "segment_count_current": 590,
23
+ "segment_count_peak": 590,
24
+ "timestamp_utc": "2026-08-01T11:40:13.440274Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66871475200,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103326691328,
30
+ "allocated_bytes_peak_delta": 23636405248,
31
+ "before": {
32
+ "active_bytes_current": 66871475200,
33
+ "active_bytes_peak": 79690286080,
34
+ "allocated_bytes_current": 66871475200,
35
+ "allocated_bytes_peak": 79690286080,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 4328932352,
38
+ "inactive_split_bytes_peak": 5229261824,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 94376034304,
46
+ "reserved_bytes_peak": 94376034304,
47
+ "segment_count_current": 520,
48
+ "segment_count_peak": 520,
49
+ "timestamp_utc": "2026-08-01T11:40:12.617749Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 3620094976,
55
+ "inactive_split_bytes_current_delta": -708837376,
56
+ "inactive_split_bytes_peak": 5229261824,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 104421392384,
68
+ "reserved_bytes_current_delta": 10045358080,
69
+ "reserved_bytes_peak": 104421392384,
70
+ "reserved_bytes_peak_delta": 10045358080,
71
+ "segment_count_current": 590,
72
+ "segment_count_current_delta": 70,
73
+ "segment_count_peak": 590,
74
+ "segment_count_peak_delta": 70,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:13.545141Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c9035cf43d8fc94c39de8f7a9fc59b0c2de1afe3ff5e2e3ec9392788bc29574
3
+ size 5578891
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66871475200,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 103326691328,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66871475200,
8
+ "active_bytes_peak": 103326691328,
9
+ "allocated_bytes_current": 66871475200,
10
+ "allocated_bytes_peak": 103326691328,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 3712369664,
13
+ "inactive_split_bytes_peak": 5229261824,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 104421392384,
21
+ "reserved_bytes_peak": 104421392384,
22
+ "segment_count_current": 590,
23
+ "segment_count_peak": 590,
24
+ "timestamp_utc": "2026-08-01T11:40:14.352684Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66871475200,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103326691328,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66871475200,
33
+ "active_bytes_peak": 103326691328,
34
+ "allocated_bytes_current": 66871475200,
35
+ "allocated_bytes_peak": 103326691328,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 3620094976,
38
+ "inactive_split_bytes_peak": 5229261824,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 104421392384,
46
+ "reserved_bytes_peak": 104421392384,
47
+ "segment_count_current": 590,
48
+ "segment_count_peak": 590,
49
+ "timestamp_utc": "2026-08-01T11:40:13.558680Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 3712369664,
55
+ "inactive_split_bytes_current_delta": 92274688,
56
+ "inactive_split_bytes_peak": 5229261824,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 104421392384,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 104421392384,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 590,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 590,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:14.456557Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:919b3d739ef62345a6435a45506cc04af73b5428803af66af59c963371607c48
3
+ size 5581958
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66888518656,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 121477455872,
5
+ "active_bytes_peak_delta": 18150764544,
6
+ "after": {
7
+ "active_bytes_current": 66888518656,
8
+ "active_bytes_peak": 121477455872,
9
+ "allocated_bytes_current": 66888518656,
10
+ "allocated_bytes_peak": 103326691328,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 4248974336,
13
+ "inactive_split_bytes_peak": 7682149376,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 132101701632,
21
+ "reserved_bytes_peak": 132101701632,
22
+ "segment_count_current": 790,
23
+ "segment_count_peak": 790,
24
+ "timestamp_utc": "2026-08-01T11:40:17.033418Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66888518656,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 103326691328,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66871475200,
33
+ "active_bytes_peak": 103326691328,
34
+ "allocated_bytes_current": 66871475200,
35
+ "allocated_bytes_peak": 103326691328,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 3712369664,
38
+ "inactive_split_bytes_peak": 5229261824,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 104421392384,
46
+ "reserved_bytes_peak": 104421392384,
47
+ "segment_count_current": 590,
48
+ "segment_count_peak": 590,
49
+ "timestamp_utc": "2026-08-01T11:40:14.469703Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4248974336,
55
+ "inactive_split_bytes_current_delta": 536604672,
56
+ "inactive_split_bytes_peak": 7682149376,
57
+ "inactive_split_bytes_peak_delta": 2452887552,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132101701632,
68
+ "reserved_bytes_current_delta": 27680309248,
69
+ "reserved_bytes_peak": 132101701632,
70
+ "reserved_bytes_peak_delta": 27680309248,
71
+ "segment_count_current": 790,
72
+ "segment_count_current_delta": 200,
73
+ "segment_count_peak": 790,
74
+ "segment_count_peak_delta": 200,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:17.153033Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:26ccef36f0e9d2cec4a938fee739eb69e8ff1132ac1575aab7473826237fbc75
3
+ size 5852185
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66888518656,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121477455872,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66888518656,
8
+ "active_bytes_peak": 121477455872,
9
+ "allocated_bytes_current": 66888518656,
10
+ "allocated_bytes_peak": 103326691328,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 4248974336,
13
+ "inactive_split_bytes_peak": 7682149376,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 132101701632,
21
+ "reserved_bytes_peak": 132101701632,
22
+ "segment_count_current": 790,
23
+ "segment_count_peak": 790,
24
+ "timestamp_utc": "2026-08-01T11:40:17.864704Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66888518656,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103326691328,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66888518656,
33
+ "active_bytes_peak": 121477455872,
34
+ "allocated_bytes_current": 66888518656,
35
+ "allocated_bytes_peak": 103326691328,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 4248974336,
38
+ "inactive_split_bytes_peak": 7682149376,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 132101701632,
46
+ "reserved_bytes_peak": 132101701632,
47
+ "segment_count_current": 790,
48
+ "segment_count_peak": 790,
49
+ "timestamp_utc": "2026-08-01T11:40:17.168178Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4248974336,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 7682149376,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132101701632,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 132101701632,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 790,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 790,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:17.872594Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:621a7e04293676d762c0de3bc2668b3b5ae6f5ebe02fb676dcf05f92e211d2dc
3
+ size 243291
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66888518656,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121477455872,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66888518656,
8
+ "active_bytes_peak": 121477455872,
9
+ "allocated_bytes_current": 66888518656,
10
+ "allocated_bytes_peak": 103326691328,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 4248974336,
13
+ "inactive_split_bytes_peak": 7682149376,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 132101701632,
21
+ "reserved_bytes_peak": 132101701632,
22
+ "segment_count_current": 790,
23
+ "segment_count_peak": 790,
24
+ "timestamp_utc": "2026-08-01T11:40:18.566062Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66888518656,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103326691328,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66888518656,
33
+ "active_bytes_peak": 121477455872,
34
+ "allocated_bytes_current": 66888518656,
35
+ "allocated_bytes_peak": 103326691328,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 4248974336,
38
+ "inactive_split_bytes_peak": 7682149376,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 132101701632,
46
+ "reserved_bytes_peak": 132101701632,
47
+ "segment_count_current": 790,
48
+ "segment_count_peak": 790,
49
+ "timestamp_utc": "2026-08-01T11:40:17.873601Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4248974336,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 7682149376,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132101701632,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 132101701632,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 790,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 790,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:18.574762Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:203517b91270fdd338a0320f887d94b09fe296284c61cb7a3d3d215f631adea6
3
+ size 243291
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66888518656,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121477455872,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66888518656,
8
+ "active_bytes_peak": 121477455872,
9
+ "allocated_bytes_current": 66888518656,
10
+ "allocated_bytes_peak": 103326691328,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 4248974336,
13
+ "inactive_split_bytes_peak": 7682149376,
14
+ "iteration": 6,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 132101701632,
21
+ "reserved_bytes_peak": 132101701632,
22
+ "segment_count_current": 790,
23
+ "segment_count_peak": 790,
24
+ "timestamp_utc": "2026-08-01T11:40:19.267574Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66888518656,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103326691328,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66888518656,
33
+ "active_bytes_peak": 121477455872,
34
+ "allocated_bytes_current": 66888518656,
35
+ "allocated_bytes_peak": 103326691328,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 4248974336,
38
+ "inactive_split_bytes_peak": 7682149376,
39
+ "iteration": 6,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 132101701632,
46
+ "reserved_bytes_peak": 132101701632,
47
+ "segment_count_current": 790,
48
+ "segment_count_peak": 790,
49
+ "timestamp_utc": "2026-08-01T11:40:18.575728Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4248974336,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 7682149376,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 6,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132101701632,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 132101701632,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 790,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 790,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:19.274628Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:84daf2896871a2726a78c16d3263a77aa76eb557618b149c658242774e14beba
3
+ size 243291
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66888518656,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121477455872,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66888518656,
8
+ "active_bytes_peak": 121477455872,
9
+ "allocated_bytes_current": 66888518656,
10
+ "allocated_bytes_peak": 103326691328,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 4248974336,
13
+ "inactive_split_bytes_peak": 7682149376,
14
+ "iteration": 7,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 132101701632,
21
+ "reserved_bytes_peak": 132101701632,
22
+ "segment_count_current": 790,
23
+ "segment_count_peak": 790,
24
+ "timestamp_utc": "2026-08-01T11:40:19.968643Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66888518656,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103326691328,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66888518656,
33
+ "active_bytes_peak": 121477455872,
34
+ "allocated_bytes_current": 66888518656,
35
+ "allocated_bytes_peak": 103326691328,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 4248974336,
38
+ "inactive_split_bytes_peak": 7682149376,
39
+ "iteration": 7,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 132101701632,
46
+ "reserved_bytes_peak": 132101701632,
47
+ "segment_count_current": 790,
48
+ "segment_count_peak": 790,
49
+ "timestamp_utc": "2026-08-01T11:40:19.275567Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4248974336,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 7682149376,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 7,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132101701632,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 132101701632,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 790,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 790,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:19.975826Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:491e864bb6d6cebfc7d4d0186fc45e79cd23addbdc1793bbe2e6978bf57b54c5
3
+ size 243291
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66888518656,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121477455872,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66888518656,
8
+ "active_bytes_peak": 121477455872,
9
+ "allocated_bytes_current": 66888518656,
10
+ "allocated_bytes_peak": 103326691328,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 4248974336,
13
+ "inactive_split_bytes_peak": 7682149376,
14
+ "iteration": 8,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 132101701632,
21
+ "reserved_bytes_peak": 132101701632,
22
+ "segment_count_current": 790,
23
+ "segment_count_peak": 790,
24
+ "timestamp_utc": "2026-08-01T11:40:20.672505Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66888518656,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103326691328,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66888518656,
33
+ "active_bytes_peak": 121477455872,
34
+ "allocated_bytes_current": 66888518656,
35
+ "allocated_bytes_peak": 103326691328,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 4248974336,
38
+ "inactive_split_bytes_peak": 7682149376,
39
+ "iteration": 8,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 132101701632,
46
+ "reserved_bytes_peak": 132101701632,
47
+ "segment_count_current": 790,
48
+ "segment_count_peak": 790,
49
+ "timestamp_utc": "2026-08-01T11:40:19.976764Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4248974336,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 7682149376,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 8,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132101701632,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 132101701632,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 790,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 790,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:20.679586Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eafba8982e23688b1a834c2913971f061d8f8840902cd0bbc65619f83b29868c
3
+ size 243291
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66854884352,
3
+ "active_bytes_current_delta": -33634304,
4
+ "active_bytes_peak": 121477455872,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 66854884352,
8
+ "active_bytes_peak": 121477455872,
9
+ "allocated_bytes_current": 66854884352,
10
+ "allocated_bytes_peak": 103326691328,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 2296605696,
13
+ "inactive_split_bytes_peak": 7682149376,
14
+ "iteration": 9,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 132124770304,
21
+ "reserved_bytes_peak": 132124770304,
22
+ "segment_count_current": 792,
23
+ "segment_count_peak": 792,
24
+ "timestamp_utc": "2026-08-01T11:41:13.363810Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66854884352,
28
+ "allocated_bytes_current_delta": -33634304,
29
+ "allocated_bytes_peak": 103326691328,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66888518656,
33
+ "active_bytes_peak": 121477455872,
34
+ "allocated_bytes_current": 66888518656,
35
+ "allocated_bytes_peak": 103326691328,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 4248974336,
38
+ "inactive_split_bytes_peak": 7682149376,
39
+ "iteration": 9,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 132101701632,
46
+ "reserved_bytes_peak": 132101701632,
47
+ "segment_count_current": 790,
48
+ "segment_count_peak": 790,
49
+ "timestamp_utc": "2026-08-01T11:40:20.680576Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2296605696,
55
+ "inactive_split_bytes_current_delta": -1952368640,
56
+ "inactive_split_bytes_peak": 7682149376,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 9,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132124770304,
68
+ "reserved_bytes_current_delta": 23068672,
69
+ "reserved_bytes_peak": 132124770304,
70
+ "reserved_bytes_peak_delta": 23068672,
71
+ "segment_count_current": 792,
72
+ "segment_count_current_delta": 2,
73
+ "segment_count_peak": 792,
74
+ "segment_count_peak_delta": 2,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:41:13.451555Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e2e99df3ac0599cd00e64a13265bc0d4fe58af3467079b99f62d72165aa9eab7
3
+ size 4121826
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66871475200,
3
+ "active_bytes_current_delta": 31640019968,
4
+ "active_bytes_peak": 79690286080,
5
+ "active_bytes_peak_delta": 44223949824,
6
+ "after": {
7
+ "active_bytes_current": 66871475200,
8
+ "active_bytes_peak": 79690286080,
9
+ "allocated_bytes_current": 66871475200,
10
+ "allocated_bytes_peak": 79690286080,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 4328932352,
13
+ "inactive_split_bytes_peak": 5464060928,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 93906272256,
21
+ "reserved_bytes_peak": 93906272256,
22
+ "segment_count_current": 517,
23
+ "segment_count_peak": 517,
24
+ "timestamp_utc": "2026-08-01T11:40:12.483142Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66871475200,
28
+ "allocated_bytes_current_delta": 31640019968,
29
+ "allocated_bytes_peak": 79690286080,
30
+ "allocated_bytes_peak_delta": 44223949824,
31
+ "before": {
32
+ "active_bytes_current": 35231455232,
33
+ "active_bytes_peak": 35466336256,
34
+ "allocated_bytes_current": 35231455232,
35
+ "allocated_bytes_peak": 35466336256,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 132818944,
38
+ "inactive_split_bytes_peak": 160386048,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 39244005376,
46
+ "reserved_bytes_peak": 39244005376,
47
+ "segment_count_current": 71,
48
+ "segment_count_peak": 71,
49
+ "timestamp_utc": "2026-08-01T11:39:04.476596Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4328932352,
55
+ "inactive_split_bytes_current_delta": 4196113408,
56
+ "inactive_split_bytes_peak": 5464060928,
57
+ "inactive_split_bytes_peak_delta": 5303674880,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 93906272256,
68
+ "reserved_bytes_current_delta": 54662266880,
69
+ "reserved_bytes_peak": 93906272256,
70
+ "reserved_bytes_peak_delta": 54662266880,
71
+ "segment_count_current": 517,
72
+ "segment_count_current_delta": 446,
73
+ "segment_count_peak": 517,
74
+ "segment_count_peak_delta": 446,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:12.593828Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eba85767f668dba66ab14017ea6b62d73b66eea84f68ff3cc013c9ba27273d4d
3
+ size 5378522
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66871475200,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 103326691328,
5
+ "active_bytes_peak_delta": 23636405248,
6
+ "after": {
7
+ "active_bytes_current": 66871475200,
8
+ "active_bytes_peak": 103326691328,
9
+ "allocated_bytes_current": 66871475200,
10
+ "allocated_bytes_peak": 103326691328,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 4033233920,
13
+ "inactive_split_bytes_peak": 5464060928,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 104421392384,
21
+ "reserved_bytes_peak": 104421392384,
22
+ "segment_count_current": 589,
23
+ "segment_count_peak": 589,
24
+ "timestamp_utc": "2026-08-01T11:40:13.440247Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66871475200,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103326691328,
30
+ "allocated_bytes_peak_delta": 23636405248,
31
+ "before": {
32
+ "active_bytes_current": 66871475200,
33
+ "active_bytes_peak": 79690286080,
34
+ "allocated_bytes_current": 66871475200,
35
+ "allocated_bytes_peak": 79690286080,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 4328932352,
38
+ "inactive_split_bytes_peak": 5464060928,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 93906272256,
46
+ "reserved_bytes_peak": 93906272256,
47
+ "segment_count_current": 517,
48
+ "segment_count_peak": 517,
49
+ "timestamp_utc": "2026-08-01T11:40:12.611672Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4033233920,
55
+ "inactive_split_bytes_current_delta": -295698432,
56
+ "inactive_split_bytes_peak": 5464060928,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 104421392384,
68
+ "reserved_bytes_current_delta": 10515120128,
69
+ "reserved_bytes_peak": 104421392384,
70
+ "reserved_bytes_peak_delta": 10515120128,
71
+ "segment_count_current": 589,
72
+ "segment_count_current_delta": 72,
73
+ "segment_count_peak": 589,
74
+ "segment_count_peak_delta": 72,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:13.545258Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a2c95ab6ab4b590631e5c6111813d6e9e421b61faaca37e1d100e8d2b6ed24d9
3
+ size 5574765
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66871475200,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 103329097728,
5
+ "active_bytes_peak_delta": 2406400,
6
+ "after": {
7
+ "active_bytes_current": 66871475200,
8
+ "active_bytes_peak": 103329097728,
9
+ "allocated_bytes_current": 66871475200,
10
+ "allocated_bytes_peak": 103329097728,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 3735438336,
13
+ "inactive_split_bytes_peak": 5464060928,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 104421392384,
21
+ "reserved_bytes_peak": 104421392384,
22
+ "segment_count_current": 589,
23
+ "segment_count_peak": 589,
24
+ "timestamp_utc": "2026-08-01T11:40:14.352653Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66871475200,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 103329097728,
30
+ "allocated_bytes_peak_delta": 2406400,
31
+ "before": {
32
+ "active_bytes_current": 66871475200,
33
+ "active_bytes_peak": 103326691328,
34
+ "allocated_bytes_current": 66871475200,
35
+ "allocated_bytes_peak": 103326691328,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 4033233920,
38
+ "inactive_split_bytes_peak": 5464060928,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 104421392384,
46
+ "reserved_bytes_peak": 104421392384,
47
+ "segment_count_current": 589,
48
+ "segment_count_peak": 589,
49
+ "timestamp_utc": "2026-08-01T11:40:13.558605Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 3735438336,
55
+ "inactive_split_bytes_current_delta": -297795584,
56
+ "inactive_split_bytes_peak": 5464060928,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 104421392384,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 104421392384,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 589,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 589,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:14.453082Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:84cb620613bf3d7a35b8afc61b5ec87a3df4a82cd8a767258abbd7428bca44df
3
+ size 5578752
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 66888518656,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 121477455872,
5
+ "active_bytes_peak_delta": 18148358144,
6
+ "after": {
7
+ "active_bytes_current": 66888518656,
8
+ "active_bytes_peak": 121477455872,
9
+ "allocated_bytes_current": 66888518656,
10
+ "allocated_bytes_peak": 103329097728,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 4125242368,
13
+ "inactive_split_bytes_peak": 7711509504,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 132124770304,
21
+ "reserved_bytes_peak": 132124770304,
22
+ "segment_count_current": 788,
23
+ "segment_count_peak": 788,
24
+ "timestamp_utc": "2026-08-01T11:40:17.033393Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 66888518656,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 103329097728,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 66871475200,
33
+ "active_bytes_peak": 103329097728,
34
+ "allocated_bytes_current": 66871475200,
35
+ "allocated_bytes_peak": 103329097728,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 3735438336,
38
+ "inactive_split_bytes_peak": 5464060928,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 104421392384,
46
+ "reserved_bytes_peak": 104421392384,
47
+ "segment_count_current": 589,
48
+ "segment_count_peak": 589,
49
+ "timestamp_utc": "2026-08-01T11:40:14.465898Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4125242368,
55
+ "inactive_split_bytes_current_delta": 389804032,
56
+ "inactive_split_bytes_peak": 7711509504,
57
+ "inactive_split_bytes_peak_delta": 2247448576,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132124770304,
68
+ "reserved_bytes_current_delta": 27703377920,
69
+ "reserved_bytes_peak": 132124770304,
70
+ "reserved_bytes_peak_delta": 27703377920,
71
+ "segment_count_current": 788,
72
+ "segment_count_current_delta": 199,
73
+ "segment_count_peak": 788,
74
+ "segment_count_peak_delta": 199,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.pickle",
76
+ "timestamp_utc": "2026-08-01T11:40:17.153026Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:57d6fe28e9cba49c9df5310e3ed2d64bf48bf43c5ae103a180cadecbce7c445c
3
+ size 5851567