AbhishekGhosh1998 commited on
Commit
5eb41b2
·
verified ·
1 Parent(s): bc3a4bb

granularity results ultraep/tierbmem @ 2026-08-26T19:21:12Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 4)

Browse files
Files changed (17) hide show
  1. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192011782232Z.json +79 -0
  2. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192011782232Z.pickle +3 -0
  3. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192012835732Z.json +79 -0
  4. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192012835732Z.pickle +3 -0
  5. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192007567550Z.json +79 -0
  6. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192007567550Z.pickle +3 -0
  7. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192009539886Z.json +79 -0
  8. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192009539886Z.pickle +3 -0
  9. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192010714051Z.json +79 -0
  10. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192010714051Z.pickle +3 -0
  11. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192011782482Z.json +79 -0
  12. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192011782482Z.pickle +3 -0
  13. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192012835943Z.json +79 -0
  14. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192012835943Z.pickle +3 -0
  15. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env +61 -0
  16. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log +0 -0
  17. ultraep/tierbmem/tierbmem_gbs8/results.tsv +8 -0
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192011782232Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 76000535552,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 112732114432,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 76000535552,
8
+ "active_bytes_peak": 112732114432,
9
+ "allocated_bytes_current": 76000535552,
10
+ "allocated_bytes_peak": 112732114432,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 4091796480,
13
+ "inactive_split_bytes_peak": 15975386112,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 113170710528,
21
+ "reserved_bytes_peak": 113170710528,
22
+ "segment_count_current": 2765,
23
+ "segment_count_peak": 2765,
24
+ "timestamp_utc": "2026-08-26T19:20:11.781814Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 76000535552,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 112732114432,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 76000535552,
33
+ "active_bytes_peak": 112732114432,
34
+ "allocated_bytes_current": 76000535552,
35
+ "allocated_bytes_peak": 112732114432,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 4091796480,
38
+ "inactive_split_bytes_peak": 14970116096,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 113170710528,
46
+ "reserved_bytes_peak": 113170710528,
47
+ "segment_count_current": 2765,
48
+ "segment_count_peak": 2765,
49
+ "timestamp_utc": "2026-08-26T19:20:10.856506Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4091796480,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 15975386112,
57
+ "inactive_split_bytes_peak_delta": 1005270016,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113170710528,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 113170710528,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2765,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2765,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T192011782232Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:20:11.906915Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192011782232Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f9838581342276b9c162cf4a7b7546397f8959dd8f3ba6181b675bd5c6c8aaa0
3
+ size 7089973
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192012835732Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 76000535552,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 112732114432,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 76000535552,
8
+ "active_bytes_peak": 112732114432,
9
+ "allocated_bytes_current": 76000535552,
10
+ "allocated_bytes_peak": 112732114432,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 4091796480,
13
+ "inactive_split_bytes_peak": 15975386112,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 113170710528,
21
+ "reserved_bytes_peak": 113170710528,
22
+ "segment_count_current": 2765,
23
+ "segment_count_peak": 2765,
24
+ "timestamp_utc": "2026-08-26T19:20:12.835325Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 76000535552,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 112732114432,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 76000535552,
33
+ "active_bytes_peak": 112732114432,
34
+ "allocated_bytes_current": 76000535552,
35
+ "allocated_bytes_peak": 112732114432,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 4091796480,
38
+ "inactive_split_bytes_peak": 15975386112,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 113170710528,
46
+ "reserved_bytes_peak": 113170710528,
47
+ "segment_count_current": 2765,
48
+ "segment_count_peak": 2765,
49
+ "timestamp_utc": "2026-08-26T19:20:11.922365Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4091796480,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 15975386112,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113170710528,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 113170710528,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2765,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2765,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T192012835732Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:20:12.966943Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192012835732Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:757b50ebefe28ced1e1146b679a1ff53b36e1fd81159288cd99c4525ec40d465
3
+ size 7089530
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192007567550Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75985415168,
3
+ "active_bytes_current_delta": 32431245312,
4
+ "active_bytes_peak": 81892606976,
5
+ "active_bytes_peak_delta": 38332145664,
6
+ "after": {
7
+ "active_bytes_current": 75985415168,
8
+ "active_bytes_peak": 81892606976,
9
+ "allocated_bytes_current": 75985415168,
10
+ "allocated_bytes_peak": 81892606976,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 4234843136,
13
+ "inactive_split_bytes_peak": 13245673472,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 82317410304,
21
+ "reserved_bytes_peak": 82317410304,
22
+ "segment_count_current": 2042,
23
+ "segment_count_peak": 2042,
24
+ "timestamp_utc": "2026-08-26T19:20:07.567056Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75985415168,
28
+ "allocated_bytes_current_delta": 32431245312,
29
+ "allocated_bytes_peak": 81892606976,
30
+ "allocated_bytes_peak_delta": 38332145664,
31
+ "before": {
32
+ "active_bytes_current": 43554169856,
33
+ "active_bytes_peak": 43560461312,
34
+ "allocated_bytes_current": 43554169856,
35
+ "allocated_bytes_peak": 43560461312,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1255676928,
38
+ "inactive_split_bytes_peak": 1612285952,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 44835012608,
46
+ "reserved_bytes_peak": 44835012608,
47
+ "segment_count_current": 883,
48
+ "segment_count_peak": 883,
49
+ "timestamp_utc": "2026-08-26T19:19:39.070373Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4234843136,
55
+ "inactive_split_bytes_current_delta": 2979166208,
56
+ "inactive_split_bytes_peak": 13245673472,
57
+ "inactive_split_bytes_peak_delta": 11633387520,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 82317410304,
68
+ "reserved_bytes_current_delta": 37482397696,
69
+ "reserved_bytes_peak": 82317410304,
70
+ "reserved_bytes_peak_delta": 37482397696,
71
+ "segment_count_current": 2042,
72
+ "segment_count_current_delta": 1159,
73
+ "segment_count_peak": 2042,
74
+ "segment_count_peak_delta": 1159,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T192007567550Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:20:07.703056Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192007567550Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:56d4a3bf31c687a59019c181fe99c388241d50779613a349e4d9a40b6d9854b8
3
+ size 6879462
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192009539886Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75985415168,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 112694591488,
5
+ "active_bytes_peak_delta": 30801984512,
6
+ "after": {
7
+ "active_bytes_current": 75985415168,
8
+ "active_bytes_peak": 112694591488,
9
+ "allocated_bytes_current": 75985415168,
10
+ "allocated_bytes_peak": 112694591488,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 4234843136,
13
+ "inactive_split_bytes_peak": 13245673472,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 113179099136,
21
+ "reserved_bytes_peak": 113179099136,
22
+ "segment_count_current": 2733,
23
+ "segment_count_peak": 2733,
24
+ "timestamp_utc": "2026-08-26T19:20:09.539414Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75985415168,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 112694591488,
30
+ "allocated_bytes_peak_delta": 30801984512,
31
+ "before": {
32
+ "active_bytes_current": 75985415168,
33
+ "active_bytes_peak": 81892606976,
34
+ "allocated_bytes_current": 75985415168,
35
+ "allocated_bytes_peak": 81892606976,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 4234843136,
38
+ "inactive_split_bytes_peak": 13245673472,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 82317410304,
46
+ "reserved_bytes_peak": 82317410304,
47
+ "segment_count_current": 2042,
48
+ "segment_count_peak": 2042,
49
+ "timestamp_utc": "2026-08-26T19:20:07.723737Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4234843136,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13245673472,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113179099136,
68
+ "reserved_bytes_current_delta": 30861688832,
69
+ "reserved_bytes_peak": 113179099136,
70
+ "reserved_bytes_peak_delta": 30861688832,
71
+ "segment_count_current": 2733,
72
+ "segment_count_current_delta": 691,
73
+ "segment_count_peak": 2733,
74
+ "segment_count_peak_delta": 691,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T192009539886Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:20:09.675108Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192009539886Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:367a06fba3055ac810e3611c1bec6ce9ebc804b39eed83a348e25f043a9f2839
3
+ size 7113304
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192010714051Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75985415168,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 112694591488,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75985415168,
8
+ "active_bytes_peak": 112694591488,
9
+ "allocated_bytes_current": 75985415168,
10
+ "allocated_bytes_peak": 112694591488,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 4234843136,
13
+ "inactive_split_bytes_peak": 14533562368,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 113179099136,
21
+ "reserved_bytes_peak": 113179099136,
22
+ "segment_count_current": 2733,
23
+ "segment_count_peak": 2733,
24
+ "timestamp_utc": "2026-08-26T19:20:10.713592Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75985415168,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 112694591488,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75985415168,
33
+ "active_bytes_peak": 112694591488,
34
+ "allocated_bytes_current": 75985415168,
35
+ "allocated_bytes_peak": 112694591488,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 4234843136,
38
+ "inactive_split_bytes_peak": 13245673472,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 113179099136,
46
+ "reserved_bytes_peak": 113179099136,
47
+ "segment_count_current": 2733,
48
+ "segment_count_peak": 2733,
49
+ "timestamp_utc": "2026-08-26T19:20:09.691456Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4234843136,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 14533562368,
57
+ "inactive_split_bytes_peak_delta": 1287888896,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113179099136,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 113179099136,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2733,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2733,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T192010714051Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:20:10.845013Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192010714051Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f4f68264259ae7b5ea43c2dd7b7662c48fc98f26ba615d185626ae0913a0884
3
+ size 7074711
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192011782482Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75985415168,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 112694591488,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75985415168,
8
+ "active_bytes_peak": 112694591488,
9
+ "allocated_bytes_current": 75985415168,
10
+ "allocated_bytes_peak": 112694591488,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 4234843136,
13
+ "inactive_split_bytes_peak": 14899931136,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 113179099136,
21
+ "reserved_bytes_peak": 113179099136,
22
+ "segment_count_current": 2733,
23
+ "segment_count_peak": 2733,
24
+ "timestamp_utc": "2026-08-26T19:20:11.782011Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75985415168,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 112694591488,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75985415168,
33
+ "active_bytes_peak": 112694591488,
34
+ "allocated_bytes_current": 75985415168,
35
+ "allocated_bytes_peak": 112694591488,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 4234843136,
38
+ "inactive_split_bytes_peak": 14533562368,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 113179099136,
46
+ "reserved_bytes_peak": 113179099136,
47
+ "segment_count_current": 2733,
48
+ "segment_count_peak": 2733,
49
+ "timestamp_utc": "2026-08-26T19:20:10.860754Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4234843136,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 14899931136,
57
+ "inactive_split_bytes_peak_delta": 366368768,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113179099136,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 113179099136,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2733,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2733,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T192011782482Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:20:11.914163Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192011782482Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f76952e21e735d7c7699ecdbdc0db523d6e05c2fe04e373d5f20bde68b07089c
3
+ size 7073460
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192012835943Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75985415168,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 112694591488,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75985415168,
8
+ "active_bytes_peak": 112694591488,
9
+ "allocated_bytes_current": 75985415168,
10
+ "allocated_bytes_peak": 112694591488,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 4234843136,
13
+ "inactive_split_bytes_peak": 15641051136,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 113179099136,
21
+ "reserved_bytes_peak": 113179099136,
22
+ "segment_count_current": 2733,
23
+ "segment_count_peak": 2733,
24
+ "timestamp_utc": "2026-08-26T19:20:12.835501Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75985415168,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 112694591488,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75985415168,
33
+ "active_bytes_peak": 112694591488,
34
+ "allocated_bytes_current": 75985415168,
35
+ "allocated_bytes_peak": 112694591488,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 4234843136,
38
+ "inactive_split_bytes_peak": 14899931136,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 113179099136,
46
+ "reserved_bytes_peak": 113179099136,
47
+ "segment_count_current": 2733,
48
+ "segment_count_peak": 2733,
49
+ "timestamp_utc": "2026-08-26T19:20:11.930240Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4234843136,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 15641051136,
57
+ "inactive_split_bytes_peak_delta": 741120000,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113179099136,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 113179099136,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2733,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2733,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T192012835943Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:20:12.964841Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192012835943Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2406225f221a5445829fd208699e27b38ea629b0de6006b846b2c4a78180e00d
3
+ size 7073201
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-better-memory-dump-support
6
+ commit=c3a8a8a7
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=0
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=dynamic
43
+ capacity=dynamic
44
+ cuda_graph=0
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=memory
48
+ memory_dump_mode=iteration
49
+ memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=5
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=256
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
ultraep/tierbmem/tierbmem_gbs8/results.tsv CHANGED
@@ -1 +1,9 @@
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
 
 
 
 
 
 
 
 
 
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
+ QWEN3_235B ultraep matched 16 8192 8 1.00 none 1 OK 308.70 308.70 121321 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1
3
+ QWEN3_235B ultraep matched 16 8192 8 1.125 none 1 OK 310.90 310.90 122445 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1
4
+ QWEN3_235B ultraep matched 16 8192 8 1.25 none 1 OK 312.40 312.40 123775 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1
5
+ QWEN3_235B ultraep matched 16 8192 8 dynamic none 1 OK NA NA 120703 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1
6
+ QWEN3_30B ultraep matched 16 4096 8 1.00 none 1 OK 375.70 375.70 118839 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1
7
+ QWEN3_30B ultraep matched 16 4096 8 1.125 none 1 OK 381.00 381.00 121803 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1
8
+ QWEN3_30B ultraep matched 16 4096 8 1.25 none 1 OK 382.90 382.90 124053 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1
9
+ QWEN3_30B ultraep matched 16 4096 8 dynamic none 1 OK NA NA 116131 /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1