AbhishekGhosh1998 commited on
Commit
c7610d5
·
verified ·
1 Parent(s): 23cfbc7

granularity results echo/tierbmem @ 2026-08-26T19:00:15Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 4)

Browse files
Files changed (14) hide show
  1. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190008909187Z.pickle +3 -0
  2. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190001924389Z.json +79 -0
  3. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190001924389Z.pickle +3 -0
  4. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190004404034Z.json +79 -0
  5. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190004404034Z.pickle +3 -0
  6. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190005872645Z.json +79 -0
  7. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190005872645Z.pickle +3 -0
  8. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190007540594Z.json +79 -0
  9. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190007540594Z.pickle +3 -0
  10. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190008909160Z.json +79 -0
  11. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190008909160Z.pickle +3 -0
  12. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env +61 -0
  13. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log +0 -0
  14. echo/tierbmem/tierbmem_gbs8/results.tsv +8 -0
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190008909187Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ef43813ec882819d9c5154fcccd42885dc9fa209ee5f1de3419f9058a9c772f
3
+ size 9833698
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190001924389Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84716535808,
3
+ "active_bytes_current_delta": 41162992640,
4
+ "active_bytes_peak": 90924607488,
5
+ "active_bytes_peak_delta": 47364772864,
6
+ "after": {
7
+ "active_bytes_current": 84716535808,
8
+ "active_bytes_peak": 90924607488,
9
+ "allocated_bytes_current": 84716535808,
10
+ "allocated_bytes_peak": 90924607488,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 4565516288,
13
+ "inactive_split_bytes_peak": 13893077504,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 92188704768,
21
+ "reserved_bytes_peak": 92188704768,
22
+ "segment_count_current": 2174,
23
+ "segment_count_peak": 2174,
24
+ "timestamp_utc": "2026-08-26T19:00:01.923922Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84716535808,
28
+ "allocated_bytes_current_delta": 41162992640,
29
+ "allocated_bytes_peak": 90924607488,
30
+ "allocated_bytes_peak_delta": 47364772864,
31
+ "before": {
32
+ "active_bytes_current": 43553543168,
33
+ "active_bytes_peak": 43559834624,
34
+ "allocated_bytes_current": 43553543168,
35
+ "allocated_bytes_peak": 43559834624,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1256303616,
38
+ "inactive_split_bytes_peak": 1612912640,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 44835012608,
46
+ "reserved_bytes_peak": 44835012608,
47
+ "segment_count_current": 883,
48
+ "segment_count_peak": 883,
49
+ "timestamp_utc": "2026-08-26T18:59:17.789141Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4565516288,
55
+ "inactive_split_bytes_current_delta": 3309212672,
56
+ "inactive_split_bytes_peak": 13893077504,
57
+ "inactive_split_bytes_peak_delta": 12280164864,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 92188704768,
68
+ "reserved_bytes_current_delta": 47353692160,
69
+ "reserved_bytes_peak": 92188704768,
70
+ "reserved_bytes_peak_delta": 47353692160,
71
+ "segment_count_current": 2174,
72
+ "segment_count_current_delta": 1291,
73
+ "segment_count_peak": 2174,
74
+ "segment_count_peak_delta": 1291,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T190001924389Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:00:02.118613Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190001924389Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d29c6524d76891b5a0d7c1423f896d747b64c506d0b31b843fc036528febacad
3
+ size 9410960
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190004404034Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84716535808,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121729455104,
5
+ "active_bytes_peak_delta": 30804847616,
6
+ "after": {
7
+ "active_bytes_current": 84716535808,
8
+ "active_bytes_peak": 121729455104,
9
+ "allocated_bytes_current": 84716535808,
10
+ "allocated_bytes_peak": 121729455104,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 4466950144,
13
+ "inactive_split_bytes_peak": 13893077504,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 123025227776,
21
+ "reserved_bytes_peak": 123025227776,
22
+ "segment_count_current": 2884,
23
+ "segment_count_peak": 2884,
24
+ "timestamp_utc": "2026-08-26T19:00:04.403587Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84716535808,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121729455104,
30
+ "allocated_bytes_peak_delta": 30804847616,
31
+ "before": {
32
+ "active_bytes_current": 84716535808,
33
+ "active_bytes_peak": 90924607488,
34
+ "allocated_bytes_current": 84716535808,
35
+ "allocated_bytes_peak": 90924607488,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 4565516288,
38
+ "inactive_split_bytes_peak": 13893077504,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 92188704768,
46
+ "reserved_bytes_peak": 92188704768,
47
+ "segment_count_current": 2174,
48
+ "segment_count_peak": 2174,
49
+ "timestamp_utc": "2026-08-26T19:00:02.155810Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4466950144,
55
+ "inactive_split_bytes_current_delta": -98566144,
56
+ "inactive_split_bytes_peak": 13893077504,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 123025227776,
68
+ "reserved_bytes_current_delta": 30836523008,
69
+ "reserved_bytes_peak": 123025227776,
70
+ "reserved_bytes_peak_delta": 30836523008,
71
+ "segment_count_current": 2884,
72
+ "segment_count_current_delta": 710,
73
+ "segment_count_peak": 2884,
74
+ "segment_count_peak_delta": 710,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T190004404034Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:00:04.597988Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190004404034Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:922e23701f2c0bdb75e212d316d6870c3eafb3fc0529d32b216eee161517ee04
3
+ size 9848547
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190005872645Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84716535808,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121729455104,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 84716535808,
8
+ "active_bytes_peak": 121729455104,
9
+ "allocated_bytes_current": 84716535808,
10
+ "allocated_bytes_peak": 121729455104,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 4567613440,
13
+ "inactive_split_bytes_peak": 14530429952,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 123025227776,
21
+ "reserved_bytes_peak": 123025227776,
22
+ "segment_count_current": 2884,
23
+ "segment_count_peak": 2884,
24
+ "timestamp_utc": "2026-08-26T19:00:05.872191Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84716535808,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121729455104,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84716535808,
33
+ "active_bytes_peak": 121729455104,
34
+ "allocated_bytes_current": 84716535808,
35
+ "allocated_bytes_peak": 121729455104,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 4466950144,
38
+ "inactive_split_bytes_peak": 13893077504,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 123025227776,
46
+ "reserved_bytes_peak": 123025227776,
47
+ "segment_count_current": 2884,
48
+ "segment_count_peak": 2884,
49
+ "timestamp_utc": "2026-08-26T19:00:04.624292Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4567613440,
55
+ "inactive_split_bytes_current_delta": 100663296,
56
+ "inactive_split_bytes_peak": 14530429952,
57
+ "inactive_split_bytes_peak_delta": 637352448,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 123025227776,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 123025227776,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2884,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2884,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T190005872645Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:00:06.061453Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190005872645Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ed8e54fc1353be58cba825db31e6273823152b3641ad415bbeec9846b26724d7
3
+ size 9822734
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190007540594Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84716535808,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121729455104,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 84716535808,
8
+ "active_bytes_peak": 121729455104,
9
+ "allocated_bytes_current": 84716535808,
10
+ "allocated_bytes_peak": 121729455104,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 4567613440,
13
+ "inactive_split_bytes_peak": 14876004352,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 123025227776,
21
+ "reserved_bytes_peak": 123025227776,
22
+ "segment_count_current": 2884,
23
+ "segment_count_peak": 2884,
24
+ "timestamp_utc": "2026-08-26T19:00:07.540156Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84716535808,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121729455104,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84716535808,
33
+ "active_bytes_peak": 121729455104,
34
+ "allocated_bytes_current": 84716535808,
35
+ "allocated_bytes_peak": 121729455104,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 4567613440,
38
+ "inactive_split_bytes_peak": 14530429952,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 123025227776,
46
+ "reserved_bytes_peak": 123025227776,
47
+ "segment_count_current": 2884,
48
+ "segment_count_peak": 2884,
49
+ "timestamp_utc": "2026-08-26T19:00:06.086174Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4567613440,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 14876004352,
57
+ "inactive_split_bytes_peak_delta": 345574400,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 123025227776,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 123025227776,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2884,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2884,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T190007540594Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:00:07.722185Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190007540594Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46722eb9811e1b2fa8f40e3638c6fca731deb707b80f66527f24ea1b8eb402ab
3
+ size 9821502
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190008909160Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84716535808,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 121729455104,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 84716535808,
8
+ "active_bytes_peak": 121729455104,
9
+ "allocated_bytes_current": 84716535808,
10
+ "allocated_bytes_peak": 121729455104,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 4567613440,
13
+ "inactive_split_bytes_peak": 14876004352,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 123025227776,
21
+ "reserved_bytes_peak": 123025227776,
22
+ "segment_count_current": 2884,
23
+ "segment_count_peak": 2884,
24
+ "timestamp_utc": "2026-08-26T19:00:08.908718Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84716535808,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 121729455104,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84716535808,
33
+ "active_bytes_peak": 121729455104,
34
+ "allocated_bytes_current": 84716535808,
35
+ "allocated_bytes_peak": 121729455104,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 4567613440,
38
+ "inactive_split_bytes_peak": 14876004352,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 123025227776,
46
+ "reserved_bytes_peak": 123025227776,
47
+ "segment_count_current": 2884,
48
+ "segment_count_peak": 2884,
49
+ "timestamp_utc": "2026-08-26T19:00:07.746011Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4567613440,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 14876004352,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 123025227776,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 123025227776,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2884,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2884,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T190008909160Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:00:09.089989Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190008909160Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:65b656243b0bbcc29d107605e347b4e4e67b48a93e70d1348e1ccbfb088375e9
3
+ size 9821229
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=QWEN3_30B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=better-memory-dump-support
6
+ commit=b7c52442
7
+ num_layers=48
8
+ hidden=2048
9
+ ffn_hidden=6144
10
+ moe_ffn_hidden=768
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=4718592
15
+ heads=32
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=48
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=0
33
+ norm_epsilon=1e-6
34
+ seq_length=4096
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=32768
39
+ mean_tokens_per_expert=2048
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=dynamic
43
+ capacity=dynamic
44
+ cuda_graph=0
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=memory
48
+ memory_dump_mode=iteration
49
+ memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=5
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
echo/tierbmem/tierbmem_gbs8/results.tsv CHANGED
@@ -1 +1,9 @@
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
 
 
 
 
 
 
 
 
 
1
  config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
2
+ QWEN3_235B echo matched 16 8192 8 1.00 none 1 OK 321.20 321.20 145347 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1
3
+ QWEN3_235B echo matched 16 8192 8 1.125 none 1 OK 325.40 325.40 146517 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1
4
+ QWEN3_235B echo matched 16 8192 8 1.25 none 1 OK 326.70 326.70 148163 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1
5
+ QWEN3_235B echo matched 16 8192 8 dynamic none 1 OK NA NA 131245 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1
6
+ QWEN3_30B echo matched 16 4096 8 1.00 none 1 OK 422.70 422.70 143105 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1
7
+ QWEN3_30B echo matched 16 4096 8 1.125 none 1 OK 425.50 425.50 143679 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1
8
+ QWEN3_30B echo matched 16 4096 8 1.25 none 1 OK 428.30 428.30 147661 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1
9
+ QWEN3_30B echo matched 16 4096 8 dynamic none 1 OK NA NA 124489 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1