granularity results echo/tierbmem @ 2026-08-26T19:00:15Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 4)
Browse files- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190008909187Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190001924389Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190001924389Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190004404034Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190004404034Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190005872645Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190005872645Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190007540594Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190007540594Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190008909160Z.json +79 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190008909160Z.pickle +3 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env +61 -0
- echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log +0 -0
- echo/tierbmem/tierbmem_gbs8/results.tsv +8 -0
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190008909187Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2ef43813ec882819d9c5154fcccd42885dc9fa209ee5f1de3419f9058a9c772f
|
| 3 |
+
size 9833698
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190001924389Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84716535808,
|
| 3 |
+
"active_bytes_current_delta": 41162992640,
|
| 4 |
+
"active_bytes_peak": 90924607488,
|
| 5 |
+
"active_bytes_peak_delta": 47364772864,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84716535808,
|
| 8 |
+
"active_bytes_peak": 90924607488,
|
| 9 |
+
"allocated_bytes_current": 84716535808,
|
| 10 |
+
"allocated_bytes_peak": 90924607488,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 4565516288,
|
| 13 |
+
"inactive_split_bytes_peak": 13893077504,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 92188704768,
|
| 21 |
+
"reserved_bytes_peak": 92188704768,
|
| 22 |
+
"segment_count_current": 2174,
|
| 23 |
+
"segment_count_peak": 2174,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:00:01.923922Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84716535808,
|
| 28 |
+
"allocated_bytes_current_delta": 41162992640,
|
| 29 |
+
"allocated_bytes_peak": 90924607488,
|
| 30 |
+
"allocated_bytes_peak_delta": 47364772864,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 43553543168,
|
| 33 |
+
"active_bytes_peak": 43559834624,
|
| 34 |
+
"allocated_bytes_current": 43553543168,
|
| 35 |
+
"allocated_bytes_peak": 43559834624,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 1256303616,
|
| 38 |
+
"inactive_split_bytes_peak": 1612912640,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 44835012608,
|
| 46 |
+
"reserved_bytes_peak": 44835012608,
|
| 47 |
+
"segment_count_current": 883,
|
| 48 |
+
"segment_count_peak": 883,
|
| 49 |
+
"timestamp_utc": "2026-08-26T18:59:17.789141Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4565516288,
|
| 55 |
+
"inactive_split_bytes_current_delta": 3309212672,
|
| 56 |
+
"inactive_split_bytes_peak": 13893077504,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 12280164864,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 92188704768,
|
| 68 |
+
"reserved_bytes_current_delta": 47353692160,
|
| 69 |
+
"reserved_bytes_peak": 92188704768,
|
| 70 |
+
"reserved_bytes_peak_delta": 47353692160,
|
| 71 |
+
"segment_count_current": 2174,
|
| 72 |
+
"segment_count_current_delta": 1291,
|
| 73 |
+
"segment_count_peak": 2174,
|
| 74 |
+
"segment_count_peak_delta": 1291,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T190001924389Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:00:02.118613Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190001924389Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d29c6524d76891b5a0d7c1423f896d747b64c506d0b31b843fc036528febacad
|
| 3 |
+
size 9410960
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190004404034Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84716535808,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121729455104,
|
| 5 |
+
"active_bytes_peak_delta": 30804847616,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84716535808,
|
| 8 |
+
"active_bytes_peak": 121729455104,
|
| 9 |
+
"allocated_bytes_current": 84716535808,
|
| 10 |
+
"allocated_bytes_peak": 121729455104,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 4466950144,
|
| 13 |
+
"inactive_split_bytes_peak": 13893077504,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 123025227776,
|
| 21 |
+
"reserved_bytes_peak": 123025227776,
|
| 22 |
+
"segment_count_current": 2884,
|
| 23 |
+
"segment_count_peak": 2884,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:00:04.403587Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84716535808,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121729455104,
|
| 30 |
+
"allocated_bytes_peak_delta": 30804847616,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84716535808,
|
| 33 |
+
"active_bytes_peak": 90924607488,
|
| 34 |
+
"allocated_bytes_current": 84716535808,
|
| 35 |
+
"allocated_bytes_peak": 90924607488,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 4565516288,
|
| 38 |
+
"inactive_split_bytes_peak": 13893077504,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 92188704768,
|
| 46 |
+
"reserved_bytes_peak": 92188704768,
|
| 47 |
+
"segment_count_current": 2174,
|
| 48 |
+
"segment_count_peak": 2174,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:00:02.155810Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4466950144,
|
| 55 |
+
"inactive_split_bytes_current_delta": -98566144,
|
| 56 |
+
"inactive_split_bytes_peak": 13893077504,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 123025227776,
|
| 68 |
+
"reserved_bytes_current_delta": 30836523008,
|
| 69 |
+
"reserved_bytes_peak": 123025227776,
|
| 70 |
+
"reserved_bytes_peak_delta": 30836523008,
|
| 71 |
+
"segment_count_current": 2884,
|
| 72 |
+
"segment_count_current_delta": 710,
|
| 73 |
+
"segment_count_peak": 2884,
|
| 74 |
+
"segment_count_peak_delta": 710,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T190004404034Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:00:04.597988Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190004404034Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:922e23701f2c0bdb75e212d316d6870c3eafb3fc0529d32b216eee161517ee04
|
| 3 |
+
size 9848547
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190005872645Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84716535808,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121729455104,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84716535808,
|
| 8 |
+
"active_bytes_peak": 121729455104,
|
| 9 |
+
"allocated_bytes_current": 84716535808,
|
| 10 |
+
"allocated_bytes_peak": 121729455104,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 4567613440,
|
| 13 |
+
"inactive_split_bytes_peak": 14530429952,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 123025227776,
|
| 21 |
+
"reserved_bytes_peak": 123025227776,
|
| 22 |
+
"segment_count_current": 2884,
|
| 23 |
+
"segment_count_peak": 2884,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:00:05.872191Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84716535808,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121729455104,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84716535808,
|
| 33 |
+
"active_bytes_peak": 121729455104,
|
| 34 |
+
"allocated_bytes_current": 84716535808,
|
| 35 |
+
"allocated_bytes_peak": 121729455104,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 4466950144,
|
| 38 |
+
"inactive_split_bytes_peak": 13893077504,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 123025227776,
|
| 46 |
+
"reserved_bytes_peak": 123025227776,
|
| 47 |
+
"segment_count_current": 2884,
|
| 48 |
+
"segment_count_peak": 2884,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:00:04.624292Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4567613440,
|
| 55 |
+
"inactive_split_bytes_current_delta": 100663296,
|
| 56 |
+
"inactive_split_bytes_peak": 14530429952,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 637352448,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 123025227776,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 123025227776,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2884,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2884,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T190005872645Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:00:06.061453Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190005872645Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ed8e54fc1353be58cba825db31e6273823152b3641ad415bbeec9846b26724d7
|
| 3 |
+
size 9822734
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190007540594Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84716535808,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121729455104,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84716535808,
|
| 8 |
+
"active_bytes_peak": 121729455104,
|
| 9 |
+
"allocated_bytes_current": 84716535808,
|
| 10 |
+
"allocated_bytes_peak": 121729455104,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 4567613440,
|
| 13 |
+
"inactive_split_bytes_peak": 14876004352,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 123025227776,
|
| 21 |
+
"reserved_bytes_peak": 123025227776,
|
| 22 |
+
"segment_count_current": 2884,
|
| 23 |
+
"segment_count_peak": 2884,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:00:07.540156Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84716535808,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121729455104,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84716535808,
|
| 33 |
+
"active_bytes_peak": 121729455104,
|
| 34 |
+
"allocated_bytes_current": 84716535808,
|
| 35 |
+
"allocated_bytes_peak": 121729455104,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 4567613440,
|
| 38 |
+
"inactive_split_bytes_peak": 14530429952,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 123025227776,
|
| 46 |
+
"reserved_bytes_peak": 123025227776,
|
| 47 |
+
"segment_count_current": 2884,
|
| 48 |
+
"segment_count_peak": 2884,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:00:06.086174Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4567613440,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 14876004352,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 345574400,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 123025227776,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 123025227776,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2884,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2884,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T190007540594Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:00:07.722185Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190007540594Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:46722eb9811e1b2fa8f40e3638c6fca731deb707b80f66527f24ea1b8eb402ab
|
| 3 |
+
size 9821502
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190008909160Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 84716535808,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 121729455104,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 84716535808,
|
| 8 |
+
"active_bytes_peak": 121729455104,
|
| 9 |
+
"allocated_bytes_current": 84716535808,
|
| 10 |
+
"allocated_bytes_peak": 121729455104,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 4567613440,
|
| 13 |
+
"inactive_split_bytes_peak": 14876004352,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 123025227776,
|
| 21 |
+
"reserved_bytes_peak": 123025227776,
|
| 22 |
+
"segment_count_current": 2884,
|
| 23 |
+
"segment_count_peak": 2884,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:00:08.908718Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 84716535808,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 121729455104,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 84716535808,
|
| 33 |
+
"active_bytes_peak": 121729455104,
|
| 34 |
+
"allocated_bytes_current": 84716535808,
|
| 35 |
+
"allocated_bytes_peak": 121729455104,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 4567613440,
|
| 38 |
+
"inactive_split_bytes_peak": 14876004352,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 123025227776,
|
| 46 |
+
"reserved_bytes_peak": 123025227776,
|
| 47 |
+
"segment_count_current": 2884,
|
| 48 |
+
"segment_count_peak": 2884,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:00:07.746011Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 4567613440,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 14876004352,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 123025227776,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 123025227776,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 2884,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 2884,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T190008909160Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:00:09.089989Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190008909160Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:65b656243b0bbcc29d107605e347b4e4e67b48a93e70d1348e1ccbfb088375e9
|
| 3 |
+
size 9821229
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=echo
|
| 2 |
+
config=QWEN3_30B
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 5 |
+
branch=better-memory-dump-support
|
| 6 |
+
commit=b7c52442
|
| 7 |
+
num_layers=48
|
| 8 |
+
hidden=2048
|
| 9 |
+
ffn_hidden=6144
|
| 10 |
+
moe_ffn_hidden=768
|
| 11 |
+
num_experts=128
|
| 12 |
+
topk=8
|
| 13 |
+
experts_per_rank=16
|
| 14 |
+
expert_params=4718592
|
| 15 |
+
heads=32
|
| 16 |
+
kv_groups=4
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=0
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=0
|
| 22 |
+
dense_first_layers=0
|
| 23 |
+
moe_layers=48
|
| 24 |
+
rotary_base=1000000
|
| 25 |
+
rotary_scaling=
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=0
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=0
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=4096
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=32768
|
| 39 |
+
mean_tokens_per_expert=2048
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=16
|
| 42 |
+
capacity_mode=dynamic
|
| 43 |
+
capacity=dynamic
|
| 44 |
+
cuda_graph=0
|
| 45 |
+
probe_mode=0
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=memory
|
| 48 |
+
memory_dump_mode=iteration
|
| 49 |
+
memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=0
|
| 53 |
+
train_iters=5
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=0
|
| 60 |
+
kernel_trace=0
|
| 61 |
+
fp8_recipe=mxfp8
|
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
echo/tierbmem/tierbmem_gbs8/results.tsv
CHANGED
|
@@ -1 +1,9 @@
|
|
| 1 |
config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
|
| 2 |
+
QWEN3_235B echo matched 16 8192 8 1.00 none 1 OK 321.20 321.20 145347 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1
|
| 3 |
+
QWEN3_235B echo matched 16 8192 8 1.125 none 1 OK 325.40 325.40 146517 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p125__memiteration__it6/rep1
|
| 4 |
+
QWEN3_235B echo matched 16 8192 8 1.25 none 1 OK 326.70 326.70 148163 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1
|
| 5 |
+
QWEN3_235B echo matched 16 8192 8 dynamic none 1 OK NA NA 131245 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1
|
| 6 |
+
QWEN3_30B echo matched 16 4096 8 1.00 none 1 OK 422.70 422.70 143105 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1
|
| 7 |
+
QWEN3_30B echo matched 16 4096 8 1.125 none 1 OK 425.50 425.50 143679 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1
|
| 8 |
+
QWEN3_30B echo matched 16 4096 8 1.25 none 1 OK 428.30 428.30 147661 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1
|
| 9 |
+
QWEN3_30B echo matched 16 4096 8 dynamic none 1 OK NA NA 124489 /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1
|