diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193255560438Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193255560438Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4eb98d750b5bb39aa65e67770d35c069fd69979b --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193255560438Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110102684160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 123471372288, + "device": 5, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6235999744, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 127089508352, + "reserved_bytes_peak": 127089508352, + "segment_count_current": 3174, + "segment_count_peak": 3174, + "timestamp_utc": "2026-08-26T19:32:55.560045Z", + "world_size": 8 + }, + "allocated_bytes_current": 110102684160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 123471372288, + "device": 5, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6235999744, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 127089508352, + "reserved_bytes_peak": 127089508352, + "segment_count_current": 3174, + "segment_count_peak": 3174, + "timestamp_utc": "2026-08-26T19:32:55.230570Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6235999744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 127089508352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127089508352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3174, + "segment_count_current_delta": 0, + "segment_count_peak": 3174, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T193255560438Z.pickle", + "timestamp_utc": "2026-08-26T19:32:55.597081Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193255560438Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193255560438Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a04a0cc2aeff2939467ba2d83dea37514f00b95f --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193255560438Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71f6d38654d73698e032165a2b69b11e77bf3290fce784b74d8f93cecb2b8e4e +size 1776965 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193256516948Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193256516948Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f76427ecbddfd4cfddecb10c1bcfb02019c63b9a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193256516948Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110119727616, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 5, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6235999744, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:56.516514Z", + "world_size": 8 + }, + "allocated_bytes_current": 110119727616, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 123471372288, + "device": 5, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6235999744, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 127089508352, + "reserved_bytes_peak": 127089508352, + "segment_count_current": 3174, + "segment_count_peak": 3174, + "timestamp_utc": "2026-08-26T19:32:55.599964Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_current_delta": 494661632, + "inactive_split_bytes_peak": 6235999744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 128417005568, + "reserved_bytes_current_delta": 1327497216, + "reserved_bytes_peak": 128417005568, + "reserved_bytes_peak_delta": 1327497216, + "segment_count_current": 3208, + "segment_count_current_delta": 34, + "segment_count_peak": 3208, + "segment_count_peak_delta": 34, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T193256516948Z.pickle", + "timestamp_utc": "2026-08-26T19:32:56.559462Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193256516948Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193256516948Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b65b17c18dc9ac5d6483464238868c6d5d2c6c37 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193256516948Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf2316fc07c08ca26d5886724359daa170f3baf81f48450bbae7e8e9a37812a5 +size 1817190 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193256893486Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193256893486Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1771284bc2abc86eab1a7ca5a6594586b99a6fbf --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193256893486Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110119727616, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 5, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6235999744, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:56.893083Z", + "world_size": 8 + }, + "allocated_bytes_current": 110119727616, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 5, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6235999744, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:56.562540Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6235999744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 128417005568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128417005568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3208, + "segment_count_current_delta": 0, + "segment_count_peak": 3208, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T193256893486Z.pickle", + "timestamp_utc": "2026-08-26T19:32:56.920770Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193256893486Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193256893486Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3cbd1efb0395de7e8bd913c72c7b482f2b184aae --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193256893486Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a0bd176afb041f2b2558aca33d1c4abbef0a1ea342c3ced8251062fa673f86e +size 1143763 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T193257250503Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T193257250503Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e924507e144b65557f33014e039c1af339407d28 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T193257250503Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110119727616, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 5, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6235999744, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:57.250103Z", + "world_size": 8 + }, + "allocated_bytes_current": 110119727616, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 5, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6235999744, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:56.922581Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6235999744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 128417005568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128417005568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3208, + "segment_count_current_delta": 0, + "segment_count_peak": 3208, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T193257250503Z.pickle", + "timestamp_utc": "2026-08-26T19:32:57.276265Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T193257250503Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T193257250503Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..87269da54836e659ee4f33d487e01af4bcad31a6 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T193257250503Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc22c318d697819977295cbb142f270e6ab9e5e6bb6f7a5885698635bc996882 +size 1143763 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193254800315Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193254800315Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad0fa191d7ce768258c521b584ac6341487c3ada --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193254800315Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110102684160, + "active_bytes_current_delta": 68305225216, + "active_bytes_peak": 110102687232, + "active_bytes_peak_delta": 68267479552, + "after": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 110102687232, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 110102687232, + "device": 6, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6237928960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113837604864, + "reserved_bytes_peak": 113837604864, + "segment_count_current": 3126, + "segment_count_peak": 3126, + "timestamp_utc": "2026-08-26T19:32:54.799733Z", + "world_size": 8 + }, + "allocated_bytes_current": 110102684160, + "allocated_bytes_current_delta": 68305225216, + "allocated_bytes_peak": 110102687232, + "allocated_bytes_peak_delta": 68267479552, + "before": { + "active_bytes_current": 41797458944, + "active_bytes_peak": 41835207680, + "allocated_bytes_current": 41797458944, + "allocated_bytes_peak": 41835207680, + "device": 6, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 44069552128, + "reserved_bytes_peak": 44069552128, + "segment_count_current": 249, + "segment_count_peak": 249, + "timestamp_utc": "2026-08-26T19:32:32.572558Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_current_delta": 3155229184, + "inactive_split_bytes_peak": 6237928960, + "inactive_split_bytes_peak_delta": 5509339648, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113837604864, + "reserved_bytes_current_delta": 69768052736, + "reserved_bytes_peak": 113837604864, + "reserved_bytes_peak_delta": 69768052736, + "segment_count_current": 3126, + "segment_count_current_delta": 2877, + "segment_count_peak": 3126, + "segment_count_peak_delta": 2877, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T193254800315Z.pickle", + "timestamp_utc": "2026-08-26T19:32:54.853529Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193254800315Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193254800315Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1bf17cb9137a0b13825f4cbe45dbc53f980fca95 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193254800315Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d31f4f1189308978c55c1d88209ae5971a461cd50fb50b6f56eec80d5f5206f7 +size 2260290 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193255191513Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193255191513Z.json new file mode 100644 index 0000000000000000000000000000000000000000..14eee78ab9d5da72088690a4d12f3a7400faa0b7 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193255191513Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110102684160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 13368685056, + "after": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 123471372288, + "device": 6, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6237928960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 127089508352, + "reserved_bytes_peak": 127089508352, + "segment_count_current": 3174, + "segment_count_peak": 3174, + "timestamp_utc": "2026-08-26T19:32:55.191094Z", + "world_size": 8 + }, + "allocated_bytes_current": 110102684160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 13368685056, + "before": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 110102687232, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 110102687232, + "device": 6, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6237928960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113837604864, + "reserved_bytes_peak": 113837604864, + "segment_count_current": 3126, + "segment_count_peak": 3126, + "timestamp_utc": "2026-08-26T19:32:54.857514Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6237928960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 127089508352, + "reserved_bytes_current_delta": 13251903488, + "reserved_bytes_peak": 127089508352, + "reserved_bytes_peak_delta": 13251903488, + "segment_count_current": 3174, + "segment_count_current_delta": 48, + "segment_count_peak": 3174, + "segment_count_peak_delta": 48, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T193255191513Z.pickle", + "timestamp_utc": "2026-08-26T19:32:55.228588Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193255191513Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193255191513Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f07ef499d8832b7cf6792f326346c64c38723ee3 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193255191513Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05b46fb2e9332de92f3e0c2e64efb110c9dd5c287150f1f1f1d3884a8ba0722c +size 1776608 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193255560433Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193255560433Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7fdc1825eda92531d2c416beba868bab7696e704 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193255560433Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110102684160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 123471372288, + "device": 6, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6237928960, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 127089508352, + "reserved_bytes_peak": 127089508352, + "segment_count_current": 3174, + "segment_count_peak": 3174, + "timestamp_utc": "2026-08-26T19:32:55.560038Z", + "world_size": 8 + }, + "allocated_bytes_current": 110102684160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 123471372288, + "device": 6, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6237928960, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 127089508352, + "reserved_bytes_peak": 127089508352, + "segment_count_current": 3174, + "segment_count_peak": 3174, + "timestamp_utc": "2026-08-26T19:32:55.231551Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6237928960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 127089508352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127089508352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3174, + "segment_count_current_delta": 0, + "segment_count_peak": 3174, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T193255560433Z.pickle", + "timestamp_utc": "2026-08-26T19:32:55.597777Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193255560433Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193255560433Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c991a3108c1cc0dad127290029c86846c3e8fabb --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193255560433Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ec73bcde03f98ab6f0a0750c7f2dfa9da92959b684e1c722240936bae074aec +size 1776965 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193256516947Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193256516947Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67cbf675c530046b0a2d8e6e388474780f0602a6 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193256516947Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110119727616, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 6, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6237928960, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:56.516530Z", + "world_size": 8 + }, + "allocated_bytes_current": 110119727616, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 123471372288, + "device": 6, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6237928960, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 127089508352, + "reserved_bytes_peak": 127089508352, + "segment_count_current": 3174, + "segment_count_peak": 3174, + "timestamp_utc": "2026-08-26T19:32:55.600611Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_current_delta": 494661632, + "inactive_split_bytes_peak": 6237928960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 128417005568, + "reserved_bytes_current_delta": 1327497216, + "reserved_bytes_peak": 128417005568, + "reserved_bytes_peak_delta": 1327497216, + "segment_count_current": 3208, + "segment_count_current_delta": 34, + "segment_count_peak": 3208, + "segment_count_peak_delta": 34, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T193256516947Z.pickle", + "timestamp_utc": "2026-08-26T19:32:56.560940Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193256516947Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193256516947Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7d66801a2b539fdd377b0e30ebadb02deaf4bd65 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193256516947Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cd0a15c21d0d93a172b93ddeb79ab1b8b4f1575409c79b0c46175dfc4bcd7f4 +size 1817190 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193256893409Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193256893409Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5c753a76e92ea857808332133d14d84ce9159018 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193256893409Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110119727616, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 6, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6237928960, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:56.893020Z", + "world_size": 8 + }, + "allocated_bytes_current": 110119727616, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 6, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6237928960, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:56.564170Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6237928960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 128417005568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128417005568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3208, + "segment_count_current_delta": 0, + "segment_count_peak": 3208, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T193256893409Z.pickle", + "timestamp_utc": "2026-08-26T19:32:56.920846Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193256893409Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193256893409Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0c8cb68c806cc4aee0b551a3df24be90fac7ca00 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193256893409Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71705907766d72f0407ded1a1bd651e9652159842557fd954dc62e3bec56a1cb +size 1143763 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T193257250489Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T193257250489Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e1c413680c146b38bb50a924510b69f6b0fdb579 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T193257250489Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110119727616, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 6, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6237928960, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:57.250095Z", + "world_size": 8 + }, + "allocated_bytes_current": 110119727616, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 6, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6237928960, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:56.922631Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6237928960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 128417005568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128417005568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3208, + "segment_count_current_delta": 0, + "segment_count_peak": 3208, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T193257250489Z.pickle", + "timestamp_utc": "2026-08-26T19:32:57.280701Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T193257250489Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T193257250489Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..20ed877697d4c680ebddac9b64c866e144106fa2 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T193257250489Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:194c0d0663665f09dbe36564fc21085ba3f6fd592d7a7d13d943b0159760d1d8 +size 1143763 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193254800293Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193254800293Z.json new file mode 100644 index 0000000000000000000000000000000000000000..110de96861be1f26209b89cbac960aebba09f282 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193254800293Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110102684160, + "active_bytes_current_delta": 68305225216, + "active_bytes_peak": 110102687232, + "active_bytes_peak_delta": 68267479552, + "after": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 110102687232, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 110102687232, + "device": 7, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6235035136, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113837604864, + "reserved_bytes_peak": 113837604864, + "segment_count_current": 3126, + "segment_count_peak": 3126, + "timestamp_utc": "2026-08-26T19:32:54.799708Z", + "world_size": 8 + }, + "allocated_bytes_current": 110102684160, + "allocated_bytes_current_delta": 68305225216, + "allocated_bytes_peak": 110102687232, + "allocated_bytes_peak_delta": 68267479552, + "before": { + "active_bytes_current": 41797458944, + "active_bytes_peak": 41835207680, + "allocated_bytes_current": 41797458944, + "allocated_bytes_peak": 41835207680, + "device": 7, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 44069552128, + "reserved_bytes_peak": 44069552128, + "segment_count_current": 249, + "segment_count_peak": 249, + "timestamp_utc": "2026-08-26T19:32:32.572650Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_current_delta": 3155229184, + "inactive_split_bytes_peak": 6235035136, + "inactive_split_bytes_peak_delta": 5506445824, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113837604864, + "reserved_bytes_current_delta": 69768052736, + "reserved_bytes_peak": 113837604864, + "reserved_bytes_peak_delta": 69768052736, + "segment_count_current": 3126, + "segment_count_current_delta": 2877, + "segment_count_peak": 3126, + "segment_count_peak_delta": 2877, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T193254800293Z.pickle", + "timestamp_utc": "2026-08-26T19:32:54.853297Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193254800293Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193254800293Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..895d8cb60999bfc4992de1be67b33bd5f467a37b --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193254800293Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6262d801b1bc98f1a5f809a4a84c85b82de69a3f134db419f2ac64304d1fbf97 +size 2260286 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193255191556Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193255191556Z.json new file mode 100644 index 0000000000000000000000000000000000000000..86c3edb9291912595b66cc8b7e84d2f07662ce44 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193255191556Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110102684160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 13368685056, + "after": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 123471372288, + "device": 7, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6235035136, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 127089508352, + "reserved_bytes_peak": 127089508352, + "segment_count_current": 3174, + "segment_count_peak": 3174, + "timestamp_utc": "2026-08-26T19:32:55.191137Z", + "world_size": 8 + }, + "allocated_bytes_current": 110102684160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 13368685056, + "before": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 110102687232, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 110102687232, + "device": 7, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6235035136, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113837604864, + "reserved_bytes_peak": 113837604864, + "segment_count_current": 3126, + "segment_count_peak": 3126, + "timestamp_utc": "2026-08-26T19:32:54.857188Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6235035136, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 127089508352, + "reserved_bytes_current_delta": 13251903488, + "reserved_bytes_peak": 127089508352, + "reserved_bytes_peak_delta": 13251903488, + "segment_count_current": 3174, + "segment_count_current_delta": 48, + "segment_count_peak": 3174, + "segment_count_peak_delta": 48, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T193255191556Z.pickle", + "timestamp_utc": "2026-08-26T19:32:55.227324Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193255191556Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193255191556Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..649d9e0cd2d9e67563a0d4b7873d4c83f1f31f80 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193255191556Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d4fc773e389b9aea647ea7a31a9e52654e0e76d3d32e5095527f42c87e7f587 +size 1776608 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193255560447Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193255560447Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f75f1e3b72125f71bd9af9f5d8ae9bdb30e49a5 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193255560447Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110102684160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 123471372288, + "device": 7, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6235035136, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 127089508352, + "reserved_bytes_peak": 127089508352, + "segment_count_current": 3174, + "segment_count_peak": 3174, + "timestamp_utc": "2026-08-26T19:32:55.560055Z", + "world_size": 8 + }, + "allocated_bytes_current": 110102684160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 123471372288, + "device": 7, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6235035136, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 127089508352, + "reserved_bytes_peak": 127089508352, + "segment_count_current": 3174, + "segment_count_peak": 3174, + "timestamp_utc": "2026-08-26T19:32:55.230292Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6235035136, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 127089508352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127089508352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3174, + "segment_count_current_delta": 0, + "segment_count_peak": 3174, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T193255560447Z.pickle", + "timestamp_utc": "2026-08-26T19:32:55.595753Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193255560447Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193255560447Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f4728987f6f7a6969c90217a264aeffa2be14cd --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193255560447Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ba2bb73463bbd1c3301cb722e2a2c8fc31d7148e2e666623d1551d0866440bf +size 1776965 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193256516946Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193256516946Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0773a1e3340fd561a8fed9b0111966b6ea9bbac0 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193256516946Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110119727616, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 7, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6235035136, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:56.516534Z", + "world_size": 8 + }, + "allocated_bytes_current": 110119727616, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110102684160, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110102684160, + "allocated_bytes_peak": 123471372288, + "device": 7, + "inactive_split_bytes_current": 3663617536, + "inactive_split_bytes_peak": 6235035136, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 127089508352, + "reserved_bytes_peak": 127089508352, + "segment_count_current": 3174, + "segment_count_peak": 3174, + "timestamp_utc": "2026-08-26T19:32:55.598483Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_current_delta": 494661632, + "inactive_split_bytes_peak": 6235035136, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 128417005568, + "reserved_bytes_current_delta": 1327497216, + "reserved_bytes_peak": 128417005568, + "reserved_bytes_peak_delta": 1327497216, + "segment_count_current": 3208, + "segment_count_current_delta": 34, + "segment_count_peak": 3208, + "segment_count_peak_delta": 34, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T193256516946Z.pickle", + "timestamp_utc": "2026-08-26T19:32:56.560374Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193256516946Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193256516946Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..215a0455b07407287d3636af604879c8a3a2ed9a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193256516946Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1f16881204806daaf445ca3128ebf578634ee2c2064b26a1f6897a1246be11a +size 1817190 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193256893423Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193256893423Z.json new file mode 100644 index 0000000000000000000000000000000000000000..08ea9a03eb545184e210be74c1da24458e974089 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193256893423Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110119727616, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 7, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6235035136, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:56.893028Z", + "world_size": 8 + }, + "allocated_bytes_current": 110119727616, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 7, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6235035136, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:56.563712Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6235035136, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 128417005568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128417005568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3208, + "segment_count_current_delta": 0, + "segment_count_peak": 3208, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T193256893423Z.pickle", + "timestamp_utc": "2026-08-26T19:32:56.919947Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193256893423Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193256893423Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..20715ac6ab5a9f2b9c19af3f7dacf65634d49c1d --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193256893423Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15badcfa480f2fc6757748d50d425f9e4ed25735a3297617c4beb8803bc6f1d3 +size 1143763 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T193257251088Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T193257251088Z.json new file mode 100644 index 0000000000000000000000000000000000000000..005d741a29493f7bcc1d8f21ec98f0c55ab2d941 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T193257251088Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110119727616, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123471372288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 7, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6235035136, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:57.250128Z", + "world_size": 8 + }, + "allocated_bytes_current": 110119727616, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123471372288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110119727616, + "active_bytes_peak": 123471372288, + "allocated_bytes_current": 110119727616, + "allocated_bytes_peak": 123471372288, + "device": 7, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_peak": 6235035136, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 128417005568, + "reserved_bytes_peak": 128417005568, + "segment_count_current": 3208, + "segment_count_peak": 3208, + "timestamp_utc": "2026-08-26T19:32:56.921741Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4158279168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6235035136, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 128417005568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 128417005568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3208, + "segment_count_current_delta": 0, + "segment_count_peak": 3208, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T193257251088Z.pickle", + "timestamp_utc": "2026-08-26T19:32:57.277053Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T193257251088Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T193257251088Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14748cc0361a6c647486fe8f506862f60904c860 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T193257251088Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cc6c77f827d039e4c4e3872c59ebf4ad7e57daf2225d2b53d95f3a802906833 +size 1143763 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/train.log b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/train.log index 2d2be22c80874cb3b7a87a0ec6da53b55bd88b10..5d68959d63900039d66ab6fd1aa901cbaa56f650 100644 --- a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/train.log +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/train.log @@ -10,3 +10,1037 @@ W0826 19:32:05.069000 686032 torch/distributed/run.py:766] W0826 19:32:05.069000 686032 torch/distributed/run.py:766] ***************************************** W0826 19:32:05.069000 686032 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. W0826 19:32:05.069000 686032 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 10 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 2560 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 3584 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2560 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_num_sms .............................. None + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 20480 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 28 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 10 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 57 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7[Gloo] Rank +4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[rank5]:[W826 19:32:23.982433890 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:32:23.982433887 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:32:23.982456234 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:32:23.984468120 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:32:23.984487765 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:32:23.984512243 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:32:23.984582916 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:32:24.145220221 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.734 seconds +time to initialize megatron (seconds): 7.983 +[after megatron is initialized] datetime: 2026-08-26 19:32:30 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5060614656 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22 +Params for bucket 1 (180289536 elements, 180289536 padded size): + module.output_layer.weight +Params for bucket 2 (73403904 elements, 73403904 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (146800640 elements, 146800640 padded size): + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight +Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.bias +Params for bucket 5 (146800640 elements, 146800640 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 7 (146800640 elements, 146800640 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight +Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight +Params for bucket 9 (146800640 elements, 146800640 padded size): + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 10 (103001600 elements, 103001600 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.bias + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 11 (146800640 elements, 146800640 padded size): + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (103001600 elements, 103001600 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.bias + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight +Params for bucket 13 (146800640 elements, 146800640 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight +Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_qkv.bias + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 15 (146800640 elements, 146800640 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight +Params for bucket 16 (103001600 elements, 103001600 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.bias + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight +Params for bucket 17 (146800640 elements, 146800640 padded size): + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.bias + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight +Params for bucket 19 (146800640 elements, 146800640 padded size): + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 20 (103001600 elements, 103001600 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.self_attention.linear_qkv.bias + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.pre_mlp_layernorm.weight +Params for bucket 21 (146800640 elements, 146800640 padded size): + module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight +Params for bucket 22 (209890816 elements, 209890816 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 +Params for bucket 1 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 +Params for bucket 2 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 +Params for bucket 3 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 4 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 +Params for bucket 5 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 +Params for bucket 6 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 7 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 +Params for bucket 8 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 9 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 +Params for bucket 10 (45875200 elements, 45875200 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 +Params for bucket 11 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 +Params for bucket 12 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 14 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 16 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 17 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 19 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 +Params for bucket 20 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 21 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 22 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 +Params for bucket 23 (45875200 elements, 45875200 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 +Params for bucket 24 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 25 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 26 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 +Params for bucket 27 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 +Params for bucket 28 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 29 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 +Params for bucket 30 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 31 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 +Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 33 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 34 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 +Params for bucket 36 (45875200 elements, 45875200 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 37 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 39 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 40 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 +Params for bucket 41 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 42 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 43 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +Params for bucket 44 (18350080 elements, 18350080 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:32:32 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:32:32 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (2399.00, 2407.25) + train/valid/test-data-iterators-setup ..........: (61.99, 70.09) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:32:32 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=4096, H=3584, E=64, K=8, NvS=34800, NvS_padded=34816 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 238.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 1.26 MiB + total_size : 241.26 MiB + [2026-08-26 19:32:54] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 22225.3 | throughput per GPU (TFLOP/s/GPU): 5.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097850E+01 | loss scale: 1.0 | grad norm: 29.828 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 20.11 + +Number of parameters in embedding layers in billions: 0.36 +Total number of parameters in billions: 20.47 +Number of parameters in most loaded shard in billions: 20.4724 +Theoretical memory footprints: weight and optimizer=146430.14 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 105000.43212890625 | max allocated: 105000.4326171875 | reserved: 108632.0 | max reserved: 108632.0 + [2026-08-26 19:32:55] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 392.7 | throughput per GPU (TFLOP/s/GPU): 314.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097848E+01 | loss scale: 1.0 | grad norm: 30.563 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:32:55] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 368.9 | throughput per GPU (TFLOP/s/GPU): 334.8 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.533536E+00 | loss scale: 1.0 | grad norm: 32.012 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:32:56] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 956.4 | throughput per GPU (TFLOP/s/GPU): 129.1 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.110648E+01 | loss scale: 1.0 | grad norm: 107.468 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:32:56] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 376.5 | throughput per GPU (TFLOP/s/GPU): 328.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.943263E+00 | loss scale: 1.0 | grad norm: 22.995 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:32:57] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 357.0 | throughput per GPU (TFLOP/s/GPU): 345.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.046553E+01 | loss scale: 1.0 | grad norm: 8.592 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 19:32:57 +[rank0]:[W826 19:32:58.711649869 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:33:03.732000 686032 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:33:03.733000 686032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 686176 closing signal SIGTERM +W0826 19:33:03.734000 686032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 686177 closing signal SIGTERM +W0826 19:33:03.736000 686032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 686178 closing signal SIGTERM +W0826 19:33:03.739000 686032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 686179 closing signal SIGTERM +W0826 19:33:03.742000 686032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 686180 closing signal SIGTERM +W0826 19:33:03.748000 686032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 686181 closing signal SIGTERM +W0826 19:33:03.750000 686032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 686182 closing signal SIGTERM +W0826 19:33:03.751000 686032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 686183 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 686032 got signal: 15 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/mem_samples.csv b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..756a17a0a565d10fe8f1f186dd27c1b6e2458dc2 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/mem_samples.csv @@ -0,0 +1,192 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 725 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 1001 +1, 1031 +2, 1013 +3, 1035 +4, 1023 +5, 1033 +6, 1023 +7, 1033 +0, 1803 +1, 1803 +2, 1803 +3, 1803 +4, 1803 +5, 1803 +6, 1803 +7, 1803 +0, 6833 +1, 6785 +2, 6825 +3, 6837 +4, 6819 +5, 6823 +6, 6827 +7, 6805 +0, 58409 +1, 57847 +2, 57845 +3, 58579 +4, 58575 +5, 58571 +6, 58577 +7, 58573 +0, 59185 +1, 58637 +2, 58657 +3, 59347 +4, 59327 +5, 59337 +6, 59327 +7, 59357 +0, 59719 +1, 59151 +2, 59151 +3, 59851 +4, 59851 +5, 59851 +6, 59851 +7, 59851 +0, 60421 +1, 59849 +2, 59851 +3, 60551 +4, 60551 +5, 60551 +6, 60551 +7, 60551 +0, 89077 +1, 86507 +2, 86613 +3, 84765 +4, 86339 +5, 85833 +6, 84951 +7, 92315 +0, 117907 +1, 117959 +2, 117933 +3, 117969 +4, 117973 +5, 117971 +6, 117975 +7, 117977 +0, 118939 +1, 118985 +2, 118965 +3, 118999 +4, 118999 +5, 118999 +6, 118999 +7, 118999 +0, 118939 +1, 118985 +2, 118967 +3, 119001 +4, 119001 +5, 119001 +6, 119001 +7, 119001 +0, 119599 +1, 119645 +2, 119625 +3, 119659 +4, 119659 +5, 119659 +6, 119659 +7, 119659 +0, 128655 +1, 128353 +2, 128439 +3, 128481 +4, 128435 +5, 128501 +6, 128527 +7, 128447 +0, 141325 +1, 141557 +2, 141529 +3, 141355 +4, 141355 +5, 141355 +6, 141355 +7, 141355 +0, 140439 +1, 140671 +2, 140643 +3, 140469 +4, 140469 +5, 140469 +6, 140469 +7, 140469 +0, 139017 +1, 139249 +2, 139221 +3, 138659 +4, 138659 +5, 139047 +6, 139047 +7, 138659 +0, 137077 +1, 137527 +2, 137331 +3, 13764 +4, 136299 +5, 13508 +6, 137035 +7, 13828 +0, 13441 +1, 13505 +2, 137216 +3, 13761 +4, 13441 +5, 13441 +6, 13444 +7, 13444 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193400773084Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193400773084Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bdc582a6ed88515bf18a24286b9a24e1b6730eb8 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193400773084Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110084827136, + "active_bytes_current_delta": 68288414720, + "active_bytes_peak": 110084830208, + "active_bytes_peak_delta": 68250669056, + "after": { + "active_bytes_current": 110084827136, + "active_bytes_peak": 110084830208, + "allocated_bytes_current": 110084827136, + "allocated_bytes_peak": 110084830208, + "device": 0, + "inactive_split_bytes_current": 3717126144, + "inactive_split_bytes_peak": 5852762624, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113908908032, + "reserved_bytes_peak": 113908908032, + "segment_count_current": 3136, + "segment_count_peak": 3136, + "timestamp_utc": "2026-08-26T19:34:00.772564Z", + "world_size": 8 + }, + "allocated_bytes_current": 110084827136, + "allocated_bytes_current_delta": 68288414720, + "allocated_bytes_peak": 110084830208, + "allocated_bytes_peak_delta": 68250669056, + "before": { + "active_bytes_current": 41796412416, + "active_bytes_peak": 41834161152, + "allocated_bytes_current": 41796412416, + "allocated_bytes_peak": 41834161152, + "device": 0, + "inactive_split_bytes_current": 373120000, + "inactive_split_bytes_peak": 593320960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 43929042944, + "reserved_bytes_peak": 43929042944, + "segment_count_current": 250, + "segment_count_peak": 250, + "timestamp_utc": "2026-08-26T19:33:37.984918Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3717126144, + "inactive_split_bytes_current_delta": 3344006144, + "inactive_split_bytes_peak": 5852762624, + "inactive_split_bytes_peak_delta": 5259441664, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113908908032, + "reserved_bytes_current_delta": 69979865088, + "reserved_bytes_peak": 113908908032, + "reserved_bytes_peak_delta": 69979865088, + "segment_count_current": 3136, + "segment_count_current_delta": 2886, + "segment_count_peak": 3136, + "segment_count_peak_delta": 2886, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T193400773084Z.pickle", + "timestamp_utc": "2026-08-26T19:34:00.827638Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193400773084Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193400773084Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..33949b8088c22e543a341231f40c14811a861b73 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193400773084Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1306b8dca7dd1ab6bc26ff6e468c4e74aacb91e1baf65ecc920f431e23e2ba38 +size 2294903 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193401258657Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193401258657Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5fca2e6d469506e95cf1c544bdb7f1492e565b57 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193401258657Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110084827136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123459971584, + "active_bytes_peak_delta": 13375141376, + "after": { + "active_bytes_current": 110084827136, + "active_bytes_peak": 123459971584, + "allocated_bytes_current": 110084827136, + "allocated_bytes_peak": 123459971584, + "device": 0, + "inactive_split_bytes_current": 3717126144, + "inactive_split_bytes_peak": 5852762624, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 127022399488, + "reserved_bytes_peak": 127022399488, + "segment_count_current": 3194, + "segment_count_peak": 3194, + "timestamp_utc": "2026-08-26T19:34:01.258253Z", + "world_size": 8 + }, + "allocated_bytes_current": 110084827136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123459971584, + "allocated_bytes_peak_delta": 13375141376, + "before": { + "active_bytes_current": 110084827136, + "active_bytes_peak": 110084830208, + "allocated_bytes_current": 110084827136, + "allocated_bytes_peak": 110084830208, + "device": 0, + "inactive_split_bytes_current": 3717126144, + "inactive_split_bytes_peak": 5852762624, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113908908032, + "reserved_bytes_peak": 113908908032, + "segment_count_current": 3136, + "segment_count_peak": 3136, + "timestamp_utc": "2026-08-26T19:34:00.831610Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3717126144, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5852762624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 127022399488, + "reserved_bytes_current_delta": 13113491456, + "reserved_bytes_peak": 127022399488, + "reserved_bytes_peak_delta": 13113491456, + "segment_count_current": 3194, + "segment_count_current_delta": 58, + "segment_count_peak": 3194, + "segment_count_peak_delta": 58, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T193401258657Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.298833Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193401258657Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193401258657Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c96a2a91181024615b50c4db155d5c69dbd35d38 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193401258657Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aea7fac12a4d68ec770a7c08d9b04bae7e23b700984893a478c99e46e6ff70a +size 1851098 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193401655878Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193401655878Z.json new file mode 100644 index 0000000000000000000000000000000000000000..63e907cf323c752588a2ef68a89170ed689ce27f --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193401655878Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110084827136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123459971584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110084827136, + "active_bytes_peak": 123459971584, + "allocated_bytes_current": 110084827136, + "allocated_bytes_peak": 123459971584, + "device": 0, + "inactive_split_bytes_current": 3717126144, + "inactive_split_bytes_peak": 5852762624, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 127022399488, + "reserved_bytes_peak": 127022399488, + "segment_count_current": 3194, + "segment_count_peak": 3194, + "timestamp_utc": "2026-08-26T19:34:01.655498Z", + "world_size": 8 + }, + "allocated_bytes_current": 110084827136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123459971584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110084827136, + "active_bytes_peak": 123459971584, + "allocated_bytes_current": 110084827136, + "allocated_bytes_peak": 123459971584, + "device": 0, + "inactive_split_bytes_current": 3717126144, + "inactive_split_bytes_peak": 5852762624, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 127022399488, + "reserved_bytes_peak": 127022399488, + "segment_count_current": 3194, + "segment_count_peak": 3194, + "timestamp_utc": "2026-08-26T19:34:01.301874Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3717126144, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5852762624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 127022399488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127022399488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3194, + "segment_count_current_delta": 0, + "segment_count_peak": 3194, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T193401655878Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.691278Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193401655878Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193401655878Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..18dab8e90aac6441187208c65e2c44241b1eaa4b --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193401655878Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa5bf148acb02fef3084afa0b9623f84acb89b3e598f6b97a6dd79332f76cc82 +size 1851147 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193402050101Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193402050101Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ff7db049d2f0df6821e3026e1a3984b6593c89c7 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193402050101Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110084827136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123459971584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110084827136, + "active_bytes_peak": 123459971584, + "allocated_bytes_current": 110084827136, + "allocated_bytes_peak": 123459971584, + "device": 0, + "inactive_split_bytes_current": 3717126144, + "inactive_split_bytes_peak": 5852762624, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 127022399488, + "reserved_bytes_peak": 127022399488, + "segment_count_current": 3194, + "segment_count_peak": 3194, + "timestamp_utc": "2026-08-26T19:34:02.049714Z", + "world_size": 8 + }, + "allocated_bytes_current": 110084827136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123459971584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110084827136, + "active_bytes_peak": 123459971584, + "allocated_bytes_current": 110084827136, + "allocated_bytes_peak": 123459971584, + "device": 0, + "inactive_split_bytes_current": 3717126144, + "inactive_split_bytes_peak": 5852762624, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 127022399488, + "reserved_bytes_peak": 127022399488, + "segment_count_current": 3194, + "segment_count_peak": 3194, + "timestamp_utc": "2026-08-26T19:34:01.694078Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3717126144, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5852762624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 127022399488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127022399488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3194, + "segment_count_current_delta": 0, + "segment_count_peak": 3194, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T193402050101Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.084508Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193402050101Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193402050101Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..269f5dbf1bf5e53992e2f9d4e9471cba7e0a9e4f --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193402050101Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa9105444117a969cd2d42a6d5799fa043418b16d31b95462490eb57ae18066e +size 1851052 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193402443262Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193402443262Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a3df49268d83a660b526426aae816fe3b42640b6 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193402443262Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110084827136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123459971584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110084827136, + "active_bytes_peak": 123459971584, + "allocated_bytes_current": 110084827136, + "allocated_bytes_peak": 123459971584, + "device": 0, + "inactive_split_bytes_current": 3719223296, + "inactive_split_bytes_peak": 5852762624, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 127022399488, + "reserved_bytes_peak": 127022399488, + "segment_count_current": 3194, + "segment_count_peak": 3194, + "timestamp_utc": "2026-08-26T19:34:02.442875Z", + "world_size": 8 + }, + "allocated_bytes_current": 110084827136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123459971584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110084827136, + "active_bytes_peak": 123459971584, + "allocated_bytes_current": 110084827136, + "allocated_bytes_peak": 123459971584, + "device": 0, + "inactive_split_bytes_current": 3717126144, + "inactive_split_bytes_peak": 5852762624, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 127022399488, + "reserved_bytes_peak": 127022399488, + "segment_count_current": 3194, + "segment_count_peak": 3194, + "timestamp_utc": "2026-08-26T19:34:02.087375Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3719223296, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 5852762624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 127022399488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127022399488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3194, + "segment_count_current_delta": 0, + "segment_count_peak": 3194, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T193402443262Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.478349Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193402443262Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193402443262Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..499b742d249e19f6c0a4428f4743459a4fd8006e --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193402443262Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:709d63ee0c723caac891efcdc8ac2bfe4209b0cb3244489409542e5420ab6393 +size 1851179 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193400773086Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193400773086Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6472f7d309ce704d03b0d7df09066255a1421d8f --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193400773086Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110085664768, + "active_bytes_current_delta": 68288205824, + "active_bytes_peak": 110085667840, + "active_bytes_peak_delta": 68251508736, + "after": { + "active_bytes_current": 110085664768, + "active_bytes_peak": 110085667840, + "allocated_bytes_current": 110085664768, + "allocated_bytes_peak": 110085667840, + "device": 1, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_peak": 5951842816, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113581752320, + "reserved_bytes_peak": 113581752320, + "segment_count_current": 3149, + "segment_count_peak": 3149, + "timestamp_utc": "2026-08-26T19:34:00.772551Z", + "world_size": 8 + }, + "allocated_bytes_current": 110085664768, + "allocated_bytes_current_delta": 68288205824, + "allocated_bytes_peak": 110085667840, + "allocated_bytes_peak_delta": 68251508736, + "before": { + "active_bytes_current": 41797458944, + "active_bytes_peak": 41834159104, + "allocated_bytes_current": 41797458944, + "allocated_bytes_peak": 41834159104, + "device": 1, + "inactive_split_bytes_current": 302867456, + "inactive_split_bytes_peak": 523068416, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 43335548928, + "reserved_bytes_peak": 43335548928, + "segment_count_current": 239, + "segment_count_peak": 239, + "timestamp_utc": "2026-08-26T19:33:37.984894Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_current_delta": 3140791296, + "inactive_split_bytes_peak": 5951842816, + "inactive_split_bytes_peak_delta": 5428774400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113581752320, + "reserved_bytes_current_delta": 70246203392, + "reserved_bytes_peak": 113581752320, + "reserved_bytes_peak_delta": 70246203392, + "segment_count_current": 3149, + "segment_count_current_delta": 2910, + "segment_count_peak": 3149, + "segment_count_peak_delta": 2910, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T193400773086Z.pickle", + "timestamp_utc": "2026-08-26T19:34:00.829891Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193400773086Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193400773086Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..38d641954c88f6afad681cccca0c5328574f52db --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193400773086Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91336bcbae2c065d3f1a7f538cc472bd73eba7ec2468195222b59f2e516930cc +size 2300146 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193401258662Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193401258662Z.json new file mode 100644 index 0000000000000000000000000000000000000000..65950da84514b9dff613ae4518a43d080835b603 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193401258662Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110085664768, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123475995136, + "active_bytes_peak_delta": 13390327296, + "after": { + "active_bytes_current": 110085664768, + "active_bytes_peak": 123475995136, + "allocated_bytes_current": 110085664768, + "allocated_bytes_peak": 123475995136, + "device": 1, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_peak": 5951842816, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 127263571968, + "reserved_bytes_peak": 127263571968, + "segment_count_current": 3200, + "segment_count_peak": 3200, + "timestamp_utc": "2026-08-26T19:34:01.258254Z", + "world_size": 8 + }, + "allocated_bytes_current": 110085664768, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123475995136, + "allocated_bytes_peak_delta": 13390327296, + "before": { + "active_bytes_current": 110085664768, + "active_bytes_peak": 110085667840, + "allocated_bytes_current": 110085664768, + "allocated_bytes_peak": 110085667840, + "device": 1, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_peak": 5951842816, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113581752320, + "reserved_bytes_peak": 113581752320, + "segment_count_current": 3149, + "segment_count_peak": 3149, + "timestamp_utc": "2026-08-26T19:34:00.833848Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5951842816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 127263571968, + "reserved_bytes_current_delta": 13681819648, + "reserved_bytes_peak": 127263571968, + "reserved_bytes_peak_delta": 13681819648, + "segment_count_current": 3200, + "segment_count_current_delta": 51, + "segment_count_peak": 3200, + "segment_count_peak_delta": 51, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T193401258662Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.298541Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193401258662Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193401258662Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3da9b6a9d19eef5668f24d0597c2615cb4c553e --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193401258662Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8df38d00d088fa1084438c31ac15c97a2c5fc7bf1426cd3ad30da25096d2df7e +size 1856614 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193401655853Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193401655853Z.json new file mode 100644 index 0000000000000000000000000000000000000000..10393bb5a61211feb0f69129949767f255100166 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193401655853Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110085664768, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123475995136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110085664768, + "active_bytes_peak": 123475995136, + "allocated_bytes_current": 110085664768, + "allocated_bytes_peak": 123475995136, + "device": 1, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_peak": 5951842816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 127263571968, + "reserved_bytes_peak": 127263571968, + "segment_count_current": 3200, + "segment_count_peak": 3200, + "timestamp_utc": "2026-08-26T19:34:01.655472Z", + "world_size": 8 + }, + "allocated_bytes_current": 110085664768, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123475995136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110085664768, + "active_bytes_peak": 123475995136, + "allocated_bytes_current": 110085664768, + "allocated_bytes_peak": 123475995136, + "device": 1, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_peak": 5951842816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 127263571968, + "reserved_bytes_peak": 127263571968, + "segment_count_current": 3200, + "segment_count_peak": 3200, + "timestamp_utc": "2026-08-26T19:34:01.301583Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5951842816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 127263571968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127263571968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3200, + "segment_count_current_delta": 0, + "segment_count_peak": 3200, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T193401655853Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.691788Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193401655853Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193401655853Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f6957ab8a4590730960e65658a1d106d8af42521 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193401655853Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc3110138a6e3be6ed633225ac5ef5bd824aa221d369e81e0e628142f553ad46 +size 1856838 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193402050063Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193402050063Z.json new file mode 100644 index 0000000000000000000000000000000000000000..102762a588bfeaabfa0b185fd64afc88c4d8a3b1 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193402050063Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110085664768, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123475995136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110085664768, + "active_bytes_peak": 123475995136, + "allocated_bytes_current": 110085664768, + "allocated_bytes_peak": 123475995136, + "device": 1, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_peak": 5951842816, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 127263571968, + "reserved_bytes_peak": 127263571968, + "segment_count_current": 3200, + "segment_count_peak": 3200, + "timestamp_utc": "2026-08-26T19:34:02.049682Z", + "world_size": 8 + }, + "allocated_bytes_current": 110085664768, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123475995136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110085664768, + "active_bytes_peak": 123475995136, + "allocated_bytes_current": 110085664768, + "allocated_bytes_peak": 123475995136, + "device": 1, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_peak": 5951842816, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 127263571968, + "reserved_bytes_peak": 127263571968, + "segment_count_current": 3200, + "segment_count_peak": 3200, + "timestamp_utc": "2026-08-26T19:34:01.694588Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5951842816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 127263571968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127263571968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3200, + "segment_count_current_delta": 0, + "segment_count_peak": 3200, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T193402050063Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.083953Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193402050063Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193402050063Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ef931e9c83dd3f17cfadb1c3d194cd415454b55 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193402050063Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e74cca85ee19a74ec9fe04dacc787a9fb8d65aa98c5f60c7da89344276a6b71 +size 1856903 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193402443206Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193402443206Z.json new file mode 100644 index 0000000000000000000000000000000000000000..54449e4fe8c95698ebd5b14c4c9aa5dfb419e4a0 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193402443206Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110085664768, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123475995136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110085664768, + "active_bytes_peak": 123475995136, + "allocated_bytes_current": 110085664768, + "allocated_bytes_peak": 123475995136, + "device": 1, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_peak": 5951842816, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 127263571968, + "reserved_bytes_peak": 127263571968, + "segment_count_current": 3200, + "segment_count_peak": 3200, + "timestamp_utc": "2026-08-26T19:34:02.442827Z", + "world_size": 8 + }, + "allocated_bytes_current": 110085664768, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123475995136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110085664768, + "active_bytes_peak": 123475995136, + "allocated_bytes_current": 110085664768, + "allocated_bytes_peak": 123475995136, + "device": 1, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_peak": 5951842816, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 127263571968, + "reserved_bytes_peak": 127263571968, + "segment_count_current": 3200, + "segment_count_peak": 3200, + "timestamp_utc": "2026-08-26T19:34:02.086765Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3443658752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5951842816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 127263571968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127263571968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3200, + "segment_count_current_delta": 0, + "segment_count_peak": 3200, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T193402443206Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.477291Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193402443206Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193402443206Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e9bbffcc83447cc3a15e2168b25ad60dfa8a52d3 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193402443206Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8130a1a747e4196dd24791e478025f2e4865e9547bdbaaac18a96a737aa560e +size 1856895 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193400773075Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193400773075Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8220fc8389050ed72388638614fae34973070560 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193400773075Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110090226688, + "active_bytes_current_delta": 68292767744, + "active_bytes_peak": 110090229760, + "active_bytes_peak_delta": 68256070656, + "after": { + "active_bytes_current": 110090226688, + "active_bytes_peak": 110090229760, + "allocated_bytes_current": 110090226688, + "allocated_bytes_peak": 110090229760, + "device": 2, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_peak": 5932995072, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 113711775744, + "reserved_bytes_peak": 113711775744, + "segment_count_current": 3151, + "segment_count_peak": 3151, + "timestamp_utc": "2026-08-26T19:34:00.772546Z", + "world_size": 8 + }, + "allocated_bytes_current": 110090226688, + "allocated_bytes_current_delta": 68292767744, + "allocated_bytes_peak": 110090229760, + "allocated_bytes_peak_delta": 68256070656, + "before": { + "active_bytes_current": 41797458944, + "active_bytes_peak": 41834159104, + "allocated_bytes_current": 41797458944, + "allocated_bytes_peak": 41834159104, + "device": 2, + "inactive_split_bytes_current": 283993088, + "inactive_split_bytes_peak": 504194048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 43335548928, + "reserved_bytes_peak": 43335548928, + "segment_count_current": 239, + "segment_count_peak": 239, + "timestamp_utc": "2026-08-26T19:33:37.984887Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_current_delta": 3287224320, + "inactive_split_bytes_peak": 5932995072, + "inactive_split_bytes_peak_delta": 5428801024, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 113711775744, + "reserved_bytes_current_delta": 70376226816, + "reserved_bytes_peak": 113711775744, + "reserved_bytes_peak_delta": 70376226816, + "segment_count_current": 3151, + "segment_count_current_delta": 2912, + "segment_count_peak": 3151, + "segment_count_peak_delta": 2912, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T193400773075Z.pickle", + "timestamp_utc": "2026-08-26T19:34:00.828972Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193400773075Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193400773075Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e47952c97778aeae4ed6e50dd3f4a0a004bdea52 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193400773075Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23afe8b2c43388f1d89b3b55f10d249d605859faaff52b229a87694afc96026a +size 2295388 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193401258656Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193401258656Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fde8f6f3c9df7e7a22217337f07d2b27617c926a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193401258656Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110090226688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123479853056, + "active_bytes_peak_delta": 13389623296, + "after": { + "active_bytes_current": 110090226688, + "active_bytes_peak": 123479853056, + "allocated_bytes_current": 110090226688, + "allocated_bytes_peak": 123479853056, + "device": 2, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_peak": 5932995072, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 127234211840, + "reserved_bytes_peak": 127234211840, + "segment_count_current": 3202, + "segment_count_peak": 3202, + "timestamp_utc": "2026-08-26T19:34:01.258248Z", + "world_size": 8 + }, + "allocated_bytes_current": 110090226688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123479853056, + "allocated_bytes_peak_delta": 13389623296, + "before": { + "active_bytes_current": 110090226688, + "active_bytes_peak": 110090229760, + "allocated_bytes_current": 110090226688, + "allocated_bytes_peak": 110090229760, + "device": 2, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_peak": 5932995072, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 113711775744, + "reserved_bytes_peak": 113711775744, + "segment_count_current": 3151, + "segment_count_peak": 3151, + "timestamp_utc": "2026-08-26T19:34:00.833031Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5932995072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 127234211840, + "reserved_bytes_current_delta": 13522436096, + "reserved_bytes_peak": 127234211840, + "reserved_bytes_peak_delta": 13522436096, + "segment_count_current": 3202, + "segment_count_current_delta": 51, + "segment_count_peak": 3202, + "segment_count_peak_delta": 51, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T193401258656Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.298591Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193401258656Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193401258656Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..164f78005dc58fed0bc1e7785db6be4e0da0c3f4 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193401258656Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d92e0d7c8bf625d049dc6573f117b1a8fd024e6d29aebce19a60cfeb1004af92 +size 1817169 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193401655890Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193401655890Z.json new file mode 100644 index 0000000000000000000000000000000000000000..781d9f15c187c2d9e0c3b3c95f060d976eb9213e --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193401655890Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110090226688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123479853056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110090226688, + "active_bytes_peak": 123479853056, + "allocated_bytes_current": 110090226688, + "allocated_bytes_peak": 123479853056, + "device": 2, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_peak": 5932995072, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 127234211840, + "reserved_bytes_peak": 127234211840, + "segment_count_current": 3202, + "segment_count_peak": 3202, + "timestamp_utc": "2026-08-26T19:34:01.655506Z", + "world_size": 8 + }, + "allocated_bytes_current": 110090226688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123479853056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110090226688, + "active_bytes_peak": 123479853056, + "allocated_bytes_current": 110090226688, + "allocated_bytes_peak": 123479853056, + "device": 2, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_peak": 5932995072, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 127234211840, + "reserved_bytes_peak": 127234211840, + "segment_count_current": 3202, + "segment_count_peak": 3202, + "timestamp_utc": "2026-08-26T19:34:01.301708Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5932995072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 127234211840, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127234211840, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3202, + "segment_count_current_delta": 0, + "segment_count_peak": 3202, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T193401655890Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.692281Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193401655890Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193401655890Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e78eaa5d2900c9d22c17938069ca7f81db075b2 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193401655890Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e5e13bbd4dc33026c307dd04d460ce8b6d742eb58e1687db01bdb1f6fc4e4c7 +size 1816310 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193402050085Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193402050085Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dc06c37095629a345efcfb41e95443e5c4549461 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193402050085Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110090226688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123479853056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110090226688, + "active_bytes_peak": 123479853056, + "allocated_bytes_current": 110090226688, + "allocated_bytes_peak": 123479853056, + "device": 2, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_peak": 5932995072, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 127234211840, + "reserved_bytes_peak": 127234211840, + "segment_count_current": 3202, + "segment_count_peak": 3202, + "timestamp_utc": "2026-08-26T19:34:02.049703Z", + "world_size": 8 + }, + "allocated_bytes_current": 110090226688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123479853056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110090226688, + "active_bytes_peak": 123479853056, + "allocated_bytes_current": 110090226688, + "allocated_bytes_peak": 123479853056, + "device": 2, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_peak": 5932995072, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 127234211840, + "reserved_bytes_peak": 127234211840, + "segment_count_current": 3202, + "segment_count_peak": 3202, + "timestamp_utc": "2026-08-26T19:34:01.695121Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5932995072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 127234211840, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127234211840, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3202, + "segment_count_current_delta": 0, + "segment_count_peak": 3202, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T193402050085Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.085471Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193402050085Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193402050085Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ee6815e77ca1477a39b648c35e4e113d3ded8d6c --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193402050085Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dee8712675123cb1523b7f3787afd91a4d9990055bdfc20ed00a579e5bb9d2c1 +size 1816338 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193402443189Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193402443189Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e155f3107a12ee0890ee5d4bff19b62788e4aa14 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193402443189Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110090226688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123479853056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110090226688, + "active_bytes_peak": 123479853056, + "allocated_bytes_current": 110090226688, + "allocated_bytes_peak": 123479853056, + "device": 2, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_peak": 5932995072, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 127234211840, + "reserved_bytes_peak": 127234211840, + "segment_count_current": 3202, + "segment_count_peak": 3202, + "timestamp_utc": "2026-08-26T19:34:02.442824Z", + "world_size": 8 + }, + "allocated_bytes_current": 110090226688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123479853056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110090226688, + "active_bytes_peak": 123479853056, + "allocated_bytes_current": 110090226688, + "allocated_bytes_peak": 123479853056, + "device": 2, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_peak": 5932995072, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 127234211840, + "reserved_bytes_peak": 127234211840, + "segment_count_current": 3202, + "segment_count_peak": 3202, + "timestamp_utc": "2026-08-26T19:34:02.088369Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3571217408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5932995072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 127234211840, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127234211840, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3202, + "segment_count_current_delta": 0, + "segment_count_peak": 3202, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T193402443189Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.478543Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193402443189Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193402443189Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c20d16307f1a63119f3274e26d14e4594d70b34 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193402443189Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dc57ec9b5857f09f4632dfdb13e42b631122fd3bf8b8bd78a9cb734377c7841 +size 1816301 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193400773089Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193400773089Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7516d8f2d33b452e71a63c33c38390b73479860c --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193400773089Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 68286233600, + "active_bytes_peak": 110083695616, + "active_bytes_peak_delta": 68248487936, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 110083695616, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 110083695616, + "device": 3, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 113709678592, + "reserved_bytes_peak": 113709678592, + "segment_count_current": 3123, + "segment_count_peak": 3123, + "timestamp_utc": "2026-08-26T19:34:00.772585Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 68286233600, + "allocated_bytes_peak": 110083695616, + "allocated_bytes_peak_delta": 68248487936, + "before": { + "active_bytes_current": 41797458944, + "active_bytes_peak": 41835207680, + "allocated_bytes_current": 41797458944, + "allocated_bytes_peak": 41835207680, + "device": 3, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 44069552128, + "reserved_bytes_peak": 44069552128, + "segment_count_current": 249, + "segment_count_peak": 249, + "timestamp_utc": "2026-08-26T19:33:37.984940Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 3010642944, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 5191342592, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 113709678592, + "reserved_bytes_current_delta": 69640126464, + "reserved_bytes_peak": 113709678592, + "reserved_bytes_peak_delta": 69640126464, + "segment_count_current": 3123, + "segment_count_current_delta": 2874, + "segment_count_peak": 3123, + "segment_count_peak_delta": 2874, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T193400773089Z.pickle", + "timestamp_utc": "2026-08-26T19:34:00.831020Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193400773089Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193400773089Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a691a8dd692c27324df4363faca17ace83fdebcc --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193400773089Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c25d9d282aabb386b654cc5f94213dbdbbad8b111bb92542fa76f91ae342b45b +size 2287175 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193401258689Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193401258689Z.json new file mode 100644 index 0000000000000000000000000000000000000000..23c9ed2ac2f2197b62692f2b90812cabec6d9f24 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193401258689Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 13383578112, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 3, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.258277Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 13383578112, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 110083695616, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 110083695616, + "device": 3, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 113709678592, + "reserved_bytes_peak": 113709678592, + "segment_count_current": 3123, + "segment_count_peak": 3123, + "timestamp_utc": "2026-08-26T19:34:00.835262Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 13342081024, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 13342081024, + "segment_count_current": 3172, + "segment_count_current_delta": 49, + "segment_count_peak": 3172, + "segment_count_peak_delta": 49, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T193401258689Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.300227Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193401258689Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193401258689Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf3b66a0f68e1b6ef60e46db0cbee1e281cad600 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193401258689Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cde17971c5a7e15d1b1367b2e79259fd4aa9e93eed5df0828c64f5b72cda47d +size 1844821 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193401655936Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193401655936Z.json new file mode 100644 index 0000000000000000000000000000000000000000..13723e08f071656d7265ec6198629328a001a603 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193401655936Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 3, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.655538Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 3, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.303424Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T193401655936Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.693206Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193401655936Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193401655936Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..55793e01f144314f9aba0cc441c45d4ee90c939d --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193401655936Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be99528758c45d52e852595518f85a1174ee8dd1f8b4a121f514a6a0b1cfa2d4 +size 1845079 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193402050131Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193402050131Z.json new file mode 100644 index 0000000000000000000000000000000000000000..daa0e95b42b8e28644f42513bb2b42f26024ff01 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193402050131Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 3, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.049750Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 3, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.696110Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T193402050131Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.086137Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193402050131Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193402050131Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2a4c7b13b856e13d539059607f561a37306a45be --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193402050131Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81c3226e02bc71fa801ed22c34709bc8512969dda03f12d9472a05f018c15bae +size 1845107 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193402443294Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193402443294Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f135f951f40e1dd716a80841cfcffb7eb0e60cbe --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193402443294Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 3, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.442899Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 3, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.089038Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T193402443294Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.479731Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193402443294Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193402443294Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da4ad1c788139eb1bc9f193c9b5e2b1b9c555336 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193402443294Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98ee9d4e77a2d86e9ebeae3e1ab75c25c5fffc077da19aa2301003a327115baa +size 1845070 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193400773510Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193400773510Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d1ccba0755790d4d34f2aee884b2d0d1fe9d566a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193400773510Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 68286233600, + "active_bytes_peak": 110083695616, + "active_bytes_peak_delta": 68248487936, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 110083695616, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 110083695616, + "device": 4, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 113709678592, + "reserved_bytes_peak": 113709678592, + "segment_count_current": 3123, + "segment_count_peak": 3123, + "timestamp_utc": "2026-08-26T19:34:00.772903Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 68286233600, + "allocated_bytes_peak": 110083695616, + "allocated_bytes_peak_delta": 68248487936, + "before": { + "active_bytes_current": 41797458944, + "active_bytes_peak": 41835207680, + "allocated_bytes_current": 41797458944, + "allocated_bytes_peak": 41835207680, + "device": 4, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 44069552128, + "reserved_bytes_peak": 44069552128, + "segment_count_current": 249, + "segment_count_peak": 249, + "timestamp_utc": "2026-08-26T19:33:37.984863Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 3010642944, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 5191342592, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 113709678592, + "reserved_bytes_current_delta": 69640126464, + "reserved_bytes_peak": 113709678592, + "reserved_bytes_peak_delta": 69640126464, + "segment_count_current": 3123, + "segment_count_current_delta": 2874, + "segment_count_peak": 3123, + "segment_count_peak_delta": 2874, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T193400773510Z.pickle", + "timestamp_utc": "2026-08-26T19:34:00.834046Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193400773510Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193400773510Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1a0f3889bf6c6919de5be1a24c6849779702b3a3 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193400773510Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3917862ba6986b4c63c1727365757fbf3991ae6ddc4a78b7ccc3ff2c18ef5c81 +size 2287175 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193401258717Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193401258717Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d774b06a868eedc64039f6587515e5a6e27966c2 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193401258717Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 13383578112, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 4, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.258297Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 13383578112, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 110083695616, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 110083695616, + "device": 4, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 113709678592, + "reserved_bytes_peak": 113709678592, + "segment_count_current": 3123, + "segment_count_peak": 3123, + "timestamp_utc": "2026-08-26T19:34:00.838875Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 13342081024, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 13342081024, + "segment_count_current": 3172, + "segment_count_current_delta": 49, + "segment_count_peak": 3172, + "segment_count_peak_delta": 49, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T193401258717Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.301997Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193401258717Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193401258717Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1259f3bd8294f8fd51f8036cfb7d0347e8c6c396 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193401258717Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52f37906fd1fc7eb4ac78c4de7a56d067cc0174be8b22ccaa471d443e55a2f67 +size 1844821 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193401655940Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193401655940Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d65757ce4afcebfbf83b37b361f12fad42fb65ed --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193401655940Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 4, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.655556Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 4, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.305257Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T193401655940Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.691647Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193401655940Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193401655940Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2239a483d0814d63baee0ff8b750015c69278eab --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193401655940Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e742cdc03af460da75529d71dd0295459dfedb430e77d18578e1e780c40341d9 +size 1845079 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193402050113Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193402050113Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4376c19c6d2c2be7ca3bb981bfbdd4559516c3fd --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193402050113Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 4, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.049728Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 4, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.694487Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T193402050113Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.084958Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193402050113Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193402050113Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..043017f7a400033e85beaa3ae93a14ab0a6a7c63 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193402050113Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95b6ea0931755a423162dcc459b33737c42f88dc79883c688c561d16c76fcb4e +size 1845107 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193402443268Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193402443268Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6327ccc1edce75e8cdca64e76dd8c82dd4ad965a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193402443268Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 4, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.442881Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 4, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.087797Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T193402443268Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.478242Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193402443268Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193402443268Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e529624619680fd708700758fcc21abbea0be36 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193402443268Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:722713024eea49920488f5ebe927ae80e530a43a4ed5df41ea9012aceeb7eef9 +size 1845070 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193400773087Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193400773087Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4430f352b88dfce766fcb8e4d912aeb5157b6048 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193400773087Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 68286233600, + "active_bytes_peak": 110083695616, + "active_bytes_peak_delta": 68248487936, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 110083695616, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 110083695616, + "device": 5, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919449600, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113709678592, + "reserved_bytes_peak": 113709678592, + "segment_count_current": 3123, + "segment_count_peak": 3123, + "timestamp_utc": "2026-08-26T19:34:00.772570Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 68286233600, + "allocated_bytes_peak": 110083695616, + "allocated_bytes_peak_delta": 68248487936, + "before": { + "active_bytes_current": 41797458944, + "active_bytes_peak": 41835207680, + "allocated_bytes_current": 41797458944, + "allocated_bytes_peak": 41835207680, + "device": 5, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 44069552128, + "reserved_bytes_peak": 44069552128, + "segment_count_current": 249, + "segment_count_peak": 249, + "timestamp_utc": "2026-08-26T19:33:37.984936Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 3010642944, + "inactive_split_bytes_peak": 5919449600, + "inactive_split_bytes_peak_delta": 5190860288, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113709678592, + "reserved_bytes_current_delta": 69640126464, + "reserved_bytes_peak": 113709678592, + "reserved_bytes_peak_delta": 69640126464, + "segment_count_current": 3123, + "segment_count_current_delta": 2874, + "segment_count_peak": 3123, + "segment_count_peak_delta": 2874, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T193400773087Z.pickle", + "timestamp_utc": "2026-08-26T19:34:00.827258Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193400773087Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193400773087Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f57ed91ec1da1c5c4ab667219297fcf3b0697690 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193400773087Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2855d27f9c44049b6c004258c5ffd4964190a10932c634b211caa8a5e334dd2b +size 2287175 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193401258673Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193401258673Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a93805eca286f8a9d72327b16f58ce67536e3d1b --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193401258673Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 13383578112, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 5, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919449600, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.258268Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 13383578112, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 110083695616, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 110083695616, + "device": 5, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919449600, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113709678592, + "reserved_bytes_peak": 113709678592, + "segment_count_current": 3123, + "segment_count_peak": 3123, + "timestamp_utc": "2026-08-26T19:34:00.831419Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919449600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 13342081024, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 13342081024, + "segment_count_current": 3172, + "segment_count_current_delta": 49, + "segment_count_peak": 3172, + "segment_count_peak_delta": 49, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T193401258673Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.300814Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193401258673Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193401258673Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b35747cfe1c7b20c558cdee5bbca06bb2810d65a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193401258673Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad925470d1c6f0d923297183b88836ecd6d8ea7b75470325a8e6e11875079718 +size 1844821 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193401655917Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193401655917Z.json new file mode 100644 index 0000000000000000000000000000000000000000..300a98a4a4d19e0b51564b0125237f8521b52f63 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193401655917Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 5, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919449600, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.655527Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 5, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919449600, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.304013Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919449600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T193401655917Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.693441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193401655917Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193401655917Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4bac31194de08657940713cfe1ea12f0d7bb8ced --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193401655917Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80426058aab814f5c5cf194db7f1556d0d724a5e7111cf8ec0c4dcca6024b7d4 +size 1845079 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193402050117Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193402050117Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd76a2c257b95745caf308b518c2dfd96d971b3d --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193402050117Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 5, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919449600, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.049715Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 5, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919449600, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.696403Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919449600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T193402050117Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.086825Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193402050117Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193402050117Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e9064cb32db95d1c969d68587d3aa8b110d7b0a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193402050117Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a574355f378f4b8600b53e2ee0723c4a38d377841fd2437578758c126c606fd +size 1845107 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193402443252Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193402443252Z.json new file mode 100644 index 0000000000000000000000000000000000000000..10984b9325e452cbbf90c6fb55be3c6c8cc3a178 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193402443252Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 5, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919449600, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.442866Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 5, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919449600, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.089876Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919449600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T193402443252Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.480469Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193402443252Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193402443252Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..93e4c782f38fcb72f7daf15858189e52e5c9e4ef --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193402443252Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fae2f206c80c514c225d38c29728332b2563a28c149203dda8d665746d6f711 +size 1845070 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193400773092Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193400773092Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fa56d061b32981f8eb9cefd82abfa005323b6f52 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193400773092Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 68286233600, + "active_bytes_peak": 110083695616, + "active_bytes_peak_delta": 68248487936, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 110083695616, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 110083695616, + "device": 6, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113709678592, + "reserved_bytes_peak": 113709678592, + "segment_count_current": 3123, + "segment_count_peak": 3123, + "timestamp_utc": "2026-08-26T19:34:00.772567Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 68286233600, + "allocated_bytes_peak": 110083695616, + "allocated_bytes_peak_delta": 68248487936, + "before": { + "active_bytes_current": 41797458944, + "active_bytes_peak": 41835207680, + "allocated_bytes_current": 41797458944, + "allocated_bytes_peak": 41835207680, + "device": 6, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 44069552128, + "reserved_bytes_peak": 44069552128, + "segment_count_current": 249, + "segment_count_peak": 249, + "timestamp_utc": "2026-08-26T19:33:37.984942Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 3010642944, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 5191342592, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113709678592, + "reserved_bytes_current_delta": 69640126464, + "reserved_bytes_peak": 113709678592, + "reserved_bytes_peak_delta": 69640126464, + "segment_count_current": 3123, + "segment_count_current_delta": 2874, + "segment_count_peak": 3123, + "segment_count_peak_delta": 2874, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T193400773092Z.pickle", + "timestamp_utc": "2026-08-26T19:34:00.827080Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193400773092Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193400773092Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..be5c1a23e50640e6d896fa49581a69bbe338bbae --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193400773092Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68128447fa4edc3da0896a23fc60773c905d01d4b032bafe85715a7a674ca984 +size 2287175 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193401258649Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193401258649Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1dc4138d1ebc217018c7fa02960a76e6f3b51388 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193401258649Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 13383578112, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 6, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.258234Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 13383578112, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 110083695616, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 110083695616, + "device": 6, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113709678592, + "reserved_bytes_peak": 113709678592, + "segment_count_current": 3123, + "segment_count_peak": 3123, + "timestamp_utc": "2026-08-26T19:34:00.831256Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 13342081024, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 13342081024, + "segment_count_current": 3172, + "segment_count_current_delta": 49, + "segment_count_peak": 3172, + "segment_count_peak_delta": 49, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T193401258649Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.300282Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193401258649Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193401258649Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a3f14e6fc007d68643734ede09818c915d1f67f7 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193401258649Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f32d6b44071d750905335ab489f2eb92ca788c78cedde6e290989bfa0b575b1 +size 1844821 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193401655890Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193401655890Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4da0c0abd35db9019d4ee461813e075410f12a56 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193401655890Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 6, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.655490Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 6, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.303469Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T193401655890Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.693399Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193401655890Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193401655890Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..87f0211b29d564a53ff636c15bfb88d2e32035cc --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193401655890Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:608b0c6450c1ed45932adf91bd3b796468527c7486bd5d9a7f959e49928918f2 +size 1845079 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193402050066Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193402050066Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bd8df2d2e9931127891a28cccff331106f33a1c2 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193402050066Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 6, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.049685Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 6, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.696280Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T193402050066Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.085988Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193402050066Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193402050066Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3b57054f15870a8fc4ee8764a026414eecf814c0 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193402050066Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3803e1067143d4ec1b90fa947f999932ad0ba8ff948ff9ba762f1a26c4e80149 +size 1845107 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193402443254Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193402443254Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9489d8d04372000e35338e694a16df40ff3eb3a4 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193402443254Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 6, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.442861Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 6, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5919931904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.088876Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5919931904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T193402443254Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.479141Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193402443254Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193402443254Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..65d1fb7cfb7e418297821be81f001d8476e08b6e --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193402443254Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0841030671e08543241c29d8cfb0f754ddbd6a75f48c4e54f60834ee99022db +size 1845070 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193400773085Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193400773085Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eea443bfcc8617aa1b2a972481d2b1686cbf9212 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193400773085Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 68286233600, + "active_bytes_peak": 110083695616, + "active_bytes_peak_delta": 68248487936, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 110083695616, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 110083695616, + "device": 7, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5918967296, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113709678592, + "reserved_bytes_peak": 113709678592, + "segment_count_current": 3123, + "segment_count_peak": 3123, + "timestamp_utc": "2026-08-26T19:34:00.772550Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 68286233600, + "allocated_bytes_peak": 110083695616, + "allocated_bytes_peak_delta": 68248487936, + "before": { + "active_bytes_current": 41797458944, + "active_bytes_peak": 41835207680, + "allocated_bytes_current": 41797458944, + "allocated_bytes_peak": 41835207680, + "device": 7, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 44069552128, + "reserved_bytes_peak": 44069552128, + "segment_count_current": 249, + "segment_count_peak": 249, + "timestamp_utc": "2026-08-26T19:33:37.984848Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 3010642944, + "inactive_split_bytes_peak": 5918967296, + "inactive_split_bytes_peak_delta": 5190377984, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113709678592, + "reserved_bytes_current_delta": 69640126464, + "reserved_bytes_peak": 113709678592, + "reserved_bytes_peak_delta": 69640126464, + "segment_count_current": 3123, + "segment_count_current_delta": 2874, + "segment_count_peak": 3123, + "segment_count_peak_delta": 2874, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T193400773085Z.pickle", + "timestamp_utc": "2026-08-26T19:34:00.829254Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193400773085Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193400773085Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1172d3aed7946a63d618b47576a6c54c9781428f --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193400773085Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1014a0994e895952dc8ac109e193617555b2ed295fd7603b0007234c0a9cea8 +size 2287176 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193401258680Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193401258680Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f337ac5ff337b584a4bf17252523b0b3c8753dab --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193401258680Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 13383578112, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 7, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5918967296, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.258274Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 13383578112, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 110083695616, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 110083695616, + "device": 7, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5918967296, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113709678592, + "reserved_bytes_peak": 113709678592, + "segment_count_current": 3123, + "segment_count_peak": 3123, + "timestamp_utc": "2026-08-26T19:34:00.833205Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5918967296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 13342081024, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 13342081024, + "segment_count_current": 3172, + "segment_count_current_delta": 49, + "segment_count_peak": 3172, + "segment_count_peak_delta": 49, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T193401258680Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.298574Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193401258680Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193401258680Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c0582198304ad89550570715f1d361e83b501447 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193401258680Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66758b950e80d3274ce2aeb1d6a9a6e1f030462898bf7da8e1b9dbd7ace241ff +size 1844824 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193401655896Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193401655896Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b58f0fd97086d7c4aee2413d24835d64430aae42 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193401655896Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 7, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5918967296, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.655509Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 7, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5918967296, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.301603Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5918967296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T193401655896Z.pickle", + "timestamp_utc": "2026-08-26T19:34:01.691283Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193401655896Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193401655896Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44a6962cb83ffd70ce90d14da262523391dd7e69 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193401655896Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9d12c01a4a25feb74cb094e5526120f942e40d3691a4d077e312a71c7379a68 +size 1845082 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193402050114Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193402050114Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a15ac38047cd53750a389877b711742ab8fecd28 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193402050114Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 7, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5918967296, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.049724Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 7, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5918967296, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:01.694086Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5918967296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T193402050114Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.084463Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193402050114Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193402050114Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f930f82e5f7de5ebf3e9fb72f2e07264ef3cbd7 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193402050114Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56b16a9b4a14cc90e6b37487352325c01812475081054e2bb74425a4145d8aa9 +size 1845110 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193402443294Z.json b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193402443294Z.json new file mode 100644 index 0000000000000000000000000000000000000000..153005e84168e007970dcdf1b689b153c6e10031 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193402443294Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 110083692544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123467273728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 7, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5918967296, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.442901Z", + "world_size": 8 + }, + "allocated_bytes_current": 110083692544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123467273728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 110083692544, + "active_bytes_peak": 123467273728, + "allocated_bytes_current": 110083692544, + "allocated_bytes_peak": 123467273728, + "device": 7, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_peak": 5918967296, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 127051759616, + "reserved_bytes_peak": 127051759616, + "segment_count_current": 3172, + "segment_count_peak": 3172, + "timestamp_utc": "2026-08-26T19:34:02.087256Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3519031296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5918967296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 127051759616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127051759616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3172, + "segment_count_current_delta": 0, + "segment_count_peak": 3172, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T193402443294Z.pickle", + "timestamp_utc": "2026-08-26T19:34:02.477854Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193402443294Z.pickle b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193402443294Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a2d88908c663f933b9525914098124968adaf31 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193402443294Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70c0cb8b0251af3d3db57feeef432930ba674628a24e5f5d309fb05e310f0029 +size 1845073 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/run.env b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..c418542dba64d894a3ab7fe6f16f33cba00daf9b --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/run.env @@ -0,0 +1,61 @@ +backend=moonep +config=QWEN2_57B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_better_memory_dump_support +commit=453b84c7 +num_layers=10 +hidden=3584 +ffn_hidden=2560 +moe_ffn_hidden=2560 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=27525120 +heads=28 +kv_groups=4 +kv_channels= +qk_layernorm=0 +shared_ffn=20480 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=10 +rotary_base=1000000 +rotary_scaling= +qkv_bias=1 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=5 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/train.log b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..73d8fb14418461132d9e4f7f7fd9f2647b2625df --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/train.log @@ -0,0 +1,1048 @@ +============================================================== + QWEN2_57B / moonep (slots/rank=8 policy=matched) + E=64 K=8 H'=2560 m=8 + L=10 seq=4096 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 453b84c7 +============================================================== +W0826 19:33:11.252000 688523 torch/distributed/run.py:766] +W0826 19:33:11.252000 688523 torch/distributed/run.py:766] ***************************************** +W0826 19:33:11.252000 688523 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:33:11.252000 688523 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 10 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 2560 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 3584 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2560 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_num_sms .............................. None + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 20480 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 28 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 10 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 5 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W826 19:33:29.329484193 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:33:29.329562179 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:33:29.332836847 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:33:29.332963525 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:33:29.332988813 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:33:29.333023585 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:33:29.334279301 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:33:29.489311892 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.588 seconds +time to initialize megatron (seconds): 7.190 +[after megatron is initialized] datetime: 2026-08-26 19:33:35 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5060614656 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22 +Params for bucket 1 (180289536 elements, 180289536 padded size): + module.output_layer.weight +Params for bucket 2 (73403904 elements, 73403904 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (146800640 elements, 146800640 padded size): + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight +Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.bias + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (146800640 elements, 146800640 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 7 (146800640 elements, 146800640 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight +Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.pre_mlp_layernorm.weight +Params for bucket 9 (146800640 elements, 146800640 padded size): + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 10 (103001600 elements, 103001600 padded size): + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.bias + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (146800640 elements, 146800640 padded size): + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (103001600 elements, 103001600 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.bias + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight +Params for bucket 13 (146800640 elements, 146800640 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight +Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.bias + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 15 (146800640 elements, 146800640 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight +Params for bucket 16 (103001600 elements, 103001600 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.bias + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight +Params for bucket 17 (146800640 elements, 146800640 padded size): + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.bias + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight +Params for bucket 19 (146800640 elements, 146800640 padded size): + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 20 (103001600 elements, 103001600 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.bias + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight +Params for bucket 21 (146800640 elements, 146800640 padded size): + module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight +Params for bucket 22 (209890816 elements, 209890816 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 +Params for bucket 1 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 +Params for bucket 2 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 +Params for bucket 3 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 +Params for bucket 5 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 +Params for bucket 6 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 +Params for bucket 7 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 +Params for bucket 8 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 9 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 10 (45875200 elements, 45875200 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 11 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 14 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 +Params for bucket 15 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 16 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 +Params for bucket 17 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 18 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 +Params for bucket 19 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 20 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 21 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 22 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 23 (45875200 elements, 45875200 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 24 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 +Params for bucket 25 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 26 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 +Params for bucket 27 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 28 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 +Params for bucket 30 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 31 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 +Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 33 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 34 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 +Params for bucket 36 (45875200 elements, 45875200 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 +Params for bucket 37 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 39 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 40 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 +Params for bucket 41 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 42 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 +Params for bucket 43 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +Params for bucket 44 (18350080 elements, 18350080 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:33:37 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 40 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(40, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 0152a0378ab8ae1561633a6718ca974c-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 0152a0378ab8ae1561633a6718ca974c-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 0152a0378ab8ae1561633a6718ca974c-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:33:37 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (2613.24, 2622.88) + train/valid/test-data-iterators-setup ..........: (61.61, 68.04) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:33:37 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=4096, H=3584, E=64, K=8, NvS=34800, NvS_padded=34816 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 238.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 1.26 MiB + total_size : 241.26 MiB +WARNING:megatron.core.rerun_state_machine:Result validation enabled + [2026-08-26 19:34:00] iteration 1/ 5 | consumed samples: 8 | elapsed time per iteration (ms): 22783.6 | throughput per GPU (TFLOP/s/GPU): 5.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097850E+01 | loss scale: 1.0 | grad norm: 29.821 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 20.11 +Number of parameters in embedding layers in billions: 0.36 +Total number of parameters in billions: 20.47 +Number of parameters in most loaded shard in billions: 20.4724 +Theoretical memory footprints: weight and optimizer=146430.14 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 104985.072265625 | max allocated: 104985.07275390625 | reserved: 108632.0 | max reserved: 108632.0 + [2026-08-26 19:34:01] iteration 2/ 5 | consumed samples: 16 | elapsed time per iteration (ms): 486.4 | throughput per GPU (TFLOP/s/GPU): 253.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097848E+01 | loss scale: 1.0 | grad norm: 30.562 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:34:01] iteration 3/ 5 | consumed samples: 24 | elapsed time per iteration (ms): 397.2 | throughput per GPU (TFLOP/s/GPU): 310.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.533937E+00 | loss scale: 1.0 | grad norm: 32.043 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:34:02] iteration 4/ 5 | consumed samples: 32 | elapsed time per iteration (ms): 394.1 | throughput per GPU (TFLOP/s/GPU): 313.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.111419E+01 | loss scale: 1.0 | grad norm: 107.604 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:34:02] iteration 5/ 5 | consumed samples: 40 | elapsed time per iteration (ms): 393.1 | throughput per GPU (TFLOP/s/GPU): 314.2 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.943912E+00 | loss scale: 1.0 | grad norm: 22.995 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 19:34:02 +[rank0]:[W826 19:34:03.945868002 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 19:34:04.460866578 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 19:34:04.475289260 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 19:34:04.489262300 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 19:34:04.489354683 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 19:34:04.489432467 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 19:34:04.535460006 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 19:34:04.535469153 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:34:09.927000 688523 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:34:09.928000 688523 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 688614 closing signal SIGTERM +W0826 19:34:09.928000 688523 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 688615 closing signal SIGTERM +W0826 19:34:09.932000 688523 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 688616 closing signal SIGTERM +W0826 19:34:09.934000 688523 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 688618 closing signal SIGTERM +W0826 19:34:09.936000 688523 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 688620 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 688523 got signal: 15 diff --git a/moonep/tierbmem/tierbmem_gbs8/memory_summary.json b/moonep/tierbmem/tierbmem_gbs8/memory_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3b161edfa7be0ae319c3843168a2a8528417b10d --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/memory_summary.json @@ -0,0 +1,128 @@ +{ + "runs": [ + { + "run_dir": "DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1", + "config": "DSV2_LITE_L24", + "backend": "moonep", + "num_layers": "24", + "moe_layers": "23", + "seq_length": "8192", + "global_batch_size": "8", + "cuda_graph": "1", + "capacity_mode": "none", + "capacity": "none", + "memory_dump_mode": "iteration", + "branch": "moonep_backend_better_memory_dump_support", + "commit": "453b84c7", + "status": "OK", + "max_allocated_gib": 122.02059412002563, + "max_reserved_gib": 124.513671875, + "fragmentation_gib": 2.4930777549743652, + "theoretical_weight_optim_gib": 95.94234375, + "peak_sampled_gib": 161.4208984375, + "snapshot_pickle_count": 48, + "snapshot_json_count": 48, + "snapshot_expected_each": 48, + "snapshot_complete": 1, + "effective_peak_gib": 161.4208984375 + }, + { + "run_dir": "DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1", + "config": "DSV2_LITE_L24", + "backend": "moonep", + "num_layers": "24", + "moe_layers": "23", + "seq_length": "8192", + "global_batch_size": "8", + "cuda_graph": "0", + "capacity_mode": "none", + "capacity": "none", + "memory_dump_mode": "iteration", + "branch": "moonep_backend_better_memory_dump_support", + "commit": "453b84c7", + "status": "OK", + "max_allocated_gib": 122.02059412002563, + "max_reserved_gib": 124.513671875, + "fragmentation_gib": 2.4930777549743652, + "theoretical_weight_optim_gib": 95.94234375, + "peak_sampled_gib": 153.4697265625, + "snapshot_pickle_count": 40, + "snapshot_json_count": 40, + "snapshot_expected_each": 40, + "snapshot_complete": 1, + "effective_peak_gib": 153.4697265625 + }, + { + "run_dir": "QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1", + "config": "QWEN2_57B", + "backend": "moonep", + "num_layers": "10", + "moe_layers": "10", + "seq_length": "4096", + "global_batch_size": "8", + "cuda_graph": "1", + "capacity_mode": "none", + "capacity": "none", + "memory_dump_mode": "iteration", + "branch": "moonep_backend_better_memory_dump_support", + "commit": "453b84c7", + "status": "OK", + "max_allocated_gib": 102.53948497772217, + "max_reserved_gib": 106.0859375, + "fragmentation_gib": 3.546452522277832, + "theoretical_weight_optim_gib": 142.99818359375, + "peak_sampled_gib": 139.3935546875, + "snapshot_pickle_count": 48, + "snapshot_json_count": 48, + "snapshot_expected_each": 48, + "snapshot_complete": 1, + "effective_peak_gib": 139.3935546875 + }, + { + "run_dir": "QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1", + "config": "QWEN2_57B", + "backend": "moonep", + "num_layers": "10", + "moe_layers": "10", + "seq_length": "4096", + "global_batch_size": "8", + "cuda_graph": "0", + "capacity_mode": "none", + "capacity": "none", + "memory_dump_mode": "iteration", + "branch": "moonep_backend_better_memory_dump_support", + "commit": "453b84c7", + "status": "OK", + "max_allocated_gib": 102.52448511123657, + "max_reserved_gib": 106.0859375, + "fragmentation_gib": 3.5614523887634277, + "theoretical_weight_optim_gib": 142.99818359375, + "peak_sampled_gib": 138.2392578125, + "snapshot_pickle_count": 40, + "snapshot_json_count": 40, + "snapshot_expected_each": 40, + "snapshot_complete": 1, + "effective_peak_gib": 138.2392578125 + } + ], + "comparisons": [ + { + "config": "DSV2_LITE_L24", + "backend": "moonep", + "moon_graph_on_status": "OK", + "moon_graph_on_peak_gib": 161.4208984375, + "moon_graph_off_status": "OK", + "moon_graph_off_peak_gib": 153.4697265625, + "moon_graph_delta_gib": 7.951171875 + }, + { + "config": "QWEN2_57B", + "backend": "moonep", + "moon_graph_on_status": "OK", + "moon_graph_on_peak_gib": 139.3935546875, + "moon_graph_off_status": "OK", + "moon_graph_off_peak_gib": 138.2392578125, + "moon_graph_delta_gib": 1.154296875 + } + ] +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/memory_summary.tsv b/moonep/tierbmem/tierbmem_gbs8/memory_summary.tsv new file mode 100644 index 0000000000000000000000000000000000000000..e117c378c9adc1933e8745bab9f2d6feb8b435cb --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/memory_summary.tsv @@ -0,0 +1,5 @@ +run_dir config backend cuda_graph capacity_mode capacity status effective_peak_gib max_allocated_gib max_reserved_gib fragmentation_gib peak_sampled_gib snapshot_pickle_count snapshot_json_count snapshot_expected_each snapshot_complete branch commit +DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1 DSV2_LITE_L24 moonep 1 none none OK 161.4208984375 122.02059412002563 124.513671875 2.4930777549743652 161.4208984375 48 48 48 1 moonep_backend_better_memory_dump_support 453b84c7 +DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1 DSV2_LITE_L24 moonep 0 none none OK 153.4697265625 122.02059412002563 124.513671875 2.4930777549743652 153.4697265625 40 40 40 1 moonep_backend_better_memory_dump_support 453b84c7 +QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1 QWEN2_57B moonep 1 none none OK 139.3935546875 102.53948497772217 106.0859375 3.546452522277832 139.3935546875 48 48 48 1 moonep_backend_better_memory_dump_support 453b84c7 +QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1 QWEN2_57B moonep 0 none none OK 138.2392578125 102.52448511123657 106.0859375 3.5614523887634277 138.2392578125 40 40 40 1 moonep_backend_better_memory_dump_support 453b84c7 diff --git a/moonep/tierbmem/tierbmem_gbs8/results.tsv b/moonep/tierbmem/tierbmem_gbs8/results.tsv index 8e2688ff419e499998344d664078abeb0098c76d..666424f8aa022239940f47c44b855571303659bf 100644 --- a/moonep/tierbmem/tierbmem_gbs8/results.tsv +++ b/moonep/tierbmem/tierbmem_gbs8/results.tsv @@ -1,3 +1,5 @@ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 767.90 767.90 165295 /workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1 DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK NA NA 157153 /workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1 +QWEN2_57B moonep matched 8 4096 8 none none 1 OK 357.00 357.00 142739 /workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1 +QWEN2_57B moonep matched 8 4096 8 none none 1 OK NA NA 141557 /workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__nocg__memiteration__it5/rep1