diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190754565447Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190754565447Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..efb503a96231b783b800eb18d21f4c8c4d555940 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190754565447Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d24c9da0aac98760181ec8f6f17923ffdf043960281198b845b188095a05a0a +size 1706466 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.json new file mode 100644 index 0000000000000000000000000000000000000000..445b9a66fb73563584b9754cb3c3334a19d4e2a2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72085230592, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 99464647168, + "active_bytes_peak_delta": 4390016000, + "after": { + "active_bytes_current": 72085230592, + "active_bytes_peak": 99464647168, + "allocated_bytes_current": 72085230592, + "allocated_bytes_peak": 95074631168, + "device": 1, + "inactive_split_bytes_current": 1931652096, + "inactive_split_bytes_peak": 5669243904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108192071680, + "reserved_bytes_peak": 108192071680, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.149539Z", + "world_size": 8 + }, + "allocated_bytes_current": 72085230592, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95074631168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72068187136, + "active_bytes_peak": 95074631168, + "allocated_bytes_current": 72068187136, + "allocated_bytes_peak": 95074631168, + "device": 1, + "inactive_split_bytes_current": 2028387328, + "inactive_split_bytes_peak": 4951808512, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 100717821952, + "reserved_bytes_peak": 100717821952, + "segment_count_current": 1144, + "segment_count_peak": 1144, + "timestamp_utc": "2026-08-26T19:07:54.597941Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1931652096, + "inactive_split_bytes_current_delta": -96735232, + "inactive_split_bytes_peak": 5669243904, + "inactive_split_bytes_peak_delta": 717435392, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108192071680, + "reserved_bytes_current_delta": 7474249728, + "reserved_bytes_peak": 108192071680, + "reserved_bytes_peak_delta": 7474249728, + "segment_count_current": 1166, + "segment_count_current_delta": 22, + "segment_count_peak": 1166, + "segment_count_peak_delta": 22, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.185975Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d5472a9dcd26773de51f1a23c485a6e715699a16 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8aed3b8a6cbd2f1450357cf223e62221bd6bbf5dc29d0ed5316ccec631d2a37 +size 2102583 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f3124bbb968b25c54dce4372ed4466b5f2782551 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72085230592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99464647168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72085230592, + "active_bytes_peak": 99464647168, + "allocated_bytes_current": 72085230592, + "allocated_bytes_peak": 95074631168, + "device": 1, + "inactive_split_bytes_current": 1931652096, + "inactive_split_bytes_peak": 5669243904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108192071680, + "reserved_bytes_peak": 108192071680, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.377676Z", + "world_size": 8 + }, + "allocated_bytes_current": 72085230592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95074631168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72085230592, + "active_bytes_peak": 99464647168, + "allocated_bytes_current": 72085230592, + "allocated_bytes_peak": 95074631168, + "device": 1, + "inactive_split_bytes_current": 1931652096, + "inactive_split_bytes_peak": 5669243904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108192071680, + "reserved_bytes_peak": 108192071680, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.190648Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1931652096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5669243904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108192071680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108192071680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1166, + "segment_count_current_delta": 0, + "segment_count_peak": 1166, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.385355Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6752ab8795ad33ca4b847e449f55edf2c670cb5a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b05a572663b534cc1983bb0494ec9d597f0839b445405b6b9594fe4cdde71f30 +size 331653 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d3a83a930c6274f72dd60a75d37e6b5174be0de6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72085230592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99464647168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72085230592, + "active_bytes_peak": 99464647168, + "allocated_bytes_current": 72085230592, + "allocated_bytes_peak": 95074631168, + "device": 1, + "inactive_split_bytes_current": 1931652096, + "inactive_split_bytes_peak": 5669243904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108192071680, + "reserved_bytes_peak": 108192071680, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.567795Z", + "world_size": 8 + }, + "allocated_bytes_current": 72085230592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95074631168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72085230592, + "active_bytes_peak": 99464647168, + "allocated_bytes_current": 72085230592, + "allocated_bytes_peak": 95074631168, + "device": 1, + "inactive_split_bytes_current": 1931652096, + "inactive_split_bytes_peak": 5669243904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108192071680, + "reserved_bytes_peak": 108192071680, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.386363Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1931652096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5669243904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108192071680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108192071680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1166, + "segment_count_current_delta": 0, + "segment_count_peak": 1166, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.575238Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c335e699b7cb0d80751ed3b1133a42da15f6fb8e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caf72eb6a4500079fb7a27beb13c68170b12803ce2b1ee37ffca8f8dc3d46af3 +size 331653 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a496f70c94e52f27c7e00053f069babebc12166a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72067058688, + "active_bytes_current_delta": 31381090304, + "active_bytes_peak": 74050460672, + "active_bytes_peak_delta": 33326743552, + "after": { + "active_bytes_current": 72067058688, + "active_bytes_peak": 74050460672, + "allocated_bytes_current": 72067058688, + "allocated_bytes_peak": 74050460672, + "device": 2, + "inactive_split_bytes_current": 1679291392, + "inactive_split_bytes_peak": 4932083200, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 84494254080, + "reserved_bytes_peak": 84494254080, + "segment_count_current": 1046, + "segment_count_peak": 1046, + "timestamp_utc": "2026-08-26T19:07:53.656629Z", + "world_size": 8 + }, + "allocated_bytes_current": 72067058688, + "allocated_bytes_current_delta": 31381090304, + "allocated_bytes_peak": 74050460672, + "allocated_bytes_peak_delta": 33326743552, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 2, + "inactive_split_bytes_current": 283993088, + "inactive_split_bytes_peak": 504194048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 43130028032, + "reserved_bytes_peak": 43130028032, + "segment_count_current": 251, + "segment_count_peak": 251, + "timestamp_utc": "2026-08-26T19:07:14.578912Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1679291392, + "inactive_split_bytes_current_delta": 1395298304, + "inactive_split_bytes_peak": 4932083200, + "inactive_split_bytes_peak_delta": 4427889152, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 84494254080, + "reserved_bytes_current_delta": 41364226048, + "reserved_bytes_peak": 84494254080, + "reserved_bytes_peak_delta": 41364226048, + "segment_count_current": 1046, + "segment_count_current_delta": 795, + "segment_count_peak": 1046, + "segment_count_peak_delta": 795, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.pickle", + "timestamp_utc": "2026-08-26T19:07:53.692854Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60d16313894b1b601d465ad1d675ef3ed4046c65 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9974cd72c5633b571d5ce6856e67df6dafcda06ff0059ce8a11a706bc3fa2877 +size 1796624 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ee025a8e11fd961c7bd3745854aa19de1d442d5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72067058688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95073281024, + "active_bytes_peak_delta": 21022820352, + "after": { + "active_bytes_current": 72067058688, + "active_bytes_peak": 95073281024, + "allocated_bytes_current": 72067058688, + "allocated_bytes_peak": 95073281024, + "device": 2, + "inactive_split_bytes_current": 1974989824, + "inactive_split_bytes_peak": 4932083200, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 100428414976, + "reserved_bytes_peak": 100428414976, + "segment_count_current": 1145, + "segment_count_peak": 1145, + "timestamp_utc": "2026-08-26T19:07:54.287965Z", + "world_size": 8 + }, + "allocated_bytes_current": 72067058688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95073281024, + "allocated_bytes_peak_delta": 21022820352, + "before": { + "active_bytes_current": 72067058688, + "active_bytes_peak": 74050460672, + "allocated_bytes_current": 72067058688, + "allocated_bytes_peak": 74050460672, + "device": 2, + "inactive_split_bytes_current": 1679291392, + "inactive_split_bytes_peak": 4932083200, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 84494254080, + "reserved_bytes_peak": 84494254080, + "segment_count_current": 1046, + "segment_count_peak": 1046, + "timestamp_utc": "2026-08-26T19:07:53.697327Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1974989824, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 4932083200, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 100428414976, + "reserved_bytes_current_delta": 15934160896, + "reserved_bytes_peak": 100428414976, + "reserved_bytes_peak_delta": 15934160896, + "segment_count_current": 1145, + "segment_count_current_delta": 99, + "segment_count_peak": 1145, + "segment_count_peak_delta": 99, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.320452Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b565a3a4bc2c8b3b184ff9bb11d6829204334ae1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e1342c84c3066cf8a9e8a290ce04b4218ac538ebcc5633c1372fef50bc9d88f +size 1695924 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4a3d49717bc97281f3f75afa1e41cb85fd5eb7e0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72067058688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95073281024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72067058688, + "active_bytes_peak": 95073281024, + "allocated_bytes_current": 72067058688, + "allocated_bytes_peak": 95073281024, + "device": 2, + "inactive_split_bytes_current": 1974989824, + "inactive_split_bytes_peak": 4932083200, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 101252595712, + "reserved_bytes_peak": 101252595712, + "segment_count_current": 1146, + "segment_count_peak": 1146, + "timestamp_utc": "2026-08-26T19:07:54.565080Z", + "world_size": 8 + }, + "allocated_bytes_current": 72067058688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95073281024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72067058688, + "active_bytes_peak": 95073281024, + "allocated_bytes_current": 72067058688, + "allocated_bytes_peak": 95073281024, + "device": 2, + "inactive_split_bytes_current": 1974989824, + "inactive_split_bytes_peak": 4932083200, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 100428414976, + "reserved_bytes_peak": 100428414976, + "segment_count_current": 1145, + "segment_count_peak": 1145, + "timestamp_utc": "2026-08-26T19:07:54.324314Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1974989824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4932083200, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 101252595712, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 101252595712, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 1146, + "segment_count_current_delta": 1, + "segment_count_peak": 1146, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.593215Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..62e7903560a42f7228389144d812259cf96fd56f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fbb0d9fbd45546f2a956bbe610efb15010ac85ccea61be642a9307391993f15 +size 1709822 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.json new file mode 100644 index 0000000000000000000000000000000000000000..916e2dcd35f2509de781ca994ed83a4db524284b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084102144, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 99463518720, + "active_bytes_peak_delta": 4390237696, + "after": { + "active_bytes_current": 72084102144, + "active_bytes_peak": 99463518720, + "allocated_bytes_current": 72084102144, + "allocated_bytes_peak": 95073281024, + "device": 2, + "inactive_split_bytes_current": 1874060288, + "inactive_split_bytes_peak": 5609554944, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108137545728, + "reserved_bytes_peak": 108137545728, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.149582Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084102144, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95073281024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72067058688, + "active_bytes_peak": 95073281024, + "allocated_bytes_current": 72067058688, + "allocated_bytes_peak": 95073281024, + "device": 2, + "inactive_split_bytes_current": 1974989824, + "inactive_split_bytes_peak": 4932083200, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 101252595712, + "reserved_bytes_peak": 101252595712, + "segment_count_current": 1146, + "segment_count_peak": 1146, + "timestamp_utc": "2026-08-26T19:07:54.596691Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1874060288, + "inactive_split_bytes_current_delta": -100929536, + "inactive_split_bytes_peak": 5609554944, + "inactive_split_bytes_peak_delta": 677471744, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108137545728, + "reserved_bytes_current_delta": 6884950016, + "reserved_bytes_peak": 108137545728, + "reserved_bytes_peak_delta": 6884950016, + "segment_count_current": 1166, + "segment_count_current_delta": 20, + "segment_count_peak": 1166, + "segment_count_peak_delta": 20, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.186358Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b25d81b4c831f8a9e054684b8c5766dc27bb1e34 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f07126cc4e9526702c515dfd2355fbcc4ce529a364dc88dd48932a58c562dcc8 +size 1933754 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6a14d429abdab6dad0332c223e20eea2bc3ce63 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084102144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99463518720, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72084102144, + "active_bytes_peak": 99463518720, + "allocated_bytes_current": 72084102144, + "allocated_bytes_peak": 95073281024, + "device": 2, + "inactive_split_bytes_current": 1874060288, + "inactive_split_bytes_peak": 5609554944, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108137545728, + "reserved_bytes_peak": 108137545728, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.377683Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084102144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95073281024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72084102144, + "active_bytes_peak": 99463518720, + "allocated_bytes_current": 72084102144, + "allocated_bytes_peak": 95073281024, + "device": 2, + "inactive_split_bytes_current": 1874060288, + "inactive_split_bytes_peak": 5609554944, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108137545728, + "reserved_bytes_peak": 108137545728, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.190467Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1874060288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5609554944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108137545728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108137545728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1166, + "segment_count_current_delta": 0, + "segment_count_peak": 1166, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.385340Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e5d6159920c208e37ac5e618795aba642bca5858 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d83f263d9ad0ad4dd8dcf3ad71c2ed2b4e16c45d8d739f8e02fdacd3110084b1 +size 307966 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca864821dafa586c7c081031c47ca6e7041a67e7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72084102144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99463518720, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72084102144, + "active_bytes_peak": 99463518720, + "allocated_bytes_current": 72084102144, + "allocated_bytes_peak": 95073281024, + "device": 2, + "inactive_split_bytes_current": 1874060288, + "inactive_split_bytes_peak": 5609554944, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108137545728, + "reserved_bytes_peak": 108137545728, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.567869Z", + "world_size": 8 + }, + "allocated_bytes_current": 72084102144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95073281024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72084102144, + "active_bytes_peak": 99463518720, + "allocated_bytes_current": 72084102144, + "allocated_bytes_peak": 95073281024, + "device": 2, + "inactive_split_bytes_current": 1874060288, + "inactive_split_bytes_peak": 5609554944, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108137545728, + "reserved_bytes_peak": 108137545728, + "segment_count_current": 1166, + "segment_count_peak": 1166, + "timestamp_utc": "2026-08-26T19:07:56.386351Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1874060288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5609554944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108137545728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108137545728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1166, + "segment_count_current_delta": 0, + "segment_count_peak": 1166, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.575890Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5782229a81d1206e77c2a6c37551adf44c703de7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c9bd3551ce330ca7aa01ba1c3f9ba3a2793be0da75f2393ed0483f90232bcaf +size 307935 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.json new file mode 100644 index 0000000000000000000000000000000000000000..07214e092285d12521587ea3f9b9c61ac788245b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 31390646784, + "active_bytes_peak": 74041179136, + "active_bytes_peak_delta": 33317462016, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 74041179136, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 74041179136, + "device": 3, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_peak": 5056163328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 85135982592, + "reserved_bytes_peak": 85135982592, + "segment_count_current": 1021, + "segment_count_peak": 1021, + "timestamp_utc": "2026-08-26T19:07:53.656598Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 31390646784, + "allocated_bytes_peak": 74041179136, + "allocated_bytes_peak_delta": 33317462016, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 3, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:07:14.578843Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_current_delta": 1327021568, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 4327574016, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 85135982592, + "reserved_bytes_current_delta": 41271951360, + "reserved_bytes_peak": 85135982592, + "reserved_bytes_peak_delta": 41271951360, + "segment_count_current": 1021, + "segment_count_current_delta": 760, + "segment_count_peak": 1021, + "segment_count_peak_delta": 760, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.pickle", + "timestamp_utc": "2026-08-26T19:07:53.693208Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dd8ddddfb54d9fe95ca0b8d3076e221a049a9a08 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3a60d377640686a765e132ac35e04aa64458b689a830bafe90b8657ffdb7cf1 +size 2050482 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c882c4a7f05532bdd096cdbebb141f76a9949f59 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95064244224, + "active_bytes_peak_delta": 21023065088, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 3, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 100415832064, + "reserved_bytes_peak": 100415832064, + "segment_count_current": 1115, + "segment_count_peak": 1115, + "timestamp_utc": "2026-08-26T19:07:54.287978Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95064244224, + "allocated_bytes_peak_delta": 21023065088, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 74041179136, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 74041179136, + "device": 3, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_peak": 5056163328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 85135982592, + "reserved_bytes_peak": 85135982592, + "segment_count_current": 1021, + "segment_count_peak": 1021, + "timestamp_utc": "2026-08-26T19:07:53.697502Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 100415832064, + "reserved_bytes_current_delta": 15279849472, + "reserved_bytes_peak": 100415832064, + "reserved_bytes_peak_delta": 15279849472, + "segment_count_current": 1115, + "segment_count_current_delta": 94, + "segment_count_peak": 1115, + "segment_count_peak_delta": 94, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.320860Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..254d0c468e94c10feec41e8a6875f4ae8f060d6d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a19a0a93d892ee5ad86e5c3730c7d4bde91f9d647376d6d844828a6589a47f3 +size 1669210 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.json new file mode 100644 index 0000000000000000000000000000000000000000..757d6eb43db9fd7f6a6db2b1a07076a2251c5557 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95064244224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 3, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 100415832064, + "reserved_bytes_peak": 100415832064, + "segment_count_current": 1115, + "segment_count_peak": 1115, + "timestamp_utc": "2026-08-26T19:07:54.565074Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95064244224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 3, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 100415832064, + "reserved_bytes_peak": 100415832064, + "segment_count_current": 1115, + "segment_count_peak": 1115, + "timestamp_utc": "2026-08-26T19:07:54.324647Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 100415832064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 100415832064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1115, + "segment_count_current_delta": 0, + "segment_count_peak": 1115, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.593327Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fde30b906747a21745298bb330bbbf292dababf3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe92993356aa1c104dce5ece68955e5b920b360d36efcc461ffb5424a5ef8b9b +size 1659945 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ca1b093c8ab70a9eb40a163c510ca4fe2a59a3a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 4408830976, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 3, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5803422208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.149547Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 4811776, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 3, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 100415832064, + "reserved_bytes_peak": 100415832064, + "segment_count_current": 1115, + "segment_count_peak": 1115, + "timestamp_utc": "2026-08-26T19:07:54.596724Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": -100929536, + "inactive_split_bytes_peak": 5803422208, + "inactive_split_bytes_peak_delta": 747258880, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 7887388672, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 7887388672, + "segment_count_current": 1158, + "segment_count_current_delta": 43, + "segment_count_peak": 1158, + "segment_count_peak_delta": 43, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.184118Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2b034f23296e3e5ef35cbb02a9065c33e742701d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13f830338122f0b64b45e22a5f1604c02d4f492f2b439322fa1273892a31c950 +size 1929358 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9f43e9ef094a876fd9b96d1b793844355ad2af05 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 3, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5803422208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.377701Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 3, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5803422208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.188145Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5803422208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1158, + "segment_count_current_delta": 0, + "segment_count_peak": 1158, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.385243Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cc9a14ff54a413fc9e20a62a0dab0bd0bab94671 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30bb8b8c390915527946035eb877a81d8e81b0ca011a02f951384fd09a6d3623 +size 304763 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e43741d4f29db4d2cf61519ce8c9636da36a588 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 3, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5803422208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.567824Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 3, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5803422208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.386233Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5803422208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1158, + "segment_count_current_delta": 0, + "segment_count_peak": 1158, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.575037Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..91f8694c92c43ac4ce7508599977dab4e5a0b4c2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:442e464633ddb74b52d832884d0d5808bfd5f065f0c596eb24804e6d095d9442 +size 304732 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5b433c8616e96ba9aa70ce65bb50f8128175e2ba --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 31390646784, + "active_bytes_peak": 74041179136, + "active_bytes_peak_delta": 33317462016, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 74041179136, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 74041179136, + "device": 4, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_peak": 5056163328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 84255178752, + "reserved_bytes_peak": 84255178752, + "segment_count_current": 1018, + "segment_count_peak": 1018, + "timestamp_utc": "2026-08-26T19:07:53.656779Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 31390646784, + "allocated_bytes_peak": 74041179136, + "allocated_bytes_peak_delta": 33317462016, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 4, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:07:14.578764Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_current_delta": 1327021568, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 4327574016, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 84255178752, + "reserved_bytes_current_delta": 40391147520, + "reserved_bytes_peak": 84255178752, + "reserved_bytes_peak_delta": 40391147520, + "segment_count_current": 1018, + "segment_count_current_delta": 757, + "segment_count_peak": 1018, + "segment_count_peak_delta": 757, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.pickle", + "timestamp_utc": "2026-08-26T19:07:53.694479Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..715b73ae31479f2dc4e69ad65ef4722d290f249f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40dc7eed517b5ebfbe4f9051bfe06ee8e4753f811601822a4a2226b0a87057d2 +size 2049975 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5e1affd030ff660f17d0fd22fce2e94702ebee3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95064244224, + "active_bytes_peak_delta": 21023065088, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 4, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 100709433344, + "reserved_bytes_peak": 100709433344, + "segment_count_current": 1116, + "segment_count_peak": 1116, + "timestamp_utc": "2026-08-26T19:07:54.287961Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95064244224, + "allocated_bytes_peak_delta": 21023065088, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 74041179136, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 74041179136, + "device": 4, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_peak": 5056163328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 84255178752, + "reserved_bytes_peak": 84255178752, + "segment_count_current": 1018, + "segment_count_peak": 1018, + "timestamp_utc": "2026-08-26T19:07:53.698779Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 100709433344, + "reserved_bytes_current_delta": 16454254592, + "reserved_bytes_peak": 100709433344, + "reserved_bytes_peak_delta": 16454254592, + "segment_count_current": 1116, + "segment_count_current_delta": 98, + "segment_count_peak": 1116, + "segment_count_peak_delta": 98, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.320885Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..27c9ccab6ee7b91b543de7f8ec5db48e508a0ba8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85e3e8ed2bfb9ab468796aae78baa71b03b8237dab69340a405d06da59f3e7ca +size 1673137 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36d6bc0bc13be4b5d52f366f7681d1eef4e10673 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95064244224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 4, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 100709433344, + "reserved_bytes_peak": 100709433344, + "segment_count_current": 1116, + "segment_count_peak": 1116, + "timestamp_utc": "2026-08-26T19:07:54.565074Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95064244224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 4, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 100709433344, + "reserved_bytes_peak": 100709433344, + "segment_count_current": 1116, + "segment_count_peak": 1116, + "timestamp_utc": "2026-08-26T19:07:54.324586Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 100709433344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 100709433344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1116, + "segment_count_current_delta": 0, + "segment_count_peak": 1116, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.594182Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..23e2f354b9bb4fe15b761af8198762e4c29b9b52 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fce0c4117776b38d996deabb0758da5b8b39d2c492b81ae68bc4504910185c4 +size 1669731 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c92a8910aa85ed3bfa90853d0ae7ab759e3f338a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 4408830976, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 4, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5860045312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.149563Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 4811776, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 4, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 100709433344, + "reserved_bytes_peak": 100709433344, + "segment_count_current": 1116, + "segment_count_peak": 1116, + "timestamp_utc": "2026-08-26T19:07:54.597543Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": -100929536, + "inactive_split_bytes_peak": 5860045312, + "inactive_split_bytes_peak_delta": 803881984, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 7593787392, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 7593787392, + "segment_count_current": 1158, + "segment_count_current_delta": 42, + "segment_count_peak": 1158, + "segment_count_peak_delta": 42, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.185386Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fa86bc14671172b9c8a7c441d0b564816ac0cd8d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:036e2442bcd51d9d76f99f3882c13d6cab46d83cc9c568862a3e08e8c9aa24f8 +size 1929444 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ffed3d183ae061fbba36e70eb0596bff479f92a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 4, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5860045312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.377662Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 4, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5860045312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.189138Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5860045312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1158, + "segment_count_current_delta": 0, + "segment_count_peak": 1158, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.384905Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e4e2aaa85e668ed8200dfe6a3559b420bfebc8d9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2992fad4156e1d350878212bb7a63982fbb2b56096c61356234d0445f67d7bad +size 304872 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.json new file mode 100644 index 0000000000000000000000000000000000000000..53e1961adfbccc64b01322cf6a5a2ee0402cae17 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 4, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5860045312, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.567820Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 4, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5860045312, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.385924Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5860045312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1158, + "segment_count_current_delta": 0, + "segment_count_peak": 1158, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.575099Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f968279995edd6fae59429f1fad65a77447fcd4d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39fae1522f7b4a64dd46e1d572997c4344de54a5d9f75590fcb817f089c395ee +size 304841 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.json new file mode 100644 index 0000000000000000000000000000000000000000..26a86ba909287e661432d327bf747236b9aa1b08 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 31390646784, + "active_bytes_peak": 74041179136, + "active_bytes_peak_delta": 33317462016, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 74041179136, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 74041179136, + "device": 5, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_peak": 5056163328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 84255178752, + "reserved_bytes_peak": 84255178752, + "segment_count_current": 1018, + "segment_count_peak": 1018, + "timestamp_utc": "2026-08-26T19:07:53.656602Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 31390646784, + "allocated_bytes_peak": 74041179136, + "allocated_bytes_peak_delta": 33317462016, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 5, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:07:14.578992Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_current_delta": 1327021568, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 4327574016, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 84255178752, + "reserved_bytes_current_delta": 40391147520, + "reserved_bytes_peak": 84255178752, + "reserved_bytes_peak_delta": 40391147520, + "segment_count_current": 1018, + "segment_count_current_delta": 757, + "segment_count_peak": 1018, + "segment_count_peak_delta": 757, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.pickle", + "timestamp_utc": "2026-08-26T19:07:53.693328Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e946cb9fdbd77781722c162e40df149d28e9c30c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c67267c2569542f43a21e5a0d76ce414c2aea750e7ab0709b11e2b441f3193fd +size 2049975 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.json new file mode 100644 index 0000000000000000000000000000000000000000..07fa248584638e839013784284406258c6060c31 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95064244224, + "active_bytes_peak_delta": 21023065088, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 5, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 99828629504, + "reserved_bytes_peak": 99828629504, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:07:54.287945Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95064244224, + "allocated_bytes_peak_delta": 21023065088, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 74041179136, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 74041179136, + "device": 5, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_peak": 5056163328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 84255178752, + "reserved_bytes_peak": 84255178752, + "segment_count_current": 1018, + "segment_count_peak": 1018, + "timestamp_utc": "2026-08-26T19:07:53.697869Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 99828629504, + "reserved_bytes_current_delta": 15573450752, + "reserved_bytes_peak": 99828629504, + "reserved_bytes_peak_delta": 15573450752, + "segment_count_current": 1113, + "segment_count_current_delta": 95, + "segment_count_peak": 1113, + "segment_count_peak_delta": 95, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.320175Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f099d7e82897016196227ce7472c8e010c11d18e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:910bc477e0ff1b4ba1c368ef808734a91bbdb16149c456dc613ad8a0de656026 +size 1669041 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70e6a260d1003477b3cc1b01e772c9adcaa968cc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95064244224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 5, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 99828629504, + "reserved_bytes_peak": 99828629504, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:07:54.565581Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95064244224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 5, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 99828629504, + "reserved_bytes_peak": 99828629504, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:07:54.324156Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 99828629504, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 99828629504, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1113, + "segment_count_current_delta": 0, + "segment_count_peak": 1113, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.600995Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6d2581b922b15ab1d82eed484dd36481af10c08f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3be16039d71d6c2058771a91dcac57af5c4f4c1798269998227160f87b9b348c +size 1659589 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190756150024Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190756150024Z.json new file mode 100644 index 0000000000000000000000000000000000000000..66cf1645808d1759f9007446e1211bc6572cdb15 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190756150024Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 4408830976, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 5, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5803422208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.149558Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 4811776, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 5, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 99828629504, + "reserved_bytes_peak": 99828629504, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:07:54.605114Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": -100929536, + "inactive_split_bytes_peak": 5803422208, + "inactive_split_bytes_peak_delta": 747258880, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 8474591232, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 8474591232, + "segment_count_current": 1158, + "segment_count_current_delta": 45, + "segment_count_peak": 1158, + "segment_count_peak_delta": 45, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T190756150024Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.186479Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190756150024Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190756150024Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8da61ed432deba4ec65dd7d42e35c33e938760db --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190756150024Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e89deef5cc08738f61e3d2b614889789f7638fd8fda9400b70f5f5f3449de54 +size 1932179 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190756378088Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190756378088Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c10625df8c69e0b5f2aaae5f920b9fd9e050b6bd --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190756378088Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 5, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5803422208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.377669Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 5, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5803422208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.191368Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5803422208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1158, + "segment_count_current_delta": 0, + "segment_count_peak": 1158, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T190756378088Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.385367Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190756378088Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190756378088Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9ba0dc66b9287460af170536509a74648b16f043 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190756378088Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53463231dbe8efb57ce0e659d4b9f440d2fecad894a07cff5e288851e4396e36 +size 307745 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190756568253Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190756568253Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc7038fb31d5c22fe6896e1a464f7f1bcf2a1df5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190756568253Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 5, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5803422208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.567833Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 5, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5803422208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.386391Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5803422208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1158, + "segment_count_current_delta": 0, + "segment_count_peak": 1158, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T190756568253Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.575308Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190756568253Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190756568253Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a51326bb631b91483e85d0f375eb22183f6fd623 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190756568253Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:637e0bd058bee4a639f418e4359f5bf8c1b452dc4a27d7c20503d8b6a46eaabe +size 307714 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190753657102Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190753657102Z.json new file mode 100644 index 0000000000000000000000000000000000000000..79d3b3670128c038c9e750cc308f8e9f29b06a21 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190753657102Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 31390646784, + "active_bytes_peak": 74041179136, + "active_bytes_peak_delta": 33317462016, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 74041179136, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 74041179136, + "device": 6, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_peak": 5056163328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 84255178752, + "reserved_bytes_peak": 84255178752, + "segment_count_current": 1018, + "segment_count_peak": 1018, + "timestamp_utc": "2026-08-26T19:07:53.656592Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 31390646784, + "allocated_bytes_peak": 74041179136, + "allocated_bytes_peak_delta": 33317462016, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 6, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:07:14.578909Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_current_delta": 1327021568, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 4327574016, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 84255178752, + "reserved_bytes_current_delta": 40391147520, + "reserved_bytes_peak": 84255178752, + "reserved_bytes_peak_delta": 40391147520, + "segment_count_current": 1018, + "segment_count_current_delta": 757, + "segment_count_peak": 1018, + "segment_count_peak_delta": 757, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T190753657102Z.pickle", + "timestamp_utc": "2026-08-26T19:07:53.692977Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190753657102Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190753657102Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7942b68584c0b4edd256bdbfa02a5a7592f445d0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190753657102Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:529377d4aebc9d1a81de19e87854f232a4a1322a2f75d775cc90a2ad8b07bf3f +size 2049972 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190754288408Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190754288408Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3971ab50890a587dd03d9b085203b909e932a15d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190754288408Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95064244224, + "active_bytes_peak_delta": 21023065088, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 6, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 100415832064, + "reserved_bytes_peak": 100415832064, + "segment_count_current": 1115, + "segment_count_peak": 1115, + "timestamp_utc": "2026-08-26T19:07:54.287976Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95064244224, + "allocated_bytes_peak_delta": 21023065088, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 74041179136, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 74041179136, + "device": 6, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_peak": 5056163328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 84255178752, + "reserved_bytes_peak": 84255178752, + "segment_count_current": 1018, + "segment_count_peak": 1018, + "timestamp_utc": "2026-08-26T19:07:53.697449Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 100415832064, + "reserved_bytes_current_delta": 16160653312, + "reserved_bytes_peak": 100415832064, + "reserved_bytes_peak_delta": 16160653312, + "segment_count_current": 1115, + "segment_count_current_delta": 97, + "segment_count_peak": 1115, + "segment_count_peak_delta": 97, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T190754288408Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.321327Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190754288408Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190754288408Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4813a42ec95da1a6c0f86d3bbc0845a2b1148610 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190754288408Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f80168c5f8c5f1bf1edab1d538818c4ebbb209cd8096d289086a13abdaac8f4d +size 1670992 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190754565449Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190754565449Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ba26d5b2b47334809d9c1422d0406ab749e8f46 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190754565449Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95064244224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 6, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 100415832064, + "reserved_bytes_peak": 100415832064, + "segment_count_current": 1115, + "segment_count_peak": 1115, + "timestamp_utc": "2026-08-26T19:07:54.565052Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95064244224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 6, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 100415832064, + "reserved_bytes_peak": 100415832064, + "segment_count_current": 1115, + "segment_count_peak": 1115, + "timestamp_utc": "2026-08-26T19:07:54.325285Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 100415832064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 100415832064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1115, + "segment_count_current_delta": 0, + "segment_count_peak": 1115, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T190754565449Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.593678Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190754565449Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190754565449Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e271f8f2efce47218689b175a336673838a1619 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190754565449Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55245344d91fd6ddc164e7365305b99390ccc5f508e8fbe642424092cccfadd3 +size 1661037 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190756149989Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190756149989Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c7f0a4b38c059f5fde8371a9a4405bf49565025 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190756149989Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 4408830976, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 6, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5765673472, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.149550Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 4811776, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 6, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 100415832064, + "reserved_bytes_peak": 100415832064, + "segment_count_current": 1115, + "segment_count_peak": 1115, + "timestamp_utc": "2026-08-26T19:07:54.597173Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": -100929536, + "inactive_split_bytes_peak": 5765673472, + "inactive_split_bytes_peak_delta": 709510144, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 7887388672, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 7887388672, + "segment_count_current": 1158, + "segment_count_current_delta": 43, + "segment_count_peak": 1158, + "segment_count_peak_delta": 43, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T190756149989Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.185054Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190756149989Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190756149989Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1aa63d78365c2e4a6a467531413d799173885ff9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190756149989Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6958e30afff51e0a11e26578b240bd7cde0f1e0bf881dba30dca4f06dd53cbe9 +size 1929441 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190756378089Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190756378089Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7629aab43ea34ae896d6ce27574773af95465930 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190756378089Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 6, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5765673472, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.377671Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 6, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5765673472, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.189212Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5765673472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1158, + "segment_count_current_delta": 0, + "segment_count_peak": 1158, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T190756378089Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.385228Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190756378089Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190756378089Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f51a81a6a1d45c6266447be4e4634d79bdd174e6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190756378089Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:168f6e82e7ee44d088cfb874a13a7c1dabfba3aa2fca50f21b9a47e4820d2f07 +size 304753 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190756568158Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190756568158Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6a76bdd81f2305600db3a8243c828a3f21cb637b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190756568158Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 6, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5765673472, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.567780Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 6, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5765673472, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.386262Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5765673472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1158, + "segment_count_current_delta": 0, + "segment_count_peak": 1158, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T190756568158Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.575065Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190756568158Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190756568158Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2a8556eb4e9197831a2d38ebb9370f1c50cfa5f1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190756568158Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:710350fc96a12274d0ccbca5d43aa2988b90b36aeed3a48b45ef9cb381746260 +size 304722 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190753657088Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190753657088Z.json new file mode 100644 index 0000000000000000000000000000000000000000..52c0c4d89f2e0ace977a54f8fba9bbd25897cf63 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190753657088Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 31390646784, + "active_bytes_peak": 74041179136, + "active_bytes_peak_delta": 33317462016, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 74041179136, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 74041179136, + "device": 7, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_peak": 5056163328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 84548780032, + "reserved_bytes_peak": 84548780032, + "segment_count_current": 1019, + "segment_count_peak": 1019, + "timestamp_utc": "2026-08-26T19:07:53.656571Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 31390646784, + "allocated_bytes_peak": 74041179136, + "allocated_bytes_peak_delta": 33317462016, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 7, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:07:14.578945Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_current_delta": 1327021568, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 4327574016, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 84548780032, + "reserved_bytes_current_delta": 40684748800, + "reserved_bytes_peak": 84548780032, + "reserved_bytes_peak_delta": 40684748800, + "segment_count_current": 1019, + "segment_count_current_delta": 758, + "segment_count_peak": 1019, + "segment_count_peak_delta": 758, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T190753657088Z.pickle", + "timestamp_utc": "2026-08-26T19:07:53.692289Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190753657088Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190753657088Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..533a4a4f61198b9931465e343685b1bc11233c84 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190753657088Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ef44e31aec160b8fed28c9677c4e55121d7a54e8043c2da7bb86f307bb5130c +size 2050144 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190754288393Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190754288393Z.json new file mode 100644 index 0000000000000000000000000000000000000000..20688e3355dde4559cccfc15ef2342b4114c2607 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190754288393Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95064244224, + "active_bytes_peak_delta": 21023065088, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 7, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 100709433344, + "reserved_bytes_peak": 100709433344, + "segment_count_current": 1116, + "segment_count_peak": 1116, + "timestamp_utc": "2026-08-26T19:07:54.287958Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95064244224, + "allocated_bytes_peak_delta": 21023065088, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 74041179136, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 74041179136, + "device": 7, + "inactive_split_bytes_current": 1835409920, + "inactive_split_bytes_peak": 5056163328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 84548780032, + "reserved_bytes_peak": 84548780032, + "segment_count_current": 1019, + "segment_count_peak": 1019, + "timestamp_utc": "2026-08-26T19:07:53.696506Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 100709433344, + "reserved_bytes_current_delta": 16160653312, + "reserved_bytes_peak": 100709433344, + "reserved_bytes_peak_delta": 16160653312, + "segment_count_current": 1116, + "segment_count_current_delta": 97, + "segment_count_peak": 1116, + "segment_count_peak_delta": 97, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T190754288393Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.319928Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190754288393Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190754288393Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7b8847034eb0bc06dcc7af3af5b89d18f62a8fdc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190754288393Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37c26c400d73b2fc268f80fe3628c42be8d1d8d0e1efe0eb7ada9c0be239c4a8 +size 1671105 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190754565457Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190754565457Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7ddc6d8a14ac578a6af92e2148c940287128fb1b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190754565457Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72076615168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95064244224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 7, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 100709433344, + "reserved_bytes_peak": 100709433344, + "segment_count_current": 1116, + "segment_count_peak": 1116, + "timestamp_utc": "2026-08-26T19:07:54.565057Z", + "world_size": 8 + }, + "allocated_bytes_current": 72076615168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95064244224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 7, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 100709433344, + "reserved_bytes_peak": 100709433344, + "segment_count_current": 1116, + "segment_count_peak": 1116, + "timestamp_utc": "2026-08-26T19:07:54.323663Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5056163328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 100709433344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 100709433344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1116, + "segment_count_current_delta": 0, + "segment_count_peak": 1116, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T190754565457Z.pickle", + "timestamp_utc": "2026-08-26T19:07:54.592848Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190754565457Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190754565457Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d61b2311f2a86fc189bae3e2fddfc3d82d80e15d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190754565457Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:441e43fa559ef7ef3eb9f81e5f11374dc4746f6593f9d990772cef77a5953777 +size 1661150 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190756149972Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190756149972Z.json new file mode 100644 index 0000000000000000000000000000000000000000..afb42067ae186eb0df80f2fd18ecf4c5a47a5676 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190756149972Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 4408830976, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 7, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5860045312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.149535Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 4811776, + "before": { + "active_bytes_current": 72076615168, + "active_bytes_peak": 95064244224, + "allocated_bytes_current": 72076615168, + "allocated_bytes_peak": 95064244224, + "device": 7, + "inactive_split_bytes_current": 2131108352, + "inactive_split_bytes_peak": 5056163328, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 100709433344, + "reserved_bytes_peak": 100709433344, + "segment_count_current": 1116, + "segment_count_peak": 1116, + "timestamp_utc": "2026-08-26T19:07:54.596194Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": -100929536, + "inactive_split_bytes_peak": 5860045312, + "inactive_split_bytes_peak_delta": 803881984, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 7593787392, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 7593787392, + "segment_count_current": 1158, + "segment_count_current_delta": 42, + "segment_count_peak": 1158, + "segment_count_peak_delta": 42, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T190756149972Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.185069Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190756149972Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190756149972Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c4441b9d9ed2278a4e2c90e159670eda63e42080 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190756149972Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97ba5db87e11ffb0293783dfdf9a140d9501addb9942a80ed5b56643a299b902 +size 1929490 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190756378071Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190756378071Z.json new file mode 100644 index 0000000000000000000000000000000000000000..55fb7760e88a3fe73805e31f26b6b87d5f84294e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190756378071Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 7, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5860045312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.377657Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 7, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5860045312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.189606Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5860045312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1158, + "segment_count_current_delta": 0, + "segment_count_peak": 1158, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T190756378071Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.385100Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190756378071Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190756378071Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c8be379e3ef5a8671d78c0248876eea645c06f33 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190756378071Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0633ff3b6209521ac055cced802d7f05c9a593e42a92881a5e9879add5c8c99 +size 304864 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190756568182Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190756568182Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3bf51032882d04b8535734679f2a352250372d44 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190756568182Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72093658624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 99473075200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 7, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5860045312, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.567796Z", + "world_size": 8 + }, + "allocated_bytes_current": 72093658624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95069056000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72093658624, + "active_bytes_peak": 99473075200, + "allocated_bytes_current": 72093658624, + "allocated_bytes_peak": 95069056000, + "device": 7, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_peak": 5860045312, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108303220736, + "reserved_bytes_peak": 108303220736, + "segment_count_current": 1158, + "segment_count_peak": 1158, + "timestamp_utc": "2026-08-26T19:07:56.386080Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2030178816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5860045312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108303220736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108303220736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1158, + "segment_count_current_delta": 0, + "segment_count_peak": 1158, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T190756568182Z.pickle", + "timestamp_utc": "2026-08-26T19:07:56.574932Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190756568182Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190756568182Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ac3fa19f6fe2b4392bc0d89eff1274cadb48ff1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190756568182Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7216962fb1987f5199ca28b7116b2266db792bbf68f691355652cd7f86bf1e29 +size 304833 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/run.env b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..862903e8a01ec5cf99d9d3501a8b5a80792ee02c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN2_57B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=better-memory-dump-support +commit=b7c52442 +num_layers=10 +hidden=3584 +ffn_hidden=2560 +moe_ffn_hidden=2560 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=27525120 +heads=28 +kv_groups=4 +kv_channels= +qk_layernorm=0 +shared_ffn=20480 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=10 +rotary_base=1000000 +rotary_scaling= +qkv_bias=1 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.125 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f4d6d97093dd825fd482ee20eb0978a80f896541 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/train.log @@ -0,0 +1,1196 @@ +============================================================== + QWEN2_57B / echo (slots/rank=8 policy=matched) + E=64 K=8 H'=2560 m=8 + L=10 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.125 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b7c52442 +============================================================== +W0826 19:06:50.357000 614151 torch/distributed/run.py:766] +W0826 19:06:50.357000 614151 torch/distributed/run.py:766] ***************************************** +W0826 19:06:50.357000 614151 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:06:50.357000 614151 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 10 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 2560 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 3584 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2560 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.125 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 20480 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 28 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 10 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7[Gloo] Rank is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 0 is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : +0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank2]:[W826 19:07:08.483444297 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:07:08.483444723 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:07:08.484464769 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:07:08.492023976 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:07:08.492025922 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:07:08.492155853 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:07:08.492180652 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:07:08.649134389 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.574 seconds +time to initialize megatron (seconds): 6.746 +[after megatron is initialized] datetime: 2026-08-26 19:07:14 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5060614656 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22 +Params for bucket 1 (180289536 elements, 180289536 padded size): + module.output_layer.weight +Params for bucket 2 (73403904 elements, 73403904 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (146800640 elements, 146800640 padded size): + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight +Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.bias + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight +Params for bucket 5 (146800640 elements, 146800640 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 7 (146800640 elements, 146800640 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight +Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_qkv.weight +Params for bucket 9 (146800640 elements, 146800640 padded size): + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 10 (103001600 elements, 103001600 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.bias + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 11 (146800640 elements, 146800640 padded size): + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (103001600 elements, 103001600 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.bias + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_qkv.weight +Params for bucket 13 (146800640 elements, 146800640 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight +Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.bias + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight +Params for bucket 15 (146800640 elements, 146800640 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight +Params for bucket 16 (103001600 elements, 103001600 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.bias +Params for bucket 17 (146800640 elements, 146800640 padded size): + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.self_attention.linear_qkv.bias + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight +Params for bucket 19 (146800640 elements, 146800640 padded size): + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 20 (103001600 elements, 103001600 padded size): + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.bias + module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight +Params for bucket 21 (146800640 elements, 146800640 padded size): + module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight +Params for bucket 22 (209890816 elements, 209890816 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 +Params for bucket 1 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 +Params for bucket 2 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 +Params for bucket 3 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 5 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 +Params for bucket 6 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 +Params for bucket 7 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 8 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 9 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 10 (45875200 elements, 45875200 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 +Params for bucket 11 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 +Params for bucket 13 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 14 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 +Params for bucket 15 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 16 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 17 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 +Params for bucket 19 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 20 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 22 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 23 (45875200 elements, 45875200 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 24 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 25 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 26 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 +Params for bucket 27 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 +Params for bucket 28 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 +Params for bucket 30 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 31 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 +Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 33 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 34 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (45875200 elements, 45875200 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 37 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 +Params for bucket 38 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 39 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 40 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 +Params for bucket 41 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 42 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 +Params for bucket 43 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 44 (18350080 elements, 18350080 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:07:14 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:07:14 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (325.26, 332.88) + train/valid/test-data-iterators-setup ..........: (58.68, 65.27)training ... + +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:07:14 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:07:53] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 39073.8 | throughput per GPU (TFLOP/s/GPU): 3.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097931E+01 | loss scale: 1.0 | grad norm: 29.873 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 20.11 +Number of parameters in embedding layers in billions: 0.36 +Total number of parameters in billions: 20.47 +Number of parameters in most loaded shard in billions: 20.4724 +Theoretical memory footprints: weight and optimizer=146430.14 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 68741.24267578125 | max allocated: 70613.1630859375 | reserved: 80552.0 | max reserved: 80552.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 19:07:54] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 635.2 | throughput per GPU (TFLOP/s/GPU): 194.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097906E+01 | loss scale: 1.0 | grad norm: 30.623 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:07:54] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 277.0 | throughput per GPU (TFLOP/s/GPU): 445.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.538547E+00 | loss scale: 1.0 | grad norm: 32.370 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:07:56] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 1584.3 | throughput per GPU (TFLOP/s/GPU): 77.9 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.109114E+01 | loss scale: 1.0 | grad norm: 107.109 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:07:56] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 228.1 | throughput per GPU (TFLOP/s/GPU): 541.5 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.996848E+00 | loss scale: 1.0 | grad norm: 26.195 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:07:56] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 190.0 | throughput per GPU (TFLOP/s/GPU): 649.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.049657E+01 | loss scale: 1.0 | grad norm: 8.715 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 19:07:56 +[rank0]:[W826 19:07:57.943772707 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:07:59.049000 614151 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:07:59.049000 614151 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 614240 closing signal SIGTERM +W0826 19:07:59.050000 614151 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 614241 closing signal SIGTERM +W0826 19:07:59.050000 614151 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 614242 closing signal SIGTERM +W0826 19:07:59.050000 614151 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 614243 closing signal SIGTERM +W0826 19:07:59.052000 614151 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 614244 closing signal SIGTERM +W0826 19:07:59.053000 614151 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 614246 closing signal SIGTERM +W0826 19:07:59.054000 614151 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 614247 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 614151 got signal: 15 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..1f0dd7916d7e70b14227da9271769e3bde526de8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,224 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1803 +1, 1803 +2, 1803 +3, 1803 +4, 1803 +5, 1803 +6, 1803 +7, 1803 +0, 43745 +1, 43195 +2, 43211 +3, 43903 +4, 43885 +5, 43879 +6, 43881 +7, 43931 +0, 44557 +1, 43989 +2, 43989 +3, 44689 +4, 44689 +5, 44689 +6, 44689 +7, 44689 +0, 47239 +1, 46671 +2, 46671 +3, 47371 +4, 47371 +5, 47371 +6, 47371 +7, 47371 +0, 47519 +1, 46951 +2, 46951 +3, 47651 +4, 47651 +5, 47651 +6, 47651 +7, 47651 +0, 47917 +1, 47349 +2, 47349 +3, 48049 +4, 48049 +5, 48049 +6, 48049 +7, 48049 +0, 48553 +1, 47985 +2, 47985 +3, 48685 +4, 48685 +5, 48685 +6, 48685 +7, 48685 +0, 48833 +1, 48265 +2, 48265 +3, 48965 +4, 48965 +5, 48965 +6, 48965 +7, 48965 +0, 49533 +1, 48963 +2, 48963 +3, 49663 +4, 49663 +5, 49663 +6, 49663 +7, 49663 +0, 50119 +1, 49551 +2, 49551 +3, 50251 +4, 50251 +5, 50251 +6, 50251 +7, 50251 +0, 78569 +1, 78515 +2, 78517 +3, 78553 +4, 78551 +5, 78553 +6, 78553 +7, 78553 +0, 78957 +1, 78903 +2, 78905 +3, 78941 +4, 78941 +5, 78941 +6, 78941 +7, 78941 +0, 79357 +1, 79303 +2, 79305 +3, 79341 +4, 79341 +5, 79341 +6, 79341 +7, 79341 +0, 79357 +1, 79303 +2, 79305 +3, 79341 +4, 79341 +5, 79341 +6, 79341 +7, 79341 +0, 79357 +1, 79303 +2, 79305 +3, 79341 +4, 79341 +5, 79341 +6, 79341 +7, 79341 +0, 80471 +1, 80417 +2, 80699 +3, 80735 +4, 80455 +5, 80455 +6, 80455 +7, 80455 +0, 83359 +1, 83305 +2, 83587 +3, 83623 +4, 83343 +5, 83343 +6, 82783 +7, 82503 +0, 89927 +1, 89765 +2, 90217 +3, 90265 +4, 89887 +5, 89907 +6, 89351 +7, 89097 +0, 102989 +1, 97483 +2, 101509 +3, 102427 +4, 113427 +5, 103281 +6, 102427 +7, 113441 +0, 113061 +1, 112769 +2, 112917 +3, 113205 +4, 113205 +5, 113205 +6, 113205 +7, 113205 +0, 2379 +1, 2382 +2, 2382 +3, 2382 +4, 2638 +5, 112559 +6, 3662 +7, 2382 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190906603276Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190906603276Z.json new file mode 100644 index 0000000000000000000000000000000000000000..785d0f70e789f18c08890000548dd070fa6ef813 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190906603276Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72070574592, + "active_bytes_current_delta": 31385652736, + "active_bytes_peak": 74772542464, + "active_bytes_peak_delta": 34049871872, + "after": { + "active_bytes_current": 72070574592, + "active_bytes_peak": 74772542464, + "allocated_bytes_current": 72070574592, + "allocated_bytes_peak": 74772542464, + "device": 0, + "inactive_split_bytes_current": 2376224256, + "inactive_split_bytes_peak": 5683204608, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 85268103168, + "reserved_bytes_peak": 85268103168, + "segment_count_current": 993, + "segment_count_peak": 993, + "timestamp_utc": "2026-08-26T19:09:06.602592Z", + "world_size": 8 + }, + "allocated_bytes_current": 72070574592, + "allocated_bytes_current_delta": 31385652736, + "allocated_bytes_peak": 74772542464, + "allocated_bytes_peak_delta": 34049871872, + "before": { + "active_bytes_current": 40684921856, + "active_bytes_peak": 40722670592, + "allocated_bytes_current": 40684921856, + "allocated_bytes_peak": 40722670592, + "device": 0, + "inactive_split_bytes_current": 373120000, + "inactive_split_bytes_peak": 593320960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 43723522048, + "reserved_bytes_peak": 43723522048, + "segment_count_current": 262, + "segment_count_peak": 262, + "timestamp_utc": "2026-08-26T19:08:29.045615Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2376224256, + "inactive_split_bytes_current_delta": 2003104256, + "inactive_split_bytes_peak": 5683204608, + "inactive_split_bytes_peak_delta": 5089883648, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 85268103168, + "reserved_bytes_current_delta": 41544581120, + "reserved_bytes_peak": 85268103168, + "reserved_bytes_peak_delta": 41544581120, + "segment_count_current": 993, + "segment_count_current_delta": 731, + "segment_count_peak": 993, + "segment_count_peak_delta": 731, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T190906603276Z.pickle", + "timestamp_utc": "2026-08-26T19:09:06.639235Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190906603276Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190906603276Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8396f36e17fbd12fea468db407a683aff9857377 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T190906603276Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3de1d98ad3c4ea476e5b13737e5938eeef5f8855fe66d9babfbe7c81115037b2 +size 2045191 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190906992113Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190906992113Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b049801433e3a570c0f2ddde214e07bb518a8327 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190906992113Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72070574592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95744868352, + "active_bytes_peak_delta": 20972325888, + "after": { + "active_bytes_current": 72070574592, + "active_bytes_peak": 95744868352, + "allocated_bytes_current": 72070574592, + "allocated_bytes_peak": 95744868352, + "device": 0, + "inactive_split_bytes_current": 2671922688, + "inactive_split_bytes_peak": 5683204608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 100663296000, + "reserved_bytes_peak": 100663296000, + "segment_count_current": 1109, + "segment_count_peak": 1109, + "timestamp_utc": "2026-08-26T19:09:06.991699Z", + "world_size": 8 + }, + "allocated_bytes_current": 72070574592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95744868352, + "allocated_bytes_peak_delta": 20972325888, + "before": { + "active_bytes_current": 72070574592, + "active_bytes_peak": 74772542464, + "allocated_bytes_current": 72070574592, + "allocated_bytes_peak": 74772542464, + "device": 0, + "inactive_split_bytes_current": 2376224256, + "inactive_split_bytes_peak": 5683204608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 85268103168, + "reserved_bytes_peak": 85268103168, + "segment_count_current": 993, + "segment_count_peak": 993, + "timestamp_utc": "2026-08-26T19:09:06.643677Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2671922688, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 5683204608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 100663296000, + "reserved_bytes_current_delta": 15395192832, + "reserved_bytes_peak": 100663296000, + "reserved_bytes_peak_delta": 15395192832, + "segment_count_current": 1109, + "segment_count_current_delta": 116, + "segment_count_peak": 1109, + "segment_count_peak_delta": 116, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T190906992113Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.022075Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190906992113Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190906992113Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7d2c31a3af783538990126b1674b78236c123f6d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T190906992113Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1dc4c8af7a8f2a683290e77db58ce1d9fddba0857045ed12914d166d487c6f3 +size 1684246 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190907270743Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190907270743Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0d1dfda110b0860c243ed23e601fc58eceb5cea4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190907270743Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72070574592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95744868352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72070574592, + "active_bytes_peak": 95744868352, + "allocated_bytes_current": 72070574592, + "allocated_bytes_peak": 95744868352, + "device": 0, + "inactive_split_bytes_current": 2671922688, + "inactive_split_bytes_peak": 5683204608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 101781078016, + "reserved_bytes_peak": 101781078016, + "segment_count_current": 1111, + "segment_count_peak": 1111, + "timestamp_utc": "2026-08-26T19:09:07.270338Z", + "world_size": 8 + }, + "allocated_bytes_current": 72070574592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95744868352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72070574592, + "active_bytes_peak": 95744868352, + "allocated_bytes_current": 72070574592, + "allocated_bytes_peak": 95744868352, + "device": 0, + "inactive_split_bytes_current": 2671922688, + "inactive_split_bytes_peak": 5683204608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 100663296000, + "reserved_bytes_peak": 100663296000, + "segment_count_current": 1109, + "segment_count_peak": 1109, + "timestamp_utc": "2026-08-26T19:09:07.025682Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2671922688, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5683204608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 101781078016, + "reserved_bytes_current_delta": 1117782016, + "reserved_bytes_peak": 101781078016, + "reserved_bytes_peak_delta": 1117782016, + "segment_count_current": 1111, + "segment_count_current_delta": 2, + "segment_count_peak": 1111, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T190907270743Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.298854Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190907270743Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190907270743Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..27676ecbdcc06f014b7fcd22d5affc097a530fd3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T190907270743Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:943ec89b4ee68a8ad8bc9d8818dfc2bc7ea73aacd13946ddd4d7e566de195abb +size 1676628 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190909116369Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190909116369Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4820ed5a1d5effeef417ce576d0c4f3beabb699f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190909116369Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72087618048, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 100121780224, + "active_bytes_peak_delta": 4376911872, + "after": { + "active_bytes_current": 72087618048, + "active_bytes_peak": 100121780224, + "allocated_bytes_current": 72087618048, + "allocated_bytes_peak": 95744868352, + "device": 0, + "inactive_split_bytes_current": 2594061824, + "inactive_split_bytes_peak": 7418544640, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109651689472, + "reserved_bytes_peak": 109651689472, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:09:09.115929Z", + "world_size": 8 + }, + "allocated_bytes_current": 72087618048, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95744868352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72070574592, + "active_bytes_peak": 95744868352, + "allocated_bytes_current": 72070574592, + "allocated_bytes_peak": 95744868352, + "device": 0, + "inactive_split_bytes_current": 2671922688, + "inactive_split_bytes_peak": 5683204608, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 101781078016, + "reserved_bytes_peak": 101781078016, + "segment_count_current": 1111, + "segment_count_peak": 1111, + "timestamp_utc": "2026-08-26T19:09:07.302251Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2594061824, + "inactive_split_bytes_current_delta": -77860864, + "inactive_split_bytes_peak": 7418544640, + "inactive_split_bytes_peak_delta": 1735340032, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109651689472, + "reserved_bytes_current_delta": 7870611456, + "reserved_bytes_peak": 109651689472, + "reserved_bytes_peak_delta": 7870611456, + "segment_count_current": 1130, + "segment_count_current_delta": 19, + "segment_count_peak": 1130, + "segment_count_peak_delta": 19, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T190909116369Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.149990Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190909116369Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190909116369Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ed4220795bc472ace0f9826f3c6132709598a0d8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T190909116369Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74a268638a9073f4a328ea94dbf3bb2c5e3527f39a0bd66da6e4fa8854994dca +size 1933550 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190909338597Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190909338597Z.json new file mode 100644 index 0000000000000000000000000000000000000000..590d8063b1e40072ef147c0fc1242c4c4dd24303 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190909338597Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72087618048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100121780224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72087618048, + "active_bytes_peak": 100121780224, + "allocated_bytes_current": 72087618048, + "allocated_bytes_peak": 95744868352, + "device": 0, + "inactive_split_bytes_current": 2594061824, + "inactive_split_bytes_peak": 7418544640, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109651689472, + "reserved_bytes_peak": 109651689472, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:09:09.338172Z", + "world_size": 8 + }, + "allocated_bytes_current": 72087618048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95744868352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72087618048, + "active_bytes_peak": 100121780224, + "allocated_bytes_current": 72087618048, + "allocated_bytes_peak": 95744868352, + "device": 0, + "inactive_split_bytes_current": 2594061824, + "inactive_split_bytes_peak": 7418544640, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 109651689472, + "reserved_bytes_peak": 109651689472, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:09:09.153904Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2594061824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7418544640, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109651689472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109651689472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1130, + "segment_count_current_delta": 0, + "segment_count_peak": 1130, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T190909338597Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.345441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190909338597Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190909338597Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72629ea8766bfdf918ddcf0da99a274c6739b762 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T190909338597Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d374ecbb60dd6f977d695bdb699a3a1dd0f396e79d30d7ed5e06fc4ac5ccfad +size 306372 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190909528841Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190909528841Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a0d20555b8c409a3c71774f6cede165d6bfd5fcc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190909528841Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72087618048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100121780224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72087618048, + "active_bytes_peak": 100121780224, + "allocated_bytes_current": 72087618048, + "allocated_bytes_peak": 95744868352, + "device": 0, + "inactive_split_bytes_current": 2594061824, + "inactive_split_bytes_peak": 7418544640, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109651689472, + "reserved_bytes_peak": 109651689472, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:09:09.528458Z", + "world_size": 8 + }, + "allocated_bytes_current": 72087618048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95744868352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72087618048, + "active_bytes_peak": 100121780224, + "allocated_bytes_current": 72087618048, + "allocated_bytes_peak": 95744868352, + "device": 0, + "inactive_split_bytes_current": 2594061824, + "inactive_split_bytes_peak": 7418544640, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 109651689472, + "reserved_bytes_peak": 109651689472, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:09:09.346452Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2594061824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7418544640, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109651689472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109651689472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1130, + "segment_count_current_delta": 0, + "segment_count_peak": 1130, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T190909528841Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.535419Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190909528841Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190909528841Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c741a8a7be7a9f594fedc39290903b365901ce7b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T190909528841Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6074fc4237331b052b4fd30ed835124ef5616c20c5f95bf07279e61fa327a9b7 +size 306372 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190906603307Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190906603307Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1cab6e72a38b24b517b386f3930a21cf5b9ed9e9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190906603307Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72054587904, + "active_bytes_current_delta": 31368619520, + "active_bytes_peak": 74778646528, + "active_bytes_peak_delta": 34054929408, + "after": { + "active_bytes_current": 72054587904, + "active_bytes_peak": 74778646528, + "allocated_bytes_current": 72054587904, + "allocated_bytes_peak": 74778646528, + "device": 1, + "inactive_split_bytes_current": 2090221056, + "inactive_split_bytes_peak": 5645156352, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 85100331008, + "reserved_bytes_peak": 85100331008, + "segment_count_current": 993, + "segment_count_peak": 993, + "timestamp_utc": "2026-08-26T19:09:06.602680Z", + "world_size": 8 + }, + "allocated_bytes_current": 72054587904, + "allocated_bytes_current_delta": 31368619520, + "allocated_bytes_peak": 74778646528, + "allocated_bytes_peak_delta": 34054929408, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 1, + "inactive_split_bytes_current": 302867456, + "inactive_split_bytes_peak": 523068416, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 43130028032, + "reserved_bytes_peak": 43130028032, + "segment_count_current": 251, + "segment_count_peak": 251, + "timestamp_utc": "2026-08-26T19:08:29.045773Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2090221056, + "inactive_split_bytes_current_delta": 1787353600, + "inactive_split_bytes_peak": 5645156352, + "inactive_split_bytes_peak_delta": 5122087936, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 85100331008, + "reserved_bytes_current_delta": 41970302976, + "reserved_bytes_peak": 85100331008, + "reserved_bytes_peak_delta": 41970302976, + "segment_count_current": 993, + "segment_count_current_delta": 742, + "segment_count_peak": 993, + "segment_count_peak_delta": 742, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T190906603307Z.pickle", + "timestamp_utc": "2026-08-26T19:09:06.639854Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190906603307Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190906603307Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..743feef26235b4447bcd1e5787e16a84dfc39bd7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T190906603307Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:925b663cb348b0adacaf9a5b48f8160166e5f10128962a9a38b768c73f2aa4c3 +size 1781106 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190906992271Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190906992271Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1832ea27c22232a3cb7b90f260281f631e7b9b3e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190906992271Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72054587904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95741568512, + "active_bytes_peak_delta": 20962921984, + "after": { + "active_bytes_current": 72054587904, + "active_bytes_peak": 95741568512, + "allocated_bytes_current": 72054587904, + "allocated_bytes_peak": 95741568512, + "device": 1, + "inactive_split_bytes_current": 2381725184, + "inactive_split_bytes_peak": 5645156352, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 100782833664, + "reserved_bytes_peak": 100782833664, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:09:06.991831Z", + "world_size": 8 + }, + "allocated_bytes_current": 72054587904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95741568512, + "allocated_bytes_peak_delta": 20962921984, + "before": { + "active_bytes_current": 72054587904, + "active_bytes_peak": 74778646528, + "allocated_bytes_current": 72054587904, + "allocated_bytes_peak": 74778646528, + "device": 1, + "inactive_split_bytes_current": 2090221056, + "inactive_split_bytes_peak": 5645156352, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 85100331008, + "reserved_bytes_peak": 85100331008, + "segment_count_current": 993, + "segment_count_peak": 993, + "timestamp_utc": "2026-08-26T19:09:06.644475Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2381725184, + "inactive_split_bytes_current_delta": 291504128, + "inactive_split_bytes_peak": 5645156352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 100782833664, + "reserved_bytes_current_delta": 15682502656, + "reserved_bytes_peak": 100782833664, + "reserved_bytes_peak_delta": 15682502656, + "segment_count_current": 1121, + "segment_count_current_delta": 128, + "segment_count_peak": 1121, + "segment_count_peak_delta": 128, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T190906992271Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.022969Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190906992271Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190906992271Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cbe7a4946b2d7ba2e2782602aa7b76ef0b8e8eb2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T190906992271Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e2c81a21b59b5d5a3e8bbf1d9b3a63ef1986b68470f0f95648b7366e1ab8629 +size 1679549 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190907270892Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190907270892Z.json new file mode 100644 index 0000000000000000000000000000000000000000..953c6119fb3be400f75f52a410fbacdcdf78290f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190907270892Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72054587904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95741568512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72054587904, + "active_bytes_peak": 95741568512, + "allocated_bytes_current": 72054587904, + "allocated_bytes_peak": 95741568512, + "device": 1, + "inactive_split_bytes_current": 2381725184, + "inactive_split_bytes_peak": 5645156352, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 101607014400, + "reserved_bytes_peak": 101607014400, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:09:07.270458Z", + "world_size": 8 + }, + "allocated_bytes_current": 72054587904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95741568512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72054587904, + "active_bytes_peak": 95741568512, + "allocated_bytes_current": 72054587904, + "allocated_bytes_peak": 95741568512, + "device": 1, + "inactive_split_bytes_current": 2381725184, + "inactive_split_bytes_peak": 5645156352, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 100782833664, + "reserved_bytes_peak": 100782833664, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:09:07.026654Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2381725184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5645156352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 101607014400, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 101607014400, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 1122, + "segment_count_current_delta": 1, + "segment_count_peak": 1122, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T190907270892Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.299608Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190907270892Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190907270892Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b4298b0460f4c2266f846a642c2cef5f5509cf84 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190907270892Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e873e49306464f2c59f7c2c952d84549a1454ca16368c67780f0d94eed9f4956 +size 1679514 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190909116651Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190909116651Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4c5bd62d9eb6d390d7837d4e5b44544b6fb4514d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190909116651Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72071631360, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 100105793536, + "active_bytes_peak_delta": 4364225024, + "after": { + "active_bytes_current": 72071631360, + "active_bytes_peak": 100105793536, + "allocated_bytes_current": 72071631360, + "allocated_bytes_peak": 95741568512, + "device": 1, + "inactive_split_bytes_current": 2305961472, + "inactive_split_bytes_peak": 7277244928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 109345505280, + "reserved_bytes_peak": 109345505280, + "segment_count_current": 1121, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:09:09.116048Z", + "world_size": 8 + }, + "allocated_bytes_current": 72071631360, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95741568512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72054587904, + "active_bytes_peak": 95741568512, + "allocated_bytes_current": 72054587904, + "allocated_bytes_peak": 95741568512, + "device": 1, + "inactive_split_bytes_current": 2381725184, + "inactive_split_bytes_peak": 5645156352, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 101607014400, + "reserved_bytes_peak": 101607014400, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:09:07.303077Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2305961472, + "inactive_split_bytes_current_delta": -75763712, + "inactive_split_bytes_peak": 7277244928, + "inactive_split_bytes_peak_delta": 1632088576, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 109345505280, + "reserved_bytes_current_delta": 7738490880, + "reserved_bytes_peak": 109345505280, + "reserved_bytes_peak_delta": 7738490880, + "segment_count_current": 1121, + "segment_count_current_delta": -1, + "segment_count_peak": 1122, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T190909116651Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.150777Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190909116651Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190909116651Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c739910e6c1ea88799499f81c241ff6bcda606ad --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190909116651Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a1d7a4af47047f9847b8d2797d3106dfafe0c603c468d98920365ca39453805 +size 1956428 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190909338695Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190909338695Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b1484e1aba31b3c4d72ab1d25643a856ab5becae --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190909338695Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72071631360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100105793536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72071631360, + "active_bytes_peak": 100105793536, + "allocated_bytes_current": 72071631360, + "allocated_bytes_peak": 95741568512, + "device": 1, + "inactive_split_bytes_current": 2305961472, + "inactive_split_bytes_peak": 7277244928, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 109345505280, + "reserved_bytes_peak": 109345505280, + "segment_count_current": 1121, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:09:09.338281Z", + "world_size": 8 + }, + "allocated_bytes_current": 72071631360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95741568512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72071631360, + "active_bytes_peak": 100105793536, + "allocated_bytes_current": 72071631360, + "allocated_bytes_peak": 95741568512, + "device": 1, + "inactive_split_bytes_current": 2305961472, + "inactive_split_bytes_peak": 7277244928, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 109345505280, + "reserved_bytes_peak": 109345505280, + "segment_count_current": 1121, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:09:09.154897Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2305961472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7277244928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 109345505280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109345505280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1121, + "segment_count_current_delta": 0, + "segment_count_peak": 1122, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T190909338695Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.345504Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190909338695Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190909338695Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..287fb2ccb48769fbc84dcd4712b2cb57af4f47e3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190909338695Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41bc15b90741ff47f3930064073867e01efb408b339c86e5ba41d207756a6c25 +size 303625 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190909528913Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190909528913Z.json new file mode 100644 index 0000000000000000000000000000000000000000..143038433325e731b20feaeda9ae9a07051ced50 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190909528913Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72071631360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100105793536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72071631360, + "active_bytes_peak": 100105793536, + "allocated_bytes_current": 72071631360, + "allocated_bytes_peak": 95741568512, + "device": 1, + "inactive_split_bytes_current": 2305961472, + "inactive_split_bytes_peak": 7277244928, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 109345505280, + "reserved_bytes_peak": 109345505280, + "segment_count_current": 1121, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:09:09.528531Z", + "world_size": 8 + }, + "allocated_bytes_current": 72071631360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95741568512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72071631360, + "active_bytes_peak": 100105793536, + "allocated_bytes_current": 72071631360, + "allocated_bytes_peak": 95741568512, + "device": 1, + "inactive_split_bytes_current": 2305961472, + "inactive_split_bytes_peak": 7277244928, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 109345505280, + "reserved_bytes_peak": 109345505280, + "segment_count_current": 1121, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:09:09.346512Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2305961472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7277244928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 109345505280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109345505280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1121, + "segment_count_current_delta": 0, + "segment_count_peak": 1122, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T190909528913Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.535506Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190909528913Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190909528913Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b9040cc5b64c702052719c59422db366e1e6aa02 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190909528913Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf08df9530e71c84a73ea812f24ceb93d73470d77a45fe4758515134095dcfea +size 303625 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190906603268Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190906603268Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0267a281e39528bb5cf47e5d351b60fbf094b385 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190906603268Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72055592448, + "active_bytes_current_delta": 31369624064, + "active_bytes_peak": 74782781952, + "active_bytes_peak_delta": 34059064832, + "after": { + "active_bytes_current": 72055592448, + "active_bytes_peak": 74782781952, + "allocated_bytes_current": 72055592448, + "allocated_bytes_peak": 74782781952, + "device": 2, + "inactive_split_bytes_current": 2240211456, + "inactive_split_bytes_peak": 5611729408, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 85618327552, + "reserved_bytes_peak": 85618327552, + "segment_count_current": 1001, + "segment_count_peak": 1001, + "timestamp_utc": "2026-08-26T19:09:06.602565Z", + "world_size": 8 + }, + "allocated_bytes_current": 72055592448, + "allocated_bytes_current_delta": 31369624064, + "allocated_bytes_peak": 74782781952, + "allocated_bytes_peak_delta": 34059064832, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 2, + "inactive_split_bytes_current": 283993088, + "inactive_split_bytes_peak": 504194048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 43130028032, + "reserved_bytes_peak": 43130028032, + "segment_count_current": 251, + "segment_count_peak": 251, + "timestamp_utc": "2026-08-26T19:08:29.045581Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2240211456, + "inactive_split_bytes_current_delta": 1956218368, + "inactive_split_bytes_peak": 5611729408, + "inactive_split_bytes_peak_delta": 5107535360, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 85618327552, + "reserved_bytes_current_delta": 42488299520, + "reserved_bytes_peak": 85618327552, + "reserved_bytes_peak_delta": 42488299520, + "segment_count_current": 1001, + "segment_count_current_delta": 750, + "segment_count_peak": 1001, + "segment_count_peak_delta": 750, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T190906603268Z.pickle", + "timestamp_utc": "2026-08-26T19:09:06.638185Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190906603268Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190906603268Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..82121441e74fd8ebea0a55ad1ab8fd4261ccab4d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190906603268Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07042bf1acde4745cd3cd96747102d6ffde197f63ce45f34b45dacb1b025e8dc +size 1779871 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190906992166Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190906992166Z.json new file mode 100644 index 0000000000000000000000000000000000000000..32c935e75e7fa3cf4d053c6be7f063015ce98a0a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190906992166Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72055592448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95742273536, + "active_bytes_peak_delta": 20959491584, + "after": { + "active_bytes_current": 72055592448, + "active_bytes_peak": 95742273536, + "allocated_bytes_current": 72055592448, + "allocated_bytes_peak": 95742273536, + "device": 2, + "inactive_split_bytes_current": 2535909888, + "inactive_split_bytes_peak": 5611729408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 101093212160, + "reserved_bytes_peak": 101093212160, + "segment_count_current": 1118, + "segment_count_peak": 1118, + "timestamp_utc": "2026-08-26T19:09:06.991754Z", + "world_size": 8 + }, + "allocated_bytes_current": 72055592448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95742273536, + "allocated_bytes_peak_delta": 20959491584, + "before": { + "active_bytes_current": 72055592448, + "active_bytes_peak": 74782781952, + "allocated_bytes_current": 72055592448, + "allocated_bytes_peak": 74782781952, + "device": 2, + "inactive_split_bytes_current": 2240211456, + "inactive_split_bytes_peak": 5611729408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 85618327552, + "reserved_bytes_peak": 85618327552, + "segment_count_current": 1001, + "segment_count_peak": 1001, + "timestamp_utc": "2026-08-26T19:09:06.642655Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2535909888, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 5611729408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 101093212160, + "reserved_bytes_current_delta": 15474884608, + "reserved_bytes_peak": 101093212160, + "reserved_bytes_peak_delta": 15474884608, + "segment_count_current": 1118, + "segment_count_current_delta": 117, + "segment_count_peak": 1118, + "segment_count_peak_delta": 117, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T190906992166Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.021811Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190906992166Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190906992166Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ad20d7595e956add29180caaae4830f828f14846 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190906992166Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ec2ce882881c13420bc8208e07f04eb2668b2c75b6f60204fabb83cd2b4f4aa +size 1676330 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190907270754Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190907270754Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e531f59ee2919a00a4cb313580430d19683a3ae3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190907270754Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72055592448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95742273536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72055592448, + "active_bytes_peak": 95742273536, + "allocated_bytes_current": 72055592448, + "allocated_bytes_peak": 95742273536, + "device": 2, + "inactive_split_bytes_current": 2535909888, + "inactive_split_bytes_peak": 5611729408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 101917392896, + "reserved_bytes_peak": 101917392896, + "segment_count_current": 1119, + "segment_count_peak": 1119, + "timestamp_utc": "2026-08-26T19:09:07.270354Z", + "world_size": 8 + }, + "allocated_bytes_current": 72055592448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95742273536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72055592448, + "active_bytes_peak": 95742273536, + "allocated_bytes_current": 72055592448, + "allocated_bytes_peak": 95742273536, + "device": 2, + "inactive_split_bytes_current": 2535909888, + "inactive_split_bytes_peak": 5611729408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 101093212160, + "reserved_bytes_peak": 101093212160, + "segment_count_current": 1118, + "segment_count_peak": 1118, + "timestamp_utc": "2026-08-26T19:09:07.025287Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2535909888, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5611729408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 101917392896, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 101917392896, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 1119, + "segment_count_current_delta": 1, + "segment_count_peak": 1119, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T190907270754Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.298229Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190907270754Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190907270754Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6be1b9cd447dc4d1a26cf8eb44fbd8019aff01e2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190907270754Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68542c201a5d994291c725fea6a2b89fa9c8e4c229ff017012256815b5feda9c +size 1673026 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190909116410Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190909116410Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cb612b5fd074d83c224fbc1f64aa8083a04148f1 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190909116410Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072635904, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 100106798080, + "active_bytes_peak_delta": 4364524544, + "after": { + "active_bytes_current": 72072635904, + "active_bytes_peak": 100106798080, + "allocated_bytes_current": 72072635904, + "allocated_bytes_peak": 95742273536, + "device": 2, + "inactive_split_bytes_current": 2458049024, + "inactive_split_bytes_peak": 7431429632, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109500694528, + "reserved_bytes_peak": 109500694528, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.115960Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072635904, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95742273536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72055592448, + "active_bytes_peak": 95742273536, + "allocated_bytes_current": 72055592448, + "allocated_bytes_peak": 95742273536, + "device": 2, + "inactive_split_bytes_current": 2535909888, + "inactive_split_bytes_peak": 5611729408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 101917392896, + "reserved_bytes_peak": 101917392896, + "segment_count_current": 1119, + "segment_count_peak": 1119, + "timestamp_utc": "2026-08-26T19:09:07.301631Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2458049024, + "inactive_split_bytes_current_delta": -77860864, + "inactive_split_bytes_peak": 7431429632, + "inactive_split_bytes_peak_delta": 1819700224, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109500694528, + "reserved_bytes_current_delta": 7583301632, + "reserved_bytes_peak": 109500694528, + "reserved_bytes_peak_delta": 7583301632, + "segment_count_current": 1126, + "segment_count_current_delta": 7, + "segment_count_peak": 1126, + "segment_count_peak_delta": 7, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T190909116410Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.149717Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190909116410Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190909116410Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f42dc05b446388d9a1ea9c76f378fb030c0d58f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190909116410Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a6fae2206c225bc8beb9eaf2623d5212b87187a030b344fdf47ed2d5ed1a2eb +size 1948022 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190909338588Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190909338588Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3b45bc572fac1d1cc0ae246fbd38a1a4b2720b98 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190909338588Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072635904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100106798080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72072635904, + "active_bytes_peak": 100106798080, + "allocated_bytes_current": 72072635904, + "allocated_bytes_peak": 95742273536, + "device": 2, + "inactive_split_bytes_current": 2458049024, + "inactive_split_bytes_peak": 7431429632, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109500694528, + "reserved_bytes_peak": 109500694528, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.338185Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072635904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95742273536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72072635904, + "active_bytes_peak": 100106798080, + "allocated_bytes_current": 72072635904, + "allocated_bytes_peak": 95742273536, + "device": 2, + "inactive_split_bytes_current": 2458049024, + "inactive_split_bytes_peak": 7431429632, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109500694528, + "reserved_bytes_peak": 109500694528, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.153643Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2458049024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7431429632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109500694528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109500694528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1126, + "segment_count_current_delta": 0, + "segment_count_peak": 1126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T190909338588Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.345340Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190909338588Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190909338588Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb5a8d93efe4979b44f213f3df8241ed8bf74c45 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190909338588Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ff919eaa6e243added57591c91db2ce7c400f7b31a2655943a9c0af202ded30 +size 301831 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190909528846Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190909528846Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0f33219693c4db0dacce9211e52d4c515d2eac9c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190909528846Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072635904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100106798080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72072635904, + "active_bytes_peak": 100106798080, + "allocated_bytes_current": 72072635904, + "allocated_bytes_peak": 95742273536, + "device": 2, + "inactive_split_bytes_current": 2458049024, + "inactive_split_bytes_peak": 7431429632, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109500694528, + "reserved_bytes_peak": 109500694528, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.528458Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072635904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95742273536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72072635904, + "active_bytes_peak": 100106798080, + "allocated_bytes_current": 72072635904, + "allocated_bytes_peak": 95742273536, + "device": 2, + "inactive_split_bytes_current": 2458049024, + "inactive_split_bytes_peak": 7431429632, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109500694528, + "reserved_bytes_peak": 109500694528, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.346306Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2458049024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7431429632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109500694528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109500694528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1126, + "segment_count_current_delta": 0, + "segment_count_peak": 1126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T190909528846Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.535474Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190909528846Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190909528846Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1e529c2f42167337d43b1119a5b12328f78facc2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190909528846Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d34e12322c0cfa1b2be1281ae50e5512823d84cadabd864c9fd30772f91a532 +size 301862 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190906603295Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190906603295Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a92442b096a7abb5562dd31441677107907f4e04 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190906603295Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 31386458624, + "active_bytes_peak": 74769438720, + "active_bytes_peak_delta": 34045721600, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 74769438720, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 74769438720, + "device": 3, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_peak": 5801770496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 85582675968, + "reserved_bytes_peak": 85582675968, + "segment_count_current": 984, + "segment_count_peak": 984, + "timestamp_utc": "2026-08-26T19:09:06.602598Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 31386458624, + "allocated_bytes_peak": 74769438720, + "allocated_bytes_peak_delta": 34045721600, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 3, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:08:29.045699Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_current_delta": 2016978432, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 5073181184, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 85582675968, + "reserved_bytes_current_delta": 41718644736, + "reserved_bytes_peak": 85582675968, + "reserved_bytes_peak_delta": 41718644736, + "segment_count_current": 984, + "segment_count_current_delta": 723, + "segment_count_peak": 984, + "segment_count_peak_delta": 723, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T190906603295Z.pickle", + "timestamp_utc": "2026-08-26T19:09:06.643764Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190906603295Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190906603295Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b27aca99d4d47fa413a040e10bc9388aa40c8641 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190906603295Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:004a3fc2a7a4d46db1bb3d923c858538eea571ceed5439b4584ba7a1b895f049 +size 2034545 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190906992125Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190906992125Z.json new file mode 100644 index 0000000000000000000000000000000000000000..372a74be8b9b9b501284ff1f36a4b107409b4bf4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190906992125Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95738017792, + "active_bytes_peak_delta": 20968579072, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 95738017792, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 3, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 100948508672, + "reserved_bytes_peak": 100948508672, + "segment_count_current": 1095, + "segment_count_peak": 1095, + "timestamp_utc": "2026-08-26T19:09:06.991718Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 20968579072, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 74769438720, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 74769438720, + "device": 3, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_peak": 5801770496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 85582675968, + "reserved_bytes_peak": 85582675968, + "segment_count_current": 984, + "segment_count_peak": 984, + "timestamp_utc": "2026-08-26T19:09:06.650265Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 100948508672, + "reserved_bytes_current_delta": 15365832704, + "reserved_bytes_peak": 100948508672, + "reserved_bytes_peak_delta": 15365832704, + "segment_count_current": 1095, + "segment_count_current_delta": 111, + "segment_count_peak": 1095, + "segment_count_peak_delta": 111, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T190906992125Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.022483Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190906992125Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190906992125Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..842895311933ac9ec42d4a568d67348f450aa289 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190906992125Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:685d7abf3839f2a8a46e67ee5e092a2f104505b97a8d662ce4d874a68bed1f30 +size 1671440 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190907270766Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190907270766Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8c5fb3576fa23180e91b60b97c84aa266bfb6cee --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190907270766Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95738017792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 95738017792, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 3, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 101772689408, + "reserved_bytes_peak": 101772689408, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:09:07.270372Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 95738017792, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 3, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 100948508672, + "reserved_bytes_peak": 100948508672, + "segment_count_current": 1095, + "segment_count_peak": 1095, + "timestamp_utc": "2026-08-26T19:09:07.026006Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 101772689408, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 101772689408, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 1096, + "segment_count_current_delta": 1, + "segment_count_peak": 1096, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T190907270766Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.298914Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190907270766Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190907270766Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cc84e4234282d8c4ac52ee8cb479aec062117db0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190907270766Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad5b0d787bece2e82338575d11a5a7160081a8974fa1648d7ed4184678d4272a +size 1665540 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190909116430Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190909116430Z.json new file mode 100644 index 0000000000000000000000000000000000000000..08be1dbafcde7ffa4ae1fcd1206dc7205bc95af2 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190909116430Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 4385614848, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 3, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.115966Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 95738017792, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 3, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 101772689408, + "reserved_bytes_peak": 101772689408, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:09:07.302394Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": -77860864, + "inactive_split_bytes_peak": 7569784320, + "inactive_split_bytes_peak_delta": 1768013824, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 8029995008, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 8029995008, + "segment_count_current": 1126, + "segment_count_current_delta": 30, + "segment_count_peak": 1126, + "segment_count_peak_delta": 30, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T190909116430Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.150262Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190909116430Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190909116430Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29c6f7e770cc854864be8042f5481d7c9099b529 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190909116430Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abb5c3841deb135e7a48453fea9442b70b7a454ae4ff010980178ee7e79f2cb3 +size 1925547 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190909338614Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190909338614Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0d7a826812ade348a3d64c393d9f0c8ce5f93694 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190909338614Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 3, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.338195Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 3, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.154932Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7569784320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1126, + "segment_count_current_delta": 0, + "segment_count_peak": 1126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T190909338614Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.345402Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190909338614Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190909338614Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..984452d49ef4bf6bf85de6011ae009447089442b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190909338614Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6ebacc942c949fc657efc98a8560358b1e7230d25e3bf6561769714fe6b1e05 +size 297859 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190909528863Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190909528863Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7db3d86e707ba057734a85daf5c2de16e3f6184 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190909528863Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 3, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.528478Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 3, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.346397Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7569784320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1126, + "segment_count_current_delta": 0, + "segment_count_peak": 1126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T190909528863Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.535486Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190909528863Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190909528863Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..454cb0873f5b2843b464b15c94732b9a9eea4bdc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190909528863Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baef484c2ab1d54dd76ca76ffea65f3fcea35842126ba0daaa155bf8f015237d +size 297890 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190906603290Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190906603290Z.json new file mode 100644 index 0000000000000000000000000000000000000000..82a64de5b5634475b80e7bbf140879b0022c6ebb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190906603290Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 31386458624, + "active_bytes_peak": 74769438720, + "active_bytes_peak_delta": 34045721600, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 74769438720, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 74769438720, + "device": 4, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_peak": 5801770496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 85289074688, + "reserved_bytes_peak": 85289074688, + "segment_count_current": 983, + "segment_count_peak": 983, + "timestamp_utc": "2026-08-26T19:09:06.602584Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 31386458624, + "allocated_bytes_peak": 74769438720, + "allocated_bytes_peak_delta": 34045721600, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 4, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:08:29.045548Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_current_delta": 2016978432, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 5073181184, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 85289074688, + "reserved_bytes_current_delta": 41425043456, + "reserved_bytes_peak": 85289074688, + "reserved_bytes_peak_delta": 41425043456, + "segment_count_current": 983, + "segment_count_current_delta": 722, + "segment_count_peak": 983, + "segment_count_peak_delta": 722, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T190906603290Z.pickle", + "timestamp_utc": "2026-08-26T19:09:06.638675Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190906603290Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190906603290Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29170b8e1fd3f56c57565778b2b7c16eba4b18ca --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190906603290Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80c1225309f3a4ce703797a4caa2d9aaeaf8130e307eb6e575b9281a03b48f8c +size 2034376 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190906992104Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190906992104Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15b0d81092a98b7720555667314a2d413900f03e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190906992104Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95738017792, + "active_bytes_peak_delta": 20968579072, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 95738017792, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 4, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 100948508672, + "reserved_bytes_peak": 100948508672, + "segment_count_current": 1095, + "segment_count_peak": 1095, + "timestamp_utc": "2026-08-26T19:09:06.991698Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 20968579072, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 74769438720, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 74769438720, + "device": 4, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_peak": 5801770496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 85289074688, + "reserved_bytes_peak": 85289074688, + "segment_count_current": 983, + "segment_count_peak": 983, + "timestamp_utc": "2026-08-26T19:09:06.642914Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 100948508672, + "reserved_bytes_current_delta": 15659433984, + "reserved_bytes_peak": 100948508672, + "reserved_bytes_peak_delta": 15659433984, + "segment_count_current": 1095, + "segment_count_current_delta": 112, + "segment_count_peak": 1095, + "segment_count_peak_delta": 112, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T190906992104Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.022068Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190906992104Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190906992104Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..181630b55d59eb35bd78afa7166bff7a33a57bc0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190906992104Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732390e6612960a852a379f71a4affde824a04923f5bac74d220eeb9e3e1f678 +size 1668170 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190907270726Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190907270726Z.json new file mode 100644 index 0000000000000000000000000000000000000000..35dcd8cb3afc521495b6ff32243aac9fada8fb8e --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190907270726Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 96175921152, + "active_bytes_peak_delta": 437903360, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 96175921152, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 4, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 101772689408, + "reserved_bytes_peak": 101772689408, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:09:07.270323Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 95738017792, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 4, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 100948508672, + "reserved_bytes_peak": 100948508672, + "segment_count_current": 1095, + "segment_count_peak": 1095, + "timestamp_utc": "2026-08-26T19:09:07.025414Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 101772689408, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 101772689408, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 1096, + "segment_count_current_delta": 1, + "segment_count_peak": 1096, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T190907270726Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.297587Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190907270726Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190907270726Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ff4c7116d9faa55c737025712631fde70799db2f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190907270726Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:401a841d98095b6c4c0eb86e46f180b3941521ab8aa8054f20f1883f5a0dca8e +size 1659077 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190909116343Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190909116343Z.json new file mode 100644 index 0000000000000000000000000000000000000000..572d0d705dfa4c47b4785a6d6449fb69c0fae5ab --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190909116343Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 3947711488, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 4, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.115914Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 96175921152, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 4, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 101772689408, + "reserved_bytes_peak": 101772689408, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:09:07.300740Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": -77860864, + "inactive_split_bytes_peak": 7569784320, + "inactive_split_bytes_peak_delta": 1768013824, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 8029995008, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 8029995008, + "segment_count_current": 1126, + "segment_count_current_delta": 30, + "segment_count_peak": 1126, + "segment_count_peak_delta": 30, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T190909116343Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.149332Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190909116343Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190909116343Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fc9de666c5b81b8366bd046ffa8839f4c731187b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190909116343Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01778677a60674072be562eac770146b4a9524cf62d0533b2c112ae3c42db92e +size 1925428 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190909338555Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190909338555Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e22cbd6a592ab7cc9e290d37d5293b7d04eeb697 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190909338555Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 4, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.338162Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 4, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.152923Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7569784320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1126, + "segment_count_current_delta": 0, + "segment_count_peak": 1126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T190909338555Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.345052Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190909338555Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190909338555Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f50d7df76c54bcc10e55a749e43905ca988c97c0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190909338555Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:338655e7500be3e359452ba65f21912755fba444f9d4fea6b9cff0b0f1fa2ab3 +size 297845 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190909528838Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190909528838Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c6575806de622d8cde61aab6cb1d617d14fc776d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190909528838Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 4, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.528446Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 4, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.346025Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7569784320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1126, + "segment_count_current_delta": 0, + "segment_count_peak": 1126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T190909528838Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.535323Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190909528838Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190909528838Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..23109a776559e7f7973b03a14e26fc9ee6bf9047 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190909528838Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77788a2ab9b90a576692037aa603678add35634289ea32ef2da59c16d102391a +size 297876 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190906603250Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190906603250Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96dd581de2a5dab44e20dccf3847cdcc4fbe3f95 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190906603250Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 31386458624, + "active_bytes_peak": 74769438720, + "active_bytes_peak_delta": 34045721600, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 74769438720, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 74769438720, + "device": 5, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_peak": 5801770496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 85289074688, + "reserved_bytes_peak": 85289074688, + "segment_count_current": 983, + "segment_count_peak": 983, + "timestamp_utc": "2026-08-26T19:09:06.602555Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 31386458624, + "allocated_bytes_peak": 74769438720, + "allocated_bytes_peak_delta": 34045721600, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 5, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:08:29.045602Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_current_delta": 2016978432, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 5073181184, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 85289074688, + "reserved_bytes_current_delta": 41425043456, + "reserved_bytes_peak": 85289074688, + "reserved_bytes_peak_delta": 41425043456, + "segment_count_current": 983, + "segment_count_current_delta": 722, + "segment_count_peak": 983, + "segment_count_peak_delta": 722, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T190906603250Z.pickle", + "timestamp_utc": "2026-08-26T19:09:06.638422Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190906603250Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190906603250Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..98b762c6c24c4e8b3aee439c07845a5451f68160 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190906603250Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66b263fb822099d7d46183bae3185cae6a1fd9fad8395a1f9d226cda413d38b8 +size 2034376 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190906992083Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190906992083Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1f1859fbcdc5b17fbadde999d93f79652eefb957 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190906992083Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 96178300928, + "active_bytes_peak_delta": 21408862208, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 96178300928, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 5, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 101242109952, + "reserved_bytes_peak": 101242109952, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:09:06.991672Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 20968579072, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 74769438720, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 74769438720, + "device": 5, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_peak": 5801770496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 85289074688, + "reserved_bytes_peak": 85289074688, + "segment_count_current": 983, + "segment_count_peak": 983, + "timestamp_utc": "2026-08-26T19:09:06.642821Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 101242109952, + "reserved_bytes_current_delta": 15953035264, + "reserved_bytes_peak": 101242109952, + "reserved_bytes_peak_delta": 15953035264, + "segment_count_current": 1096, + "segment_count_current_delta": 113, + "segment_count_peak": 1096, + "segment_count_peak_delta": 113, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T190906992083Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.022309Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190906992083Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190906992083Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ddf46b8c5c9d7113531cdb745897053dd404b43 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190906992083Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10e81a6eeccf4902fa5b57d6c69cb069a42978668255818c0deabc9b53c328fd +size 1668329 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190907270742Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190907270742Z.json new file mode 100644 index 0000000000000000000000000000000000000000..51a819e0469fa75e874741707a3dd9d83e38aaa3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190907270742Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 96178300928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 96178300928, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 5, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 102066290688, + "reserved_bytes_peak": 102066290688, + "segment_count_current": 1097, + "segment_count_peak": 1097, + "timestamp_utc": "2026-08-26T19:09:07.270334Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 96178300928, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 5, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 101242109952, + "reserved_bytes_peak": 101242109952, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:09:07.025886Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 102066290688, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 102066290688, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 1097, + "segment_count_current_delta": 1, + "segment_count_peak": 1097, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T190907270742Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.298860Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190907270742Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190907270742Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c7a44284285fc87e49653943c774bc7c302d173 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190907270742Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47bd6c683d48c10e4145523fb1606fd7cb1a84f4a7e8c03495131fde40827a54 +size 1659083 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190909116427Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190909116427Z.json new file mode 100644 index 0000000000000000000000000000000000000000..985898f5a3d910931bf6c7c383378aa20c1d11d4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190909116427Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 3945331712, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 5, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7571881472, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.115944Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 96178300928, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 5, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 102066290688, + "reserved_bytes_peak": 102066290688, + "segment_count_current": 1097, + "segment_count_peak": 1097, + "timestamp_utc": "2026-08-26T19:09:07.302358Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": -77860864, + "inactive_split_bytes_peak": 7571881472, + "inactive_split_bytes_peak_delta": 1770110976, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 7736393728, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 7736393728, + "segment_count_current": 1126, + "segment_count_current_delta": 29, + "segment_count_peak": 1126, + "segment_count_peak_delta": 29, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T190909116427Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.150032Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190909116427Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190909116427Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b79db08acc251c21314f3c6cb1a0df17bc92400 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190909116427Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf6c150ae2a493e05b1d95b87db54046fe7c1c4d56fd359a3e009be0ac9d50d0 +size 1925746 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190909338561Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190909338561Z.json new file mode 100644 index 0000000000000000000000000000000000000000..71ec63434e45bef3ee1b2de90d2bcbc67de91b94 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190909338561Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 5, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7571881472, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.338153Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 5, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7571881472, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.154025Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7571881472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1126, + "segment_count_current_delta": 0, + "segment_count_peak": 1126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T190909338561Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.345467Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190909338561Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190909338561Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..212092f8a53dc997e9489993f6275c5cc03dc076 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190909338561Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:101ca35b5d261711de11e8fc566da1fb844578a85460142d696c63a5608a757c +size 297946 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190909528824Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190909528824Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c7fa7490d9976bdeb3254cbac88e7d37e6ffeb8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190909528824Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 5, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7571881472, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.528434Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 5, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7571881472, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.346451Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7571881472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1126, + "segment_count_current_delta": 0, + "segment_count_peak": 1126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T190909528824Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.535585Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190909528824Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190909528824Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ed5de146632412ae9a438953089355328cdc77d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190909528824Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea402e8f94ef491fff4d721da140ae1e40ebeacaeb4dc85454248accf2b7e534 +size 297977 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190906603283Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190906603283Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a5628f531988e6a6c2f175242bf45c190e078718 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190906603283Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 31386458624, + "active_bytes_peak": 74769438720, + "active_bytes_peak_delta": 34045721600, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 74769438720, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 74769438720, + "device": 6, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_peak": 5801770496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 84701872128, + "reserved_bytes_peak": 84701872128, + "segment_count_current": 981, + "segment_count_peak": 981, + "timestamp_utc": "2026-08-26T19:09:06.602656Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 31386458624, + "allocated_bytes_peak": 74769438720, + "allocated_bytes_peak_delta": 34045721600, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 6, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:08:29.045635Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_current_delta": 2016978432, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 5073181184, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 84701872128, + "reserved_bytes_current_delta": 40837840896, + "reserved_bytes_peak": 84701872128, + "reserved_bytes_peak_delta": 40837840896, + "segment_count_current": 981, + "segment_count_current_delta": 720, + "segment_count_peak": 981, + "segment_count_peak_delta": 720, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T190906603283Z.pickle", + "timestamp_utc": "2026-08-26T19:09:06.638474Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190906603283Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190906603283Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b88b1602e77aa15e1ab4e6d6739bd93633877fd8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190906603283Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb29186ccd71d91695084e4274997e69a59449be5cfe3acfb9af5df4cdcb6ab3 +size 2034035 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190906992067Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190906992067Z.json new file mode 100644 index 0000000000000000000000000000000000000000..45a6f1e0c1c9492649080d0aeaf466cd94342d65 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190906992067Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95738017792, + "active_bytes_peak_delta": 20968579072, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 95738017792, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 6, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 101242109952, + "reserved_bytes_peak": 101242109952, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:09:06.991666Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 20968579072, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 74769438720, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 74769438720, + "device": 6, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_peak": 5801770496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 84701872128, + "reserved_bytes_peak": 84701872128, + "segment_count_current": 981, + "segment_count_peak": 981, + "timestamp_utc": "2026-08-26T19:09:06.642864Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 101242109952, + "reserved_bytes_current_delta": 16540237824, + "reserved_bytes_peak": 101242109952, + "reserved_bytes_peak_delta": 16540237824, + "segment_count_current": 1096, + "segment_count_current_delta": 115, + "segment_count_peak": 1096, + "segment_count_peak_delta": 115, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T190906992067Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.022002Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190906992067Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190906992067Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d670601e54639e47c32225d963f16d02443a07ae --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190906992067Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed69c8e820bc00ab593bb6ffca7bb657340a8377b66f5af9df3f5b5750f7896b +size 1669731 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190907270707Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190907270707Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15a57624693a8b477e77799646c1827e63559ba8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190907270707Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 96175921152, + "active_bytes_peak_delta": 437903360, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 96175921152, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 6, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 102066290688, + "reserved_bytes_peak": 102066290688, + "segment_count_current": 1097, + "segment_count_peak": 1097, + "timestamp_utc": "2026-08-26T19:09:07.270309Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 95738017792, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 6, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 101242109952, + "reserved_bytes_peak": 101242109952, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:09:07.025439Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 102066290688, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 102066290688, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 1097, + "segment_count_current_delta": 1, + "segment_count_peak": 1097, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T190907270707Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.298087Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190907270707Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190907270707Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d565043b0f357273c065b0b119aa9791ada3901d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190907270707Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae2033502902c5d33ce2b148466f7dad11d43f2931d746f0915b9dd62ca17e8b +size 1660368 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190909116414Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190909116414Z.json new file mode 100644 index 0000000000000000000000000000000000000000..968dc0a1d31ea1400f89aa06ef77b052a52d1099 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190909116414Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 3947711488, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 6, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.115929Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 96175921152, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 6, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 102066290688, + "reserved_bytes_peak": 102066290688, + "segment_count_current": 1097, + "segment_count_peak": 1097, + "timestamp_utc": "2026-08-26T19:09:07.301430Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": -77860864, + "inactive_split_bytes_peak": 7569784320, + "inactive_split_bytes_peak_delta": 1768013824, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 7736393728, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 7736393728, + "segment_count_current": 1126, + "segment_count_current_delta": 29, + "segment_count_peak": 1126, + "segment_count_peak_delta": 29, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T190909116414Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.149356Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190909116414Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190909116414Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44756738f972a2972cf8a8d82d3bc9be9669757d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190909116414Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dbb8274d3b080f33addd95e3b43469c6c3bb6d59f2a68a0fbd443d25bde3da3 +size 1925740 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190909338557Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190909338557Z.json new file mode 100644 index 0000000000000000000000000000000000000000..348af0dfe5bc76be6919410c14405a1a870215f4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190909338557Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 6, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.338143Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 6, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.153313Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7569784320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1126, + "segment_count_current_delta": 0, + "segment_count_peak": 1126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T190909338557Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.345296Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190909338557Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190909338557Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e498ef2033b421cd68893e78ab97c494587e4e4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190909338557Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3360d7d2111aa9772309672c33424447e7b5feb49f6f44c35171db7b057dce72 +size 297940 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190909528833Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190909528833Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9e6403a923446cef24f1e405bccdfdd5843f7ea6 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190909528833Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 6, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.528442Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 6, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7569784320, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.346256Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7569784320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1126, + "segment_count_current_delta": 0, + "segment_count_peak": 1126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T190909528833Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.535542Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190909528833Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190909528833Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eeaf93a41a5ab8b9337671779d6bc38d647c74f3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190909528833Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cba78a6b0d9a9cbb3552e217a96ae451eb019c39c077e62f44ff9608d210ee49 +size 297971 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190906606494Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190906606494Z.json new file mode 100644 index 0000000000000000000000000000000000000000..63c0c80d3012d72ae99c6e27e867c3049480931c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190906606494Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 31386458624, + "active_bytes_peak": 74769438720, + "active_bytes_peak_delta": 34045721600, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 74769438720, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 74769438720, + "device": 7, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_peak": 5801770496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 84408270848, + "reserved_bytes_peak": 84408270848, + "segment_count_current": 980, + "segment_count_peak": 980, + "timestamp_utc": "2026-08-26T19:09:06.605808Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 31386458624, + "allocated_bytes_peak": 74769438720, + "allocated_bytes_peak_delta": 34045721600, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 7, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:08:29.045670Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_current_delta": 2016978432, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 5073181184, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 84408270848, + "reserved_bytes_current_delta": 40544239616, + "reserved_bytes_peak": 84408270848, + "reserved_bytes_peak_delta": 40544239616, + "segment_count_current": 980, + "segment_count_current_delta": 719, + "segment_count_peak": 980, + "segment_count_peak_delta": 719, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T190906606494Z.pickle", + "timestamp_utc": "2026-08-26T19:09:06.642959Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190906606494Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190906606494Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ffa8ffbdf591ecf8ccd895e8c05be57f48103a43 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190906606494Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2aa2d394d23d122702dd8c4e5ea9782a8d81805062bc184a5c44fd0e502c5426 +size 2033869 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190906992121Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190906992121Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04df7c33de3059bbad614993daa1dce627cbe565 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190906992121Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95738017792, + "active_bytes_peak_delta": 20968579072, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 95738017792, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 7, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 100948508672, + "reserved_bytes_peak": 100948508672, + "segment_count_current": 1095, + "segment_count_peak": 1095, + "timestamp_utc": "2026-08-26T19:09:06.991718Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 20968579072, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 74769438720, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 74769438720, + "device": 7, + "inactive_split_bytes_current": 2525366784, + "inactive_split_bytes_peak": 5801770496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 84408270848, + "reserved_bytes_peak": 84408270848, + "segment_count_current": 980, + "segment_count_peak": 980, + "timestamp_utc": "2026-08-26T19:09:06.647293Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 100948508672, + "reserved_bytes_current_delta": 16540237824, + "reserved_bytes_peak": 100948508672, + "reserved_bytes_peak_delta": 16540237824, + "segment_count_current": 1095, + "segment_count_current_delta": 115, + "segment_count_peak": 1095, + "segment_count_peak_delta": 115, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T190906992121Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.020699Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190906992121Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190906992121Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a1c6703f6be61bef47910a8fdfe3d9e38508b125 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190906992121Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cf9281d25fea36ffff74c9ea5ee65cdcc2e6f90a9524cd7426450409532e900 +size 1669601 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190907270725Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190907270725Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9acff91aef137bcb3d4a2c274e3674f6074c5b0c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190907270725Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72072427008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 95738017792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 95738017792, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 7, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 101772689408, + "reserved_bytes_peak": 101772689408, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:09:07.270336Z", + "world_size": 8 + }, + "allocated_bytes_current": 72072427008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 95738017792, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 7, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 100948508672, + "reserved_bytes_peak": 100948508672, + "segment_count_current": 1095, + "segment_count_peak": 1095, + "timestamp_utc": "2026-08-26T19:09:07.024024Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5801770496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 101772689408, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 101772689408, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 1096, + "segment_count_current_delta": 1, + "segment_count_peak": 1096, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T190907270725Z.pickle", + "timestamp_utc": "2026-08-26T19:09:07.297470Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190907270725Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190907270725Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5b4721337d8370bc183dc74063dc4afb5ab7e24 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190907270725Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75883fa1c72cba96d75322de0e1871a35a748600bd358015e439ef7309434acf +size 1660086 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190909116412Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190909116412Z.json new file mode 100644 index 0000000000000000000000000000000000000000..093b490bc93100182ca3087912ba41c1c2106dd7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190909116412Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 4385614848, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 7, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7716584960, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.115969Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72072427008, + "active_bytes_peak": 95738017792, + "allocated_bytes_current": 72072427008, + "allocated_bytes_peak": 95738017792, + "device": 7, + "inactive_split_bytes_current": 2821065216, + "inactive_split_bytes_peak": 5801770496, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 101772689408, + "reserved_bytes_peak": 101772689408, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:09:07.300654Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": -77860864, + "inactive_split_bytes_peak": 7716584960, + "inactive_split_bytes_peak_delta": 1914814464, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 8029995008, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 8029995008, + "segment_count_current": 1126, + "segment_count_current_delta": 30, + "segment_count_peak": 1126, + "segment_count_peak_delta": 30, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T190909116412Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.148872Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190909116412Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190909116412Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e0894284765f6cb6bb25eb826bb3b854c142c3f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190909116412Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0e664c8bf04e83e184bd3e2b2f9da4d26b06417edd4bc6a29370dd01d86ed94 +size 1925422 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190909338690Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190909338690Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5719dfbb69922157c738747fa75bdb92d8d9089f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190909338690Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 7, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7716584960, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.338267Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 7, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7716584960, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.152542Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7716584960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1126, + "segment_count_current_delta": 0, + "segment_count_peak": 1126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T190909338690Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.345638Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190909338690Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190909338690Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5ebe88328cda4b7cb45b99da9a1c0e6db9b71a9 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190909338690Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfd63fb0bf6775484538df9a0342dc2f6f53be06af6365b7547a9575cbf8909d +size 297839 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190909528946Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190909528946Z.json new file mode 100644 index 0000000000000000000000000000000000000000..29547512825ad215fde311c2270d911efca75fb4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190909528946Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72089470464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 100123632640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 7, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7716584960, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.528523Z", + "world_size": 8 + }, + "allocated_bytes_current": 72089470464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 95738017792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72089470464, + "active_bytes_peak": 100123632640, + "allocated_bytes_current": 72089470464, + "allocated_bytes_peak": 95738017792, + "device": 7, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_peak": 7716584960, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 109802684416, + "reserved_bytes_peak": 109802684416, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:09:09.346635Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2743204352, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7716584960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 109802684416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109802684416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1126, + "segment_count_current_delta": 0, + "segment_count_peak": 1126, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T190909528946Z.pickle", + "timestamp_utc": "2026-08-26T19:09:09.535780Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190909528946Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190909528946Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe28e96313a6ada620b8b0a4abbc50af31eb83ac --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190909528946Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c317dc2dfddc371a2f8bf192a710eedf146fb86f1507dde455a9859dce924fe8 +size 297870 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/run.env b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..13299605058b067f988ad82fafe2371465123747 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN2_57B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=better-memory-dump-support +commit=b7c52442 +num_layers=10 +hidden=3584 +ffn_hidden=2560 +moe_ffn_hidden=2560 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=27525120 +heads=28 +kv_groups=4 +kv_channels= +qk_layernorm=0 +shared_ffn=20480 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=10 +rotary_base=1000000 +rotary_scaling= +qkv_bias=1 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.25 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/train.log b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..89bc2eebc997c2450f3826f466a31579aa7af828 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/train.log @@ -0,0 +1,1194 @@ +============================================================== + QWEN2_57B / echo (slots/rank=8 policy=matched) + E=64 K=8 H'=2560 m=8 + L=10 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b7c52442 +============================================================== +W0826 19:08:04.578000 618092 torch/distributed/run.py:766] +W0826 19:08:04.578000 618092 torch/distributed/run.py:766] ***************************************** +W0826 19:08:04.578000 618092 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:08:04.578000 618092 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 10 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 2560 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 3584 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2560 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 20480 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 28 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 10 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank [Gloo] Rank 0 is connected to 17 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 + +[Gloo] Rank 2 is connected to [Gloo] Rank 7 peer ranks. Expected number of connected peer ranks is : 77 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 57 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 153467 is connected to is connected to is connected to is connected to is connected to is connected to 777777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 +77 +77 + + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W826 19:08:22.673225982 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:08:22.673385469 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:08:22.673596738 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:08:22.673660053 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:08:22.673704736 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:08:22.673745194 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:08:22.678064251 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:08:22.827992397 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.806 seconds +time to initialize megatron (seconds): 6.787 +[after megatron is initialized] datetime: 2026-08-26 19:08:28 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5060614656 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22 +Params for bucket 1 (180289536 elements, 180289536 padded size): + module.output_layer.weight +Params for bucket 2 (73403904 elements, 73403904 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (146800640 elements, 146800640 padded size): + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight +Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.bias + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight +Params for bucket 5 (146800640 elements, 146800640 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 7 (146800640 elements, 146800640 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight +Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 9 (146800640 elements, 146800640 padded size): + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 10 (103001600 elements, 103001600 padded size): + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.bias + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight +Params for bucket 11 (146800640 elements, 146800640 padded size): + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (103001600 elements, 103001600 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_qkv.bias + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 13 (146800640 elements, 146800640 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight +Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.self_attention.linear_qkv.bias + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight +Params for bucket 15 (146800640 elements, 146800640 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight +Params for bucket 16 (103001600 elements, 103001600 padded size): + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.bias + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight +Params for bucket 17 (146800640 elements, 146800640 padded size): + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.bias + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight +Params for bucket 19 (146800640 elements, 146800640 padded size): + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 20 (103001600 elements, 103001600 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.bias + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight +Params for bucket 21 (146800640 elements, 146800640 padded size): + module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight +Params for bucket 22 (209890816 elements, 209890816 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 +Params for bucket 1 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 +Params for bucket 2 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 +Params for bucket 3 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 4 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 +Params for bucket 5 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 +Params for bucket 6 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 +Params for bucket 7 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 8 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 9 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 10 (45875200 elements, 45875200 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 11 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 +Params for bucket 12 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 +Params for bucket 13 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 14 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 16 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 +Params for bucket 17 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 18 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 +Params for bucket 19 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 20 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 21 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 22 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 +Params for bucket 23 (45875200 elements, 45875200 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 +Params for bucket 24 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 25 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 26 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 +Params for bucket 27 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 +Params for bucket 28 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 +Params for bucket 30 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 31 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 +Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 33 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 34 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (45875200 elements, 45875200 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 +Params for bucket 37 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 39 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 40 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 +Params for bucket 41 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 +Params for bucket 42 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 +Params for bucket 43 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +Params for bucket 44 (18350080 elements, 18350080 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:08:28 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:08:28 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (331.83, 336.21) + train/valid/test-data-iterators-setup ..........: (58.70, 76.35) +[before the start of training step] datetime: 2026-08-26 19:08:29 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +Number of parameters in transformer block in billions: 20.11 [2026-08-26 19:09:06] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 37556.1 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097931E+01 | loss scale: 1.0 | grad norm: 29.871 | number of skipped iterations: 0 | number of nan iterations: 0 | + +Number of parameters in embedding layers in billions: 0.36 +Total number of parameters in billions: 20.47 +Number of parameters in most loaded shard in billions: 20.4724 +Theoretical memory footprints: weight and optimizer=146430.14 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 68731.85595703125 | max allocated: 71308.6533203125 | reserved: 81318.0 | max reserved: 81318.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 19:09:06] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 389.9 | throughput per GPU (TFLOP/s/GPU): 316.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097906E+01 | loss scale: 1.0 | grad norm: 30.629 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:09:07] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 278.6 | throughput per GPU (TFLOP/s/GPU): 443.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.538876E+00 | loss scale: 1.0 | grad norm: 32.400 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:09:09] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 1845.5 | throughput per GPU (TFLOP/s/GPU): 66.9 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.109478E+01 | loss scale: 1.0 | grad norm: 107.149 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:09:09] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 222.2 | throughput per GPU (TFLOP/s/GPU): 555.8 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.998207E+00 | loss scale: 1.0 | grad norm: 26.264 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:09:09] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 190.2 | throughput per GPU (TFLOP/s/GPU): 649.2 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.049684E+01 | loss scale: 1.0 | grad norm: 8.719 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 19:09:09 +[rank0]:[W826 19:09:10.873739957 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:09:13.257000 618092 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:09:13.257000 618092 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 618181 closing signal SIGTERM +W0826 19:09:13.258000 618092 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 618183 closing signal SIGTERM +W0826 19:09:13.258000 618092 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 618184 closing signal SIGTERM +W0826 19:09:13.258000 618092 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 618186 closing signal SIGTERM +W0826 19:09:13.258000 618092 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 618187 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 618092 got signal: 15 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..81f2f899b33e840062c7f8b479771e97a2dba451 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv @@ -0,0 +1,240 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 43875 +1, 43307 +2, 43307 +3, 44007 +4, 44007 +5, 44007 +6, 44007 +7, 44007 +0, 44539 +1, 43973 +2, 43971 +3, 44671 +4, 44671 +5, 44671 +6, 44671 +7, 44671 +0, 47221 +1, 46653 +2, 46653 +3, 47353 +4, 47353 +5, 47353 +6, 47353 +7, 47353 +0, 47501 +1, 46933 +2, 46933 +3, 47633 +4, 47633 +5, 47633 +6, 47633 +7, 47633 +0, 47845 +1, 47263 +2, 47281 +3, 47957 +4, 47963 +5, 47955 +6, 47933 +7, 47951 +0, 48535 +1, 47967 +2, 47967 +3, 48667 +4, 48667 +5, 48667 +6, 48667 +7, 48667 +0, 48765 +1, 48197 +2, 48197 +3, 48897 +4, 48895 +5, 48897 +6, 48893 +7, 48895 +0, 49357 +1, 48783 +2, 48783 +3, 49483 +4, 49477 +5, 49483 +6, 49475 +7, 49479 +0, 49885 +1, 49313 +2, 49313 +3, 50013 +4, 50003 +5, 50013 +6, 49999 +7, 50007 +0, 55401 +1, 55339 +2, 55389 +3, 55337 +4, 55385 +5, 53867 +6, 55385 +7, 55329 +0, 77553 +1, 77569 +2, 77441 +3, 77547 +4, 77477 +5, 77439 +6, 77417 +7, 77513 +0, 77963 +1, 77969 +2, 77885 +3, 77945 +4, 77947 +5, 77887 +6, 77881 +7, 77951 +0, 77963 +1, 77969 +2, 77885 +3, 77945 +4, 77947 +5, 77887 +6, 77881 +7, 77951 +0, 78287 +1, 78293 +2, 78213 +3, 78273 +4, 78271 +5, 78217 +6, 78209 +7, 78275 +0, 78287 +1, 78295 +2, 78215 +3, 78273 +4, 78271 +5, 78217 +6, 78209 +7, 78275 +0, 78553 +1, 78547 +2, 78517 +3, 78551 +4, 78483 +5, 78443 +6, 78489 +7, 78563 +0, 79681 +1, 79699 +2, 80195 +3, 81061 +4, 79665 +5, 80475 +6, 79621 +7, 80249 +0, 87827 +1, 87595 +2, 88393 +3, 89135 +4, 87761 +5, 88595 +6, 87849 +7, 88107 +0, 102407 +1, 102767 +2, 102975 +3, 103023 +4, 103365 +5, 103541 +6, 101587 +7, 102877 +0, 102407 +1, 103553 +2, 102975 +3, 103809 +4, 103365 +5, 104327 +6, 102767 +7, 103663 +0, 2627 +1, 2627 +2, 2627 +3, 2630 +4, 2950 +5, 104327 +6, 4230 +7, 2630 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191024263855Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191024263855Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f3af36bce92fcde6b8fcfe9c74a897f87cde93c8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191024263855Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72125093888, + "active_bytes_current_delta": 31440172032, + "active_bytes_peak": 73629576704, + "active_bytes_peak_delta": 32906906112, + "after": { + "active_bytes_current": 72125093888, + "active_bytes_peak": 73629576704, + "allocated_bytes_current": 72125093888, + "allocated_bytes_peak": 73629576704, + "device": 0, + "inactive_split_bytes_current": 2200070144, + "inactive_split_bytes_peak": 4567912960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 83449872384, + "reserved_bytes_peak": 83449872384, + "segment_count_current": 1031, + "segment_count_peak": 1031, + "timestamp_utc": "2026-08-26T19:10:24.263346Z", + "world_size": 8 + }, + "allocated_bytes_current": 72125093888, + "allocated_bytes_current_delta": 31440172032, + "allocated_bytes_peak": 73629576704, + "allocated_bytes_peak_delta": 32906906112, + "before": { + "active_bytes_current": 40684921856, + "active_bytes_peak": 40722670592, + "allocated_bytes_current": 40684921856, + "allocated_bytes_peak": 40722670592, + "device": 0, + "inactive_split_bytes_current": 373120000, + "inactive_split_bytes_peak": 593320960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 43723522048, + "reserved_bytes_peak": 43723522048, + "segment_count_current": 262, + "segment_count_peak": 262, + "timestamp_utc": "2026-08-26T19:09:42.057363Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2200070144, + "inactive_split_bytes_current_delta": 1826950144, + "inactive_split_bytes_peak": 4567912960, + "inactive_split_bytes_peak_delta": 3974592000, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 83449872384, + "reserved_bytes_current_delta": 39726350336, + "reserved_bytes_peak": 83449872384, + "reserved_bytes_peak_delta": 39726350336, + "segment_count_current": 1031, + "segment_count_current_delta": 769, + "segment_count_peak": 1031, + "segment_count_peak_delta": 769, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T191024263855Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.305391Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191024263855Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191024263855Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..991a79fcffdc1527b3f99169fe5ebf3c76f8a360 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191024263855Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df91e32534d104471b79216e7ce34790dbcf80637e766f61fdd64223c20f647e +size 1789908 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191024742460Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191024742460Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9230a246112b4dcddf126c769bc4cf0dbadc8c77 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191024742460Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72125093888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94553344000, + "active_bytes_peak_delta": 20923767296, + "after": { + "active_bytes_current": 72125093888, + "active_bytes_peak": 94553344000, + "allocated_bytes_current": 72125093888, + "allocated_bytes_peak": 94553344000, + "device": 0, + "inactive_split_bytes_current": 2493671424, + "inactive_split_bytes_peak": 4567912960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 97421099008, + "reserved_bytes_peak": 97421099008, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:10:24.742017Z", + "world_size": 8 + }, + "allocated_bytes_current": 72125093888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94553344000, + "allocated_bytes_peak_delta": 20923767296, + "before": { + "active_bytes_current": 72125093888, + "active_bytes_peak": 73629576704, + "allocated_bytes_current": 72125093888, + "allocated_bytes_peak": 73629576704, + "device": 0, + "inactive_split_bytes_current": 2200070144, + "inactive_split_bytes_peak": 4567912960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 83449872384, + "reserved_bytes_peak": 83449872384, + "segment_count_current": 1031, + "segment_count_peak": 1031, + "timestamp_utc": "2026-08-26T19:10:24.310148Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2493671424, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 4567912960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 97421099008, + "reserved_bytes_current_delta": 13971226624, + "reserved_bytes_peak": 97421099008, + "reserved_bytes_peak_delta": 13971226624, + "segment_count_current": 1123, + "segment_count_current_delta": 92, + "segment_count_peak": 1123, + "segment_count_peak_delta": 92, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T191024742460Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.775570Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191024742460Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191024742460Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d2a97cbebba92410bbd178d00363c283d2cf4e7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191024742460Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7932765338920493f247a7e4a3fb99ba3bd30ee800e69cced23c545d079fd437 +size 1701763 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191025066190Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191025066190Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0389efaceea3cc6425e23a037db49d0fbf569938 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191025066190Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72125093888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94553344000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72125093888, + "active_bytes_peak": 94553344000, + "allocated_bytes_current": 72125093888, + "allocated_bytes_peak": 94553344000, + "device": 0, + "inactive_split_bytes_current": 2493671424, + "inactive_split_bytes_peak": 4630721536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98245279744, + "reserved_bytes_peak": 98245279744, + "segment_count_current": 1124, + "segment_count_peak": 1124, + "timestamp_utc": "2026-08-26T19:10:25.065787Z", + "world_size": 8 + }, + "allocated_bytes_current": 72125093888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94553344000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72125093888, + "active_bytes_peak": 94553344000, + "allocated_bytes_current": 72125093888, + "allocated_bytes_peak": 94553344000, + "device": 0, + "inactive_split_bytes_current": 2493671424, + "inactive_split_bytes_peak": 4567912960, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 97421099008, + "reserved_bytes_peak": 97421099008, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:10:24.779461Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2493671424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4630721536, + "inactive_split_bytes_peak_delta": 62808576, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98245279744, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 98245279744, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 1124, + "segment_count_current_delta": 1, + "segment_count_peak": 1124, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T191025066190Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.094484Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191025066190Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191025066190Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6230994c36be8f06627306df818f4a808e0b4082 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191025066190Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bda23d925947e422d777d9ba6becd4de123a8e544bc4c49b185df4c502af879 +size 1696525 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191025387949Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191025387949Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a4db55f7838e3d4266476e3f7324eafc014e869 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191025387949Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72125093888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94556896768, + "active_bytes_peak_delta": 3552768, + "after": { + "active_bytes_current": 72125093888, + "active_bytes_peak": 94556896768, + "allocated_bytes_current": 72125093888, + "allocated_bytes_peak": 94556896768, + "device": 0, + "inactive_split_bytes_current": 2493671424, + "inactive_split_bytes_peak": 5055816704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98245279744, + "reserved_bytes_peak": 98245279744, + "segment_count_current": 1124, + "segment_count_peak": 1124, + "timestamp_utc": "2026-08-26T19:10:25.387544Z", + "world_size": 8 + }, + "allocated_bytes_current": 72125093888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94556896768, + "allocated_bytes_peak_delta": 3552768, + "before": { + "active_bytes_current": 72125093888, + "active_bytes_peak": 94553344000, + "allocated_bytes_current": 72125093888, + "allocated_bytes_peak": 94553344000, + "device": 0, + "inactive_split_bytes_current": 2493671424, + "inactive_split_bytes_peak": 4630721536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98245279744, + "reserved_bytes_peak": 98245279744, + "segment_count_current": 1124, + "segment_count_peak": 1124, + "timestamp_utc": "2026-08-26T19:10:25.098059Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2493671424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5055816704, + "inactive_split_bytes_peak_delta": 425095168, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98245279744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 98245279744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1124, + "segment_count_current_delta": 0, + "segment_count_peak": 1124, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T191025387949Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.416055Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191025387949Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191025387949Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea088f6e3c4cdd8449e6e611cf3d69171164ce53 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191025387949Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1433000880820eecce93e217b93e51a91a0e4ce8eb74eb37129fd9b20229461 +size 1696522 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191025709621Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191025709621Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e401800074e03b44ace9102fdcbb6f0e48980f96 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191025709621Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72125093888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94556896768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72125093888, + "active_bytes_peak": 94556896768, + "allocated_bytes_current": 72125093888, + "allocated_bytes_peak": 94556896768, + "device": 0, + "inactive_split_bytes_current": 2493671424, + "inactive_split_bytes_peak": 5055816704, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98245279744, + "reserved_bytes_peak": 98245279744, + "segment_count_current": 1124, + "segment_count_peak": 1124, + "timestamp_utc": "2026-08-26T19:10:25.709224Z", + "world_size": 8 + }, + "allocated_bytes_current": 72125093888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94556896768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72125093888, + "active_bytes_peak": 94556896768, + "allocated_bytes_current": 72125093888, + "allocated_bytes_peak": 94556896768, + "device": 0, + "inactive_split_bytes_current": 2493671424, + "inactive_split_bytes_peak": 5055816704, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98245279744, + "reserved_bytes_peak": 98245279744, + "segment_count_current": 1124, + "segment_count_peak": 1124, + "timestamp_utc": "2026-08-26T19:10:25.419578Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2493671424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5055816704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98245279744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 98245279744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1124, + "segment_count_current_delta": 0, + "segment_count_peak": 1124, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T191025709621Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.737484Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191025709621Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191025709621Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..931b4c55ea8c6b11efebcc78351c484406764ccc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191025709621Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:324ab9858ce92a3021c09d4d229331a88fcb6da9a2c558fb426b37f05b153d82 +size 1696471 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191024263885Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191024263885Z.json new file mode 100644 index 0000000000000000000000000000000000000000..23aea77c8bad99871073b624821337cdfa850805 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191024263885Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72110034432, + "active_bytes_current_delta": 31424066048, + "active_bytes_peak": 73619110912, + "active_bytes_peak_delta": 32895393792, + "after": { + "active_bytes_current": 72110034432, + "active_bytes_peak": 73619110912, + "allocated_bytes_current": 72110034432, + "allocated_bytes_peak": 73619110912, + "device": 1, + "inactive_split_bytes_current": 1877488128, + "inactive_split_bytes_peak": 4565201920, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 83160465408, + "reserved_bytes_peak": 83160465408, + "segment_count_current": 1028, + "segment_count_peak": 1028, + "timestamp_utc": "2026-08-26T19:10:24.263251Z", + "world_size": 8 + }, + "allocated_bytes_current": 72110034432, + "allocated_bytes_current_delta": 31424066048, + "allocated_bytes_peak": 73619110912, + "allocated_bytes_peak_delta": 32895393792, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 1, + "inactive_split_bytes_current": 302867456, + "inactive_split_bytes_peak": 523068416, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 43130028032, + "reserved_bytes_peak": 43130028032, + "segment_count_current": 251, + "segment_count_peak": 251, + "timestamp_utc": "2026-08-26T19:09:42.057263Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1877488128, + "inactive_split_bytes_current_delta": 1574620672, + "inactive_split_bytes_peak": 4565201920, + "inactive_split_bytes_peak_delta": 4042133504, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 83160465408, + "reserved_bytes_current_delta": 40030437376, + "reserved_bytes_peak": 83160465408, + "reserved_bytes_peak_delta": 40030437376, + "segment_count_current": 1028, + "segment_count_current_delta": 777, + "segment_count_peak": 1028, + "segment_count_peak_delta": 777, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T191024263885Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.299656Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191024263885Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191024263885Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..64e823a9eab4affae7454641f2177464911773c5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191024263885Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a394863f3c5e192441afc1e4c24a9a552b17cee1ef0c9ac6bd425d5038c8fb61 +size 2032850 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191024742477Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191024742477Z.json new file mode 100644 index 0000000000000000000000000000000000000000..57aaee0965d28e29256156089aaf13a46cb8fa6b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191024742477Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72110034432, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94541315072, + "active_bytes_peak_delta": 20922204160, + "after": { + "active_bytes_current": 72110034432, + "active_bytes_peak": 94541315072, + "allocated_bytes_current": 72110034432, + "allocated_bytes_peak": 94541315072, + "device": 1, + "inactive_split_bytes_current": 2171089408, + "inactive_split_bytes_peak": 4565201920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 97385447424, + "reserved_bytes_peak": 97385447424, + "segment_count_current": 1128, + "segment_count_peak": 1128, + "timestamp_utc": "2026-08-26T19:10:24.742029Z", + "world_size": 8 + }, + "allocated_bytes_current": 72110034432, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94541315072, + "allocated_bytes_peak_delta": 20922204160, + "before": { + "active_bytes_current": 72110034432, + "active_bytes_peak": 73619110912, + "allocated_bytes_current": 72110034432, + "allocated_bytes_peak": 73619110912, + "device": 1, + "inactive_split_bytes_current": 1877488128, + "inactive_split_bytes_peak": 4565201920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 83160465408, + "reserved_bytes_peak": 83160465408, + "segment_count_current": 1028, + "segment_count_peak": 1028, + "timestamp_utc": "2026-08-26T19:10:24.304094Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2171089408, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 4565201920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 97385447424, + "reserved_bytes_current_delta": 14224982016, + "reserved_bytes_peak": 97385447424, + "reserved_bytes_peak_delta": 14224982016, + "segment_count_current": 1128, + "segment_count_current_delta": 100, + "segment_count_peak": 1128, + "segment_count_peak_delta": 100, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T191024742477Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.775541Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191024742477Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191024742477Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8f96815d8d9729bfc76eb95e70df63d76f5e9843 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191024742477Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd00d8dbc8ce00eab3ecc65ce9a5a4fa7e7012afbfe495f9de42dfec72b181ff +size 1699459 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191025066224Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191025066224Z.json new file mode 100644 index 0000000000000000000000000000000000000000..018c3984267957e5620b60a2c3161c7e9a7287aa --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191025066224Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72110034432, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94545565696, + "active_bytes_peak_delta": 4250624, + "after": { + "active_bytes_current": 72110034432, + "active_bytes_peak": 94545565696, + "allocated_bytes_current": 72110034432, + "allocated_bytes_peak": 94545565696, + "device": 1, + "inactive_split_bytes_current": 2171089408, + "inactive_split_bytes_peak": 5603483648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 98622767104, + "reserved_bytes_peak": 98622767104, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:10:25.065797Z", + "world_size": 8 + }, + "allocated_bytes_current": 72110034432, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94545565696, + "allocated_bytes_peak_delta": 4250624, + "before": { + "active_bytes_current": 72110034432, + "active_bytes_peak": 94541315072, + "allocated_bytes_current": 72110034432, + "allocated_bytes_peak": 94541315072, + "device": 1, + "inactive_split_bytes_current": 2171089408, + "inactive_split_bytes_peak": 4565201920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 97385447424, + "reserved_bytes_peak": 97385447424, + "segment_count_current": 1128, + "segment_count_peak": 1128, + "timestamp_utc": "2026-08-26T19:10:24.779544Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2171089408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5603483648, + "inactive_split_bytes_peak_delta": 1038281728, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 98622767104, + "reserved_bytes_current_delta": 1237319680, + "reserved_bytes_peak": 98622767104, + "reserved_bytes_peak_delta": 1237319680, + "segment_count_current": 1130, + "segment_count_current_delta": 2, + "segment_count_peak": 1130, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T191025066224Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.095092Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191025066224Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191025066224Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f0f46548cb59a162576117605ce9557356b8fb69 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191025066224Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da06339e459126fab87002d5f345466be2c12a6ece7309e4ffa77483015968f7 +size 1692784 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191025387958Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191025387958Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bcee0bae6edf71225817379868341d506c80d052 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191025387958Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72110034432, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94565731840, + "active_bytes_peak_delta": 20166144, + "after": { + "active_bytes_current": 72110034432, + "active_bytes_peak": 94565731840, + "allocated_bytes_current": 72110034432, + "allocated_bytes_peak": 94565731840, + "device": 1, + "inactive_split_bytes_current": 2171089408, + "inactive_split_bytes_peak": 5603483648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 99446947840, + "reserved_bytes_peak": 99446947840, + "segment_count_current": 1131, + "segment_count_peak": 1131, + "timestamp_utc": "2026-08-26T19:10:25.387555Z", + "world_size": 8 + }, + "allocated_bytes_current": 72110034432, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94565731840, + "allocated_bytes_peak_delta": 20166144, + "before": { + "active_bytes_current": 72110034432, + "active_bytes_peak": 94545565696, + "allocated_bytes_current": 72110034432, + "allocated_bytes_peak": 94545565696, + "device": 1, + "inactive_split_bytes_current": 2171089408, + "inactive_split_bytes_peak": 5603483648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 98622767104, + "reserved_bytes_peak": 98622767104, + "segment_count_current": 1130, + "segment_count_peak": 1130, + "timestamp_utc": "2026-08-26T19:10:25.098763Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2171089408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5603483648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 99446947840, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 99446947840, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 1131, + "segment_count_current_delta": 1, + "segment_count_peak": 1131, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T191025387958Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.417272Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191025387958Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191025387958Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c5ed9cbe93fc1c065375f0a2727101f71364e14d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191025387958Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be909c89b46fa4fba8350b22ed18d6cdbea793a6736f2f4844f1948f30576751 +size 1692432 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191025709638Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191025709638Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f39e7c27c5075f68fbc5a92349c4fcb169df341 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191025709638Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72110034432, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94565731840, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72110034432, + "active_bytes_peak": 94565731840, + "allocated_bytes_current": 72110034432, + "allocated_bytes_peak": 94565731840, + "device": 1, + "inactive_split_bytes_current": 2173186560, + "inactive_split_bytes_peak": 5603483648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 99446947840, + "reserved_bytes_peak": 99446947840, + "segment_count_current": 1131, + "segment_count_peak": 1131, + "timestamp_utc": "2026-08-26T19:10:25.709233Z", + "world_size": 8 + }, + "allocated_bytes_current": 72110034432, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94565731840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72110034432, + "active_bytes_peak": 94565731840, + "allocated_bytes_current": 72110034432, + "allocated_bytes_peak": 94565731840, + "device": 1, + "inactive_split_bytes_current": 2171089408, + "inactive_split_bytes_peak": 5603483648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 99446947840, + "reserved_bytes_peak": 99446947840, + "segment_count_current": 1131, + "segment_count_peak": 1131, + "timestamp_utc": "2026-08-26T19:10:25.420905Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2173186560, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 5603483648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 99446947840, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 99446947840, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1131, + "segment_count_current_delta": 0, + "segment_count_peak": 1131, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T191025709638Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.738801Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191025709638Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191025709638Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11980b50de6ede67108936da658e43c442bac8f4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191025709638Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d86abcd3f1811e5bd152efeea7695776e51bd44284b672312c22c07dfa02d64f +size 1697046 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191024263879Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191024263879Z.json new file mode 100644 index 0000000000000000000000000000000000000000..58e606ae7dbb62f34923f8afb1f42a7da5122ecc --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191024263879Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72113342464, + "active_bytes_current_delta": 31427374080, + "active_bytes_peak": 73642454528, + "active_bytes_peak_delta": 32918737408, + "after": { + "active_bytes_current": 72113342464, + "active_bytes_peak": 73642454528, + "allocated_bytes_current": 72113342464, + "allocated_bytes_peak": 73642454528, + "device": 2, + "inactive_split_bytes_current": 2037757952, + "inactive_split_bytes_peak": 4918854144, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 83959480320, + "reserved_bytes_peak": 83959480320, + "segment_count_current": 1035, + "segment_count_peak": 1035, + "timestamp_utc": "2026-08-26T19:10:24.263274Z", + "world_size": 8 + }, + "allocated_bytes_current": 72113342464, + "allocated_bytes_current_delta": 31427374080, + "allocated_bytes_peak": 73642454528, + "allocated_bytes_peak_delta": 32918737408, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 2, + "inactive_split_bytes_current": 283993088, + "inactive_split_bytes_peak": 504194048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 43130028032, + "reserved_bytes_peak": 43130028032, + "segment_count_current": 251, + "segment_count_peak": 251, + "timestamp_utc": "2026-08-26T19:09:42.057251Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2037757952, + "inactive_split_bytes_current_delta": 1753764864, + "inactive_split_bytes_peak": 4918854144, + "inactive_split_bytes_peak_delta": 4414660096, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 83959480320, + "reserved_bytes_current_delta": 40829452288, + "reserved_bytes_peak": 83959480320, + "reserved_bytes_peak_delta": 40829452288, + "segment_count_current": 1035, + "segment_count_current_delta": 784, + "segment_count_peak": 1035, + "segment_count_peak_delta": 784, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T191024263879Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.299571Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191024263879Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191024263879Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..38455a533eef6e91e03ce81c1ad16c4c2aa66c7c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191024263879Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0a64c45e3db645cda468f4fcec37124c1e2e04a2175d0477ab7e5711a51eb16 +size 2030943 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191024742469Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191024742469Z.json new file mode 100644 index 0000000000000000000000000000000000000000..48af61b98ed7afc23ae893c739f26392c778d708 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191024742469Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72113342464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94545827328, + "active_bytes_peak_delta": 20903372800, + "after": { + "active_bytes_current": 72113342464, + "active_bytes_peak": 94545827328, + "allocated_bytes_current": 72113342464, + "allocated_bytes_peak": 94545827328, + "device": 2, + "inactive_split_bytes_current": 2331359232, + "inactive_split_bytes_peak": 4918854144, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98016690176, + "reserved_bytes_peak": 98016690176, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:10:24.742014Z", + "world_size": 8 + }, + "allocated_bytes_current": 72113342464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94545827328, + "allocated_bytes_peak_delta": 20903372800, + "before": { + "active_bytes_current": 72113342464, + "active_bytes_peak": 73642454528, + "allocated_bytes_current": 72113342464, + "allocated_bytes_peak": 73642454528, + "device": 2, + "inactive_split_bytes_current": 2037757952, + "inactive_split_bytes_peak": 4918854144, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 83959480320, + "reserved_bytes_peak": 83959480320, + "segment_count_current": 1035, + "segment_count_peak": 1035, + "timestamp_utc": "2026-08-26T19:10:24.304082Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2331359232, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 4918854144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98016690176, + "reserved_bytes_current_delta": 14057209856, + "reserved_bytes_peak": 98016690176, + "reserved_bytes_peak_delta": 14057209856, + "segment_count_current": 1126, + "segment_count_current_delta": 91, + "segment_count_peak": 1126, + "segment_count_peak_delta": 91, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T191024742469Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.775246Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191024742469Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191024742469Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..882095699a73fa55faa20ee385592c8c5675b3a7 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191024742469Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9273bd42fb64fc340a808f088a8f015cda6859e253afc7f921d1cd3f19f0131 +size 1696505 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191025066223Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191025066223Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2de984f2b2e8905e56ac4dfaf19294a1f609bd98 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191025066223Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72113342464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94549302784, + "active_bytes_peak_delta": 3475456, + "after": { + "active_bytes_current": 72113342464, + "active_bytes_peak": 94549302784, + "allocated_bytes_current": 72113342464, + "allocated_bytes_peak": 94549302784, + "device": 2, + "inactive_split_bytes_current": 2331359232, + "inactive_split_bytes_peak": 5284359168, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98840870912, + "reserved_bytes_peak": 98840870912, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:10:25.065802Z", + "world_size": 8 + }, + "allocated_bytes_current": 72113342464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94549302784, + "allocated_bytes_peak_delta": 3475456, + "before": { + "active_bytes_current": 72113342464, + "active_bytes_peak": 94545827328, + "allocated_bytes_current": 72113342464, + "allocated_bytes_peak": 94545827328, + "device": 2, + "inactive_split_bytes_current": 2331359232, + "inactive_split_bytes_peak": 4918854144, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98016690176, + "reserved_bytes_peak": 98016690176, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:10:24.779248Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2331359232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5284359168, + "inactive_split_bytes_peak_delta": 365505024, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98840870912, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 98840870912, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 1127, + "segment_count_current_delta": 1, + "segment_count_peak": 1127, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T191025066223Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.095441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191025066223Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191025066223Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bca036f7f08df82704e5a325f840b7276c94c00b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191025066223Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:184aa7c41501e973d969a2b77c38731d3a567d4f136dfcb8a79ff704223bf5f9 +size 1690525 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191025387977Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191025387977Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fe8fc0d31c7e72292bf133014e0d549c146994fe --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191025387977Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72113342464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94558984192, + "active_bytes_peak_delta": 9681408, + "after": { + "active_bytes_current": 72113342464, + "active_bytes_peak": 94558984192, + "allocated_bytes_current": 72113342464, + "allocated_bytes_peak": 94558984192, + "device": 2, + "inactive_split_bytes_current": 2331359232, + "inactive_split_bytes_peak": 5284359168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98840870912, + "reserved_bytes_peak": 98840870912, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:10:25.387568Z", + "world_size": 8 + }, + "allocated_bytes_current": 72113342464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94558984192, + "allocated_bytes_peak_delta": 9681408, + "before": { + "active_bytes_current": 72113342464, + "active_bytes_peak": 94549302784, + "allocated_bytes_current": 72113342464, + "allocated_bytes_peak": 94549302784, + "device": 2, + "inactive_split_bytes_current": 2331359232, + "inactive_split_bytes_peak": 5284359168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98840870912, + "reserved_bytes_peak": 98840870912, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:10:25.099144Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2331359232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5284359168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98840870912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 98840870912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1127, + "segment_count_current_delta": 0, + "segment_count_peak": 1127, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T191025387977Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.416679Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191025387977Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191025387977Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f779641bbb7ec981452f92f0672ec8911370e32a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191025387977Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:078f2a6fb751f853f81b44c2e17e05b4a57b9ae86cdabbef634e74e17696001a +size 1690301 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191025709623Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191025709623Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2fc14fbe8ffbdd74ddda559be9e3bf2bb08f39d4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191025709623Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72113342464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94558984192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72113342464, + "active_bytes_peak": 94558984192, + "allocated_bytes_current": 72113342464, + "allocated_bytes_peak": 94558984192, + "device": 2, + "inactive_split_bytes_current": 2331359232, + "inactive_split_bytes_peak": 5284359168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98840870912, + "reserved_bytes_peak": 98840870912, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:10:25.709217Z", + "world_size": 8 + }, + "allocated_bytes_current": 72113342464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94558984192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72113342464, + "active_bytes_peak": 94558984192, + "allocated_bytes_current": 72113342464, + "allocated_bytes_peak": 94558984192, + "device": 2, + "inactive_split_bytes_current": 2331359232, + "inactive_split_bytes_peak": 5284359168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98840870912, + "reserved_bytes_peak": 98840870912, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:10:25.420345Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2331359232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5284359168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98840870912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 98840870912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1127, + "segment_count_current_delta": 0, + "segment_count_peak": 1127, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T191025709623Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.737672Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191025709623Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191025709623Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..91887f3a3bd33c4ee92fb7f428bab261cb1e425a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191025709623Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1043155e6cfee1d4923bcaa974dc0547076ed8551c1bde8590d5cc407c3b0a4 +size 1690190 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191024263871Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191024263871Z.json new file mode 100644 index 0000000000000000000000000000000000000000..acb83987f84e44f6c05843e8ce66e80230b5ab14 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191024263871Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72124667904, + "active_bytes_current_delta": 31438699520, + "active_bytes_peak": 73627423744, + "active_bytes_peak_delta": 32903706624, + "after": { + "active_bytes_current": 72124667904, + "active_bytes_peak": 73627423744, + "allocated_bytes_current": 72124667904, + "allocated_bytes_peak": 73627423744, + "device": 3, + "inactive_split_bytes_current": 2171136000, + "inactive_split_bytes_peak": 4718119424, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 84754300928, + "reserved_bytes_peak": 84754300928, + "segment_count_current": 1021, + "segment_count_peak": 1021, + "timestamp_utc": "2026-08-26T19:10:24.263246Z", + "world_size": 8 + }, + "allocated_bytes_current": 72124667904, + "allocated_bytes_current_delta": 31438699520, + "allocated_bytes_peak": 73627423744, + "allocated_bytes_peak_delta": 32903706624, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 3, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:09:42.057301Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2171136000, + "inactive_split_bytes_current_delta": 1662747648, + "inactive_split_bytes_peak": 4718119424, + "inactive_split_bytes_peak_delta": 3989530112, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 84754300928, + "reserved_bytes_current_delta": 40890269696, + "reserved_bytes_peak": 84754300928, + "reserved_bytes_peak_delta": 40890269696, + "segment_count_current": 1021, + "segment_count_current_delta": 760, + "segment_count_peak": 1021, + "segment_count_peak_delta": 760, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T191024263871Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.299423Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191024263871Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191024263871Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1592a45bb091b14ea4db8a95ac1cff06de21d66a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191024263871Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bac5787f069807387207494fd087922507cef88da867cf7a51a745377de10311 +size 1779313 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191024742436Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191024742436Z.json new file mode 100644 index 0000000000000000000000000000000000000000..39d9bedfe7a2caf95ddd3841d94386951cfadf59 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191024742436Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72124667904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94559214592, + "active_bytes_peak_delta": 20931790848, + "after": { + "active_bytes_current": 72124667904, + "active_bytes_peak": 94559214592, + "allocated_bytes_current": 72124667904, + "allocated_bytes_peak": 94559214592, + "device": 3, + "inactive_split_bytes_current": 2464737280, + "inactive_split_bytes_peak": 4718119424, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 98891202560, + "reserved_bytes_peak": 98891202560, + "segment_count_current": 1110, + "segment_count_peak": 1110, + "timestamp_utc": "2026-08-26T19:10:24.742000Z", + "world_size": 8 + }, + "allocated_bytes_current": 72124667904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94559214592, + "allocated_bytes_peak_delta": 20931790848, + "before": { + "active_bytes_current": 72124667904, + "active_bytes_peak": 73627423744, + "allocated_bytes_current": 72124667904, + "allocated_bytes_peak": 73627423744, + "device": 3, + "inactive_split_bytes_current": 2171136000, + "inactive_split_bytes_peak": 4718119424, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 84754300928, + "reserved_bytes_peak": 84754300928, + "segment_count_current": 1021, + "segment_count_peak": 1021, + "timestamp_utc": "2026-08-26T19:10:24.303935Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2464737280, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 4718119424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 98891202560, + "reserved_bytes_current_delta": 14136901632, + "reserved_bytes_peak": 98891202560, + "reserved_bytes_peak_delta": 14136901632, + "segment_count_current": 1110, + "segment_count_current_delta": 89, + "segment_count_peak": 1110, + "segment_count_peak_delta": 89, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T191024742436Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.775190Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191024742436Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191024742436Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2bff0084ba8c3c246f747c6936ca514776b957e0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191024742436Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5780158aac137c144c5e2134d3573ee2bb843bfbf8d4d72f49990692025657d6 +size 1690483 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191025066191Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191025066191Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9dfc4e27e2d0f290b2f33c0e071013ad0dbfc95b --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191025066191Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72124667904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94559214592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72124667904, + "active_bytes_peak": 94559214592, + "allocated_bytes_current": 72124667904, + "allocated_bytes_peak": 94559214592, + "device": 3, + "inactive_split_bytes_current": 2464737280, + "inactive_split_bytes_peak": 4718119424, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 98891202560, + "reserved_bytes_peak": 98891202560, + "segment_count_current": 1110, + "segment_count_peak": 1110, + "timestamp_utc": "2026-08-26T19:10:25.065794Z", + "world_size": 8 + }, + "allocated_bytes_current": 72124667904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94559214592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72124667904, + "active_bytes_peak": 94559214592, + "allocated_bytes_current": 72124667904, + "allocated_bytes_peak": 94559214592, + "device": 3, + "inactive_split_bytes_current": 2464737280, + "inactive_split_bytes_peak": 4718119424, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 98891202560, + "reserved_bytes_peak": 98891202560, + "segment_count_current": 1110, + "segment_count_peak": 1110, + "timestamp_utc": "2026-08-26T19:10:24.779097Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2464737280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4718119424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 98891202560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 98891202560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1110, + "segment_count_current_delta": 0, + "segment_count_peak": 1110, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T191025066191Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.094851Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191025066191Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191025066191Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3105d0eee2fdd6103310cbb461c0fc5c2cbed8e4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191025066191Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ff747427eec75764f7d3eba7b81b0b2da3999c359cc8b3dc1b88c7f029c48b2 +size 1684805 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191025387926Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191025387926Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a1c031d53345dafb22160803068cef244cf152e3 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191025387926Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72124667904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94559214592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72124667904, + "active_bytes_peak": 94559214592, + "allocated_bytes_current": 72124667904, + "allocated_bytes_peak": 94559214592, + "device": 3, + "inactive_split_bytes_current": 2464737280, + "inactive_split_bytes_peak": 4718119424, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 99715383296, + "reserved_bytes_peak": 99715383296, + "segment_count_current": 1111, + "segment_count_peak": 1111, + "timestamp_utc": "2026-08-26T19:10:25.387537Z", + "world_size": 8 + }, + "allocated_bytes_current": 72124667904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94559214592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72124667904, + "active_bytes_peak": 94559214592, + "allocated_bytes_current": 72124667904, + "allocated_bytes_peak": 94559214592, + "device": 3, + "inactive_split_bytes_current": 2464737280, + "inactive_split_bytes_peak": 4718119424, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 98891202560, + "reserved_bytes_peak": 98891202560, + "segment_count_current": 1110, + "segment_count_peak": 1110, + "timestamp_utc": "2026-08-26T19:10:25.098458Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2464737280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4718119424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 99715383296, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 99715383296, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 1111, + "segment_count_current_delta": 1, + "segment_count_peak": 1111, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T191025387926Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.416248Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191025387926Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191025387926Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..26a2c898aaf9045711b6657e5e889e4a4c6be779 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191025387926Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1f88a03acec1bec8d343cc2589c4f452c9b44daa256378d4525acb854f00214 +size 1684794 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191025709610Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191025709610Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bb2b5418c1c0e6008799826e7a28148b36db5dac --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191025709610Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72124667904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94559214592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72124667904, + "active_bytes_peak": 94559214592, + "allocated_bytes_current": 72124667904, + "allocated_bytes_peak": 94559214592, + "device": 3, + "inactive_split_bytes_current": 2464737280, + "inactive_split_bytes_peak": 4718119424, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 99715383296, + "reserved_bytes_peak": 99715383296, + "segment_count_current": 1111, + "segment_count_peak": 1111, + "timestamp_utc": "2026-08-26T19:10:25.709210Z", + "world_size": 8 + }, + "allocated_bytes_current": 72124667904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94559214592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72124667904, + "active_bytes_peak": 94559214592, + "allocated_bytes_current": 72124667904, + "allocated_bytes_peak": 94559214592, + "device": 3, + "inactive_split_bytes_current": 2464737280, + "inactive_split_bytes_peak": 4718119424, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 99715383296, + "reserved_bytes_peak": 99715383296, + "segment_count_current": 1111, + "segment_count_peak": 1111, + "timestamp_utc": "2026-08-26T19:10:25.419826Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2464737280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4718119424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 99715383296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 99715383296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1111, + "segment_count_current_delta": 0, + "segment_count_peak": 1111, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T191025709610Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.737727Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191025709610Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191025709610Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94d8a0b543f41e9fde8c86f92184e106ee91593a --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191025709610Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7c2842a226833bc510f8c2b30bfb1c9bf4ce9244c73b7572da1b8f227eda57d +size 1684425 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191024263867Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191024263867Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9557c9e7fbeb9c2a138a8e72232cfcebbbf6c937 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191024263867Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72125944832, + "active_bytes_current_delta": 31439976448, + "active_bytes_peak": 73618637824, + "active_bytes_peak_delta": 32894920704, + "after": { + "active_bytes_current": 72125944832, + "active_bytes_peak": 73618637824, + "allocated_bytes_current": 72125944832, + "allocated_bytes_peak": 73618637824, + "device": 4, + "inactive_split_bytes_current": 2239065088, + "inactive_split_bytes_peak": 4422115840, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 83359694848, + "reserved_bytes_peak": 83359694848, + "segment_count_current": 1019, + "segment_count_peak": 1019, + "timestamp_utc": "2026-08-26T19:10:24.263240Z", + "world_size": 8 + }, + "allocated_bytes_current": 72125944832, + "allocated_bytes_current_delta": 31439976448, + "allocated_bytes_peak": 73618637824, + "allocated_bytes_peak_delta": 32894920704, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 4, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:09:42.057240Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2239065088, + "inactive_split_bytes_current_delta": 1730676736, + "inactive_split_bytes_peak": 4422115840, + "inactive_split_bytes_peak_delta": 3693526528, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 83359694848, + "reserved_bytes_current_delta": 39495663616, + "reserved_bytes_peak": 83359694848, + "reserved_bytes_peak_delta": 39495663616, + "segment_count_current": 1019, + "segment_count_current_delta": 758, + "segment_count_peak": 1019, + "segment_count_peak_delta": 758, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T191024263867Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.296635Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191024263867Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191024263867Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f843e4ca18df2264907f4a831dc3f6f6fd4b76ff --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191024263867Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ad310fe150c67a3de0e405e2b69f3bb65ad378309968e6f264c014b88649e3a +size 1778907 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191024742367Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191024742367Z.json new file mode 100644 index 0000000000000000000000000000000000000000..49e23f456b164f0443885e1d15914f5bfa60f360 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191024742367Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72125944832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94554253312, + "active_bytes_peak_delta": 20935615488, + "after": { + "active_bytes_current": 72125944832, + "active_bytes_peak": 94554253312, + "allocated_bytes_current": 72125944832, + "allocated_bytes_peak": 94554253312, + "device": 4, + "inactive_split_bytes_current": 2532666368, + "inactive_split_bytes_peak": 4422115840, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 97324630016, + "reserved_bytes_peak": 97324630016, + "segment_count_current": 1104, + "segment_count_peak": 1104, + "timestamp_utc": "2026-08-26T19:10:24.741953Z", + "world_size": 8 + }, + "allocated_bytes_current": 72125944832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94554253312, + "allocated_bytes_peak_delta": 20935615488, + "before": { + "active_bytes_current": 72125944832, + "active_bytes_peak": 73618637824, + "allocated_bytes_current": 72125944832, + "allocated_bytes_peak": 73618637824, + "device": 4, + "inactive_split_bytes_current": 2239065088, + "inactive_split_bytes_peak": 4422115840, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 83359694848, + "reserved_bytes_peak": 83359694848, + "segment_count_current": 1019, + "segment_count_peak": 1019, + "timestamp_utc": "2026-08-26T19:10:24.300782Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2532666368, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 4422115840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 97324630016, + "reserved_bytes_current_delta": 13964935168, + "reserved_bytes_peak": 97324630016, + "reserved_bytes_peak_delta": 13964935168, + "segment_count_current": 1104, + "segment_count_current_delta": 85, + "segment_count_peak": 1104, + "segment_count_peak_delta": 85, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T191024742367Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.774409Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191024742367Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191024742367Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3b1baa2ca334341a500ec3f06dc8b5a4191aea4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191024742367Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64ec546fbafe585f73ec5fc4e7213d47e8df36a67369fd3ea0b1cf33bb80c76d +size 1689629 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191025066132Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191025066132Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f9637b95899df83c5a65c004c41416712862e25c --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191025066132Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72125944832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94561236480, + "active_bytes_peak_delta": 6983168, + "after": { + "active_bytes_current": 72125944832, + "active_bytes_peak": 94561236480, + "allocated_bytes_current": 72125944832, + "allocated_bytes_peak": 94561236480, + "device": 4, + "inactive_split_bytes_current": 2532666368, + "inactive_split_bytes_peak": 5721604096, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 99249815552, + "reserved_bytes_peak": 99249815552, + "segment_count_current": 1108, + "segment_count_peak": 1108, + "timestamp_utc": "2026-08-26T19:10:25.065745Z", + "world_size": 8 + }, + "allocated_bytes_current": 72125944832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94561236480, + "allocated_bytes_peak_delta": 6983168, + "before": { + "active_bytes_current": 72125944832, + "active_bytes_peak": 94554253312, + "allocated_bytes_current": 72125944832, + "allocated_bytes_peak": 94554253312, + "device": 4, + "inactive_split_bytes_current": 2532666368, + "inactive_split_bytes_peak": 4422115840, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 97324630016, + "reserved_bytes_peak": 97324630016, + "segment_count_current": 1104, + "segment_count_peak": 1104, + "timestamp_utc": "2026-08-26T19:10:24.777920Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2532666368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5721604096, + "inactive_split_bytes_peak_delta": 1299488256, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 99249815552, + "reserved_bytes_current_delta": 1925185536, + "reserved_bytes_peak": 99249815552, + "reserved_bytes_peak_delta": 1925185536, + "segment_count_current": 1108, + "segment_count_current_delta": 4, + "segment_count_peak": 1108, + "segment_count_peak_delta": 4, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T191025066132Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.096434Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191025066132Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191025066132Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..38e653f41f3d3edfb067ff7a61b50f24919bebe4 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191025066132Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c88cffce3ce1ef0319dbfb7b964cb765c21e875d92650e7987d0919721252b0 +size 1686416 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191025387911Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191025387911Z.json new file mode 100644 index 0000000000000000000000000000000000000000..969eed10d4a95ad4d7b086a660c7ed0a358bea31 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191025387911Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72125944832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94561236480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72125944832, + "active_bytes_peak": 94561236480, + "allocated_bytes_current": 72125944832, + "allocated_bytes_peak": 94561236480, + "device": 4, + "inactive_split_bytes_current": 2532666368, + "inactive_split_bytes_peak": 5721604096, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 99249815552, + "reserved_bytes_peak": 99249815552, + "segment_count_current": 1108, + "segment_count_peak": 1108, + "timestamp_utc": "2026-08-26T19:10:25.387516Z", + "world_size": 8 + }, + "allocated_bytes_current": 72125944832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94561236480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72125944832, + "active_bytes_peak": 94561236480, + "allocated_bytes_current": 72125944832, + "allocated_bytes_peak": 94561236480, + "device": 4, + "inactive_split_bytes_current": 2532666368, + "inactive_split_bytes_peak": 5721604096, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 99249815552, + "reserved_bytes_peak": 99249815552, + "segment_count_current": 1108, + "segment_count_peak": 1108, + "timestamp_utc": "2026-08-26T19:10:25.099768Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2532666368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5721604096, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 99249815552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 99249815552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1108, + "segment_count_current_delta": 0, + "segment_count_peak": 1108, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T191025387911Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.415530Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191025387911Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191025387911Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6107c1e6760dc2885b4696056f3f74a3420f8212 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191025387911Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41f69d9e1e3e895f35c074700051337466c4ccc8aea3a3e4bce6801212a8aedd +size 1687067 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191025709579Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191025709579Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a879d4245266f8aaa595a483308b95599efa23e0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191025709579Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72125944832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94569341952, + "active_bytes_peak_delta": 8105472, + "after": { + "active_bytes_current": 72125944832, + "active_bytes_peak": 94569341952, + "allocated_bytes_current": 72125944832, + "allocated_bytes_peak": 94569341952, + "device": 4, + "inactive_split_bytes_current": 2532666368, + "inactive_split_bytes_peak": 5721604096, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 99249815552, + "reserved_bytes_peak": 99249815552, + "segment_count_current": 1108, + "segment_count_peak": 1108, + "timestamp_utc": "2026-08-26T19:10:25.709183Z", + "world_size": 8 + }, + "allocated_bytes_current": 72125944832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94569341952, + "allocated_bytes_peak_delta": 8105472, + "before": { + "active_bytes_current": 72125944832, + "active_bytes_peak": 94561236480, + "allocated_bytes_current": 72125944832, + "allocated_bytes_peak": 94561236480, + "device": 4, + "inactive_split_bytes_current": 2532666368, + "inactive_split_bytes_peak": 5721604096, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 99249815552, + "reserved_bytes_peak": 99249815552, + "segment_count_current": 1108, + "segment_count_peak": 1108, + "timestamp_utc": "2026-08-26T19:10:25.418806Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2532666368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5721604096, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 99249815552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 99249815552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1108, + "segment_count_current_delta": 0, + "segment_count_peak": 1108, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T191025709579Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.736823Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191025709579Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191025709579Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f27862d719bbfc72eadfa9b4dc885d487f19607 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191025709579Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a85bd9ed068438eb79a14eb0dbf46391b2e65ecc0acc1abc17f8423a618ef70 +size 1687050 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191024264070Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191024264070Z.json new file mode 100644 index 0000000000000000000000000000000000000000..270b773923a2c9faca2a9fba289ead95aea84302 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191024264070Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72122308608, + "active_bytes_current_delta": 31436340224, + "active_bytes_peak": 73636120576, + "active_bytes_peak_delta": 32912403456, + "after": { + "active_bytes_current": 72122308608, + "active_bytes_peak": 73636120576, + "allocated_bytes_current": 72122308608, + "allocated_bytes_peak": 73636120576, + "device": 5, + "inactive_split_bytes_current": 2316101632, + "inactive_split_bytes_peak": 4889299456, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 84217430016, + "reserved_bytes_peak": 84217430016, + "segment_count_current": 1015, + "segment_count_peak": 1015, + "timestamp_utc": "2026-08-26T19:10:24.263460Z", + "world_size": 8 + }, + "allocated_bytes_current": 72122308608, + "allocated_bytes_current_delta": 31436340224, + "allocated_bytes_peak": 73636120576, + "allocated_bytes_peak_delta": 32912403456, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 5, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:09:42.057230Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2316101632, + "inactive_split_bytes_current_delta": 1807713280, + "inactive_split_bytes_peak": 4889299456, + "inactive_split_bytes_peak_delta": 4160710144, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 84217430016, + "reserved_bytes_current_delta": 40353398784, + "reserved_bytes_peak": 84217430016, + "reserved_bytes_peak_delta": 40353398784, + "segment_count_current": 1015, + "segment_count_current_delta": 754, + "segment_count_peak": 1015, + "segment_count_peak_delta": 754, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T191024264070Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.301380Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191024264070Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191024264070Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a1fa26eaf5b983606a2f090330e32a631c2dd3f8 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191024264070Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97209b12190f34c5d2ad8dbc6cf95b6391ecbd19c894554712b5631b6f2d72f8 +size 1778438 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191024742486Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191024742486Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e572d75c67153c9397f858181517f589c9c1952 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191024742486Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72122308608, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94542465536, + "active_bytes_peak_delta": 20906344960, + "after": { + "active_bytes_current": 72122308608, + "active_bytes_peak": 94542465536, + "allocated_bytes_current": 72122308608, + "allocated_bytes_peak": 94542465536, + "device": 5, + "inactive_split_bytes_current": 2609702912, + "inactive_split_bytes_peak": 4889299456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 98197045248, + "reserved_bytes_peak": 98197045248, + "segment_count_current": 1100, + "segment_count_peak": 1100, + "timestamp_utc": "2026-08-26T19:10:24.742027Z", + "world_size": 8 + }, + "allocated_bytes_current": 72122308608, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94542465536, + "allocated_bytes_peak_delta": 20906344960, + "before": { + "active_bytes_current": 72122308608, + "active_bytes_peak": 73636120576, + "allocated_bytes_current": 72122308608, + "allocated_bytes_peak": 73636120576, + "device": 5, + "inactive_split_bytes_current": 2316101632, + "inactive_split_bytes_peak": 4889299456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 84217430016, + "reserved_bytes_peak": 84217430016, + "segment_count_current": 1015, + "segment_count_peak": 1015, + "timestamp_utc": "2026-08-26T19:10:24.305848Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2609702912, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 4889299456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 98197045248, + "reserved_bytes_current_delta": 13979615232, + "reserved_bytes_peak": 98197045248, + "reserved_bytes_peak_delta": 13979615232, + "segment_count_current": 1100, + "segment_count_current_delta": 85, + "segment_count_peak": 1100, + "segment_count_peak_delta": 85, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T191024742486Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.774994Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191024742486Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191024742486Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f0fda04386959ecdb6872f24b9b79e603d8de79 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191024742486Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6b4d02b3ea969eda80d2d5dcf0083a9ccd15048c3193b9d3005873999c32d35 +size 1688938 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191025066178Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191025066178Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15985b6a5ba79fcd6da043ecde0819666811b4ae --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191025066178Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72122308608, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94542465536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72122308608, + "active_bytes_peak": 94542465536, + "allocated_bytes_current": 72122308608, + "allocated_bytes_peak": 94542465536, + "device": 5, + "inactive_split_bytes_current": 2609702912, + "inactive_split_bytes_peak": 5177871360, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 99434364928, + "reserved_bytes_peak": 99434364928, + "segment_count_current": 1102, + "segment_count_peak": 1102, + "timestamp_utc": "2026-08-26T19:10:25.065785Z", + "world_size": 8 + }, + "allocated_bytes_current": 72122308608, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94542465536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72122308608, + "active_bytes_peak": 94542465536, + "allocated_bytes_current": 72122308608, + "allocated_bytes_peak": 94542465536, + "device": 5, + "inactive_split_bytes_current": 2609702912, + "inactive_split_bytes_peak": 4889299456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 98197045248, + "reserved_bytes_peak": 98197045248, + "segment_count_current": 1100, + "segment_count_peak": 1100, + "timestamp_utc": "2026-08-26T19:10:24.778668Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2609702912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5177871360, + "inactive_split_bytes_peak_delta": 288571904, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 99434364928, + "reserved_bytes_current_delta": 1237319680, + "reserved_bytes_peak": 99434364928, + "reserved_bytes_peak_delta": 1237319680, + "segment_count_current": 1102, + "segment_count_current_delta": 2, + "segment_count_peak": 1102, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T191025066178Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.093936Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191025066178Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191025066178Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..576a966b35c5b6e82e00792b40d3dcb15d26831f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191025066178Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0882af7ae9509d630b8361fc0ade65e3d2ff64dee486832e0ce83a9d6da90906 +size 1683245 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191025387948Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191025387948Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2e25401872db283b26601a6c8f9155bdec654c2d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191025387948Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72122308608, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94561219584, + "active_bytes_peak_delta": 18754048, + "after": { + "active_bytes_current": 72122308608, + "active_bytes_peak": 94561219584, + "allocated_bytes_current": 72122308608, + "allocated_bytes_peak": 94561219584, + "device": 5, + "inactive_split_bytes_current": 2609702912, + "inactive_split_bytes_peak": 5349460992, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 100258545664, + "reserved_bytes_peak": 100258545664, + "segment_count_current": 1103, + "segment_count_peak": 1103, + "timestamp_utc": "2026-08-26T19:10:25.387562Z", + "world_size": 8 + }, + "allocated_bytes_current": 72122308608, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94561219584, + "allocated_bytes_peak_delta": 18754048, + "before": { + "active_bytes_current": 72122308608, + "active_bytes_peak": 94542465536, + "allocated_bytes_current": 72122308608, + "allocated_bytes_peak": 94542465536, + "device": 5, + "inactive_split_bytes_current": 2609702912, + "inactive_split_bytes_peak": 5177871360, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 99434364928, + "reserved_bytes_peak": 99434364928, + "segment_count_current": 1102, + "segment_count_peak": 1102, + "timestamp_utc": "2026-08-26T19:10:25.097284Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2609702912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5349460992, + "inactive_split_bytes_peak_delta": 171589632, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 100258545664, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 100258545664, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 1103, + "segment_count_current_delta": 1, + "segment_count_peak": 1103, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T191025387948Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.415031Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191025387948Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191025387948Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3302605b200cf6178698fd9e78f5e3b7cb9424e5 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191025387948Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1135d9fdd0d926b2c1e4175c615ce069d6badb2b42122020db102d7f7f9b2847 +size 1683404 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191025709617Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191025709617Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eab8071014f0f34d4aff183740f807efae10df03 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191025709617Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72122308608, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94561219584, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72122308608, + "active_bytes_peak": 94561219584, + "allocated_bytes_current": 72122308608, + "allocated_bytes_peak": 94561219584, + "device": 5, + "inactive_split_bytes_current": 2609702912, + "inactive_split_bytes_peak": 5349460992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 100258545664, + "reserved_bytes_peak": 100258545664, + "segment_count_current": 1103, + "segment_count_peak": 1103, + "timestamp_utc": "2026-08-26T19:10:25.709220Z", + "world_size": 8 + }, + "allocated_bytes_current": 72122308608, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94561219584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72122308608, + "active_bytes_peak": 94561219584, + "allocated_bytes_current": 72122308608, + "allocated_bytes_peak": 94561219584, + "device": 5, + "inactive_split_bytes_current": 2609702912, + "inactive_split_bytes_peak": 5349460992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 100258545664, + "reserved_bytes_peak": 100258545664, + "segment_count_current": 1103, + "segment_count_peak": 1103, + "timestamp_utc": "2026-08-26T19:10:25.418293Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2609702912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5349460992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 100258545664, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 100258545664, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1103, + "segment_count_current_delta": 0, + "segment_count_peak": 1103, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T191025709617Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.736340Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191025709617Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191025709617Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4cbfa6c30e255989207f2f309bd7dd0e637e0dba --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191025709617Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0cd10a8657a07b4fc679fd2476a0c98a95adb77be00d19e11bdf8e5e7dc1c31 +size 1683041 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191024263847Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191024263847Z.json new file mode 100644 index 0000000000000000000000000000000000000000..17250905e5bd0c5e8d06f88c6d38e713198102cb --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191024263847Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72124148736, + "active_bytes_current_delta": 31438180352, + "active_bytes_peak": 73618441216, + "active_bytes_peak_delta": 32894724096, + "after": { + "active_bytes_current": 72124148736, + "active_bytes_peak": 73618441216, + "allocated_bytes_current": 72124148736, + "allocated_bytes_peak": 73618441216, + "device": 6, + "inactive_split_bytes_current": 2249249792, + "inactive_split_bytes_peak": 4612917760, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 83477135360, + "reserved_bytes_peak": 83477135360, + "segment_count_current": 1027, + "segment_count_peak": 1027, + "timestamp_utc": "2026-08-26T19:10:24.263209Z", + "world_size": 8 + }, + "allocated_bytes_current": 72124148736, + "allocated_bytes_current_delta": 31438180352, + "allocated_bytes_peak": 73618441216, + "allocated_bytes_peak_delta": 32894724096, + "before": { + "active_bytes_current": 40685968384, + "active_bytes_peak": 40723717120, + "allocated_bytes_current": 40685968384, + "allocated_bytes_peak": 40723717120, + "device": 6, + "inactive_split_bytes_current": 508388352, + "inactive_split_bytes_peak": 728589312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:09:42.057231Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2249249792, + "inactive_split_bytes_current_delta": 1740861440, + "inactive_split_bytes_peak": 4612917760, + "inactive_split_bytes_peak_delta": 3884328448, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 83477135360, + "reserved_bytes_current_delta": 39613104128, + "reserved_bytes_peak": 83477135360, + "reserved_bytes_peak_delta": 39613104128, + "segment_count_current": 1027, + "segment_count_current_delta": 766, + "segment_count_peak": 1027, + "segment_count_peak_delta": 766, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T191024263847Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.301379Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191024263847Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191024263847Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94a81a10dce03c5581b9cdfad30b96584bd2ae47 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191024263847Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3fc1289ba7e15f870140665b43bf233a8a7e28db36fc84c45f5e9b8f9d15d33 +size 1780203 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191024742424Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191024742424Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9b4472c248a914ccb49165f72dca7482b861ed73 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191024742424Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72124148736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94544064000, + "active_bytes_peak_delta": 20925622784, + "after": { + "active_bytes_current": 72124148736, + "active_bytes_peak": 94544064000, + "allocated_bytes_current": 72124148736, + "allocated_bytes_peak": 94544064000, + "device": 6, + "inactive_split_bytes_current": 2542851072, + "inactive_split_bytes_peak": 4612917760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 97385447424, + "reserved_bytes_peak": 97385447424, + "segment_count_current": 1108, + "segment_count_peak": 1108, + "timestamp_utc": "2026-08-26T19:10:24.741993Z", + "world_size": 8 + }, + "allocated_bytes_current": 72124148736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94544064000, + "allocated_bytes_peak_delta": 20925622784, + "before": { + "active_bytes_current": 72124148736, + "active_bytes_peak": 73618441216, + "allocated_bytes_current": 72124148736, + "allocated_bytes_peak": 73618441216, + "device": 6, + "inactive_split_bytes_current": 2249249792, + "inactive_split_bytes_peak": 4612917760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 83477135360, + "reserved_bytes_peak": 83477135360, + "segment_count_current": 1027, + "segment_count_peak": 1027, + "timestamp_utc": "2026-08-26T19:10:24.305789Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2542851072, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 4612917760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 97385447424, + "reserved_bytes_current_delta": 13908312064, + "reserved_bytes_peak": 97385447424, + "reserved_bytes_peak_delta": 13908312064, + "segment_count_current": 1108, + "segment_count_current_delta": 81, + "segment_count_peak": 1108, + "segment_count_peak_delta": 81, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T191024742424Z.pickle", + "timestamp_utc": "2026-08-26T19:10:24.774056Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191024742424Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191024742424Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..22450d0e584d037ba9ab3f028dfbd43e2604da14 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191024742424Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b3929a5bc4f2eb745a436ec1ec69e7fe2238918fd04d72ab235fe938c6a42fc +size 1690503 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191025066144Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191025066144Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eae5693043593014f28368d2fa5522adce2c360f --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191025066144Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72124148736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94544064000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72124148736, + "active_bytes_peak": 94544064000, + "allocated_bytes_current": 72124148736, + "allocated_bytes_peak": 94544064000, + "device": 6, + "inactive_split_bytes_current": 2542851072, + "inactive_split_bytes_peak": 4612917760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 97385447424, + "reserved_bytes_peak": 97385447424, + "segment_count_current": 1108, + "segment_count_peak": 1108, + "timestamp_utc": "2026-08-26T19:10:25.065748Z", + "world_size": 8 + }, + "allocated_bytes_current": 72124148736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94544064000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72124148736, + "active_bytes_peak": 94544064000, + "allocated_bytes_current": 72124148736, + "allocated_bytes_peak": 94544064000, + "device": 6, + "inactive_split_bytes_current": 2542851072, + "inactive_split_bytes_peak": 4612917760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 97385447424, + "reserved_bytes_peak": 97385447424, + "segment_count_current": 1108, + "segment_count_peak": 1108, + "timestamp_utc": "2026-08-26T19:10:24.777592Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2542851072, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4612917760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 97385447424, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 97385447424, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1108, + "segment_count_current_delta": 0, + "segment_count_peak": 1108, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T191025066144Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.094478Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191025066144Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191025066144Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..935fa121e38fb6272ab989db49945a9bb5d16aba --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191025066144Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d6d360c7a0943e1dad9a955002ee3866afcfec5c56597de867c44405f9fbcf1 +size 1685272 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191025387921Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191025387921Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a997df3c18c3239188d8882c69ec32c789de764d --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191025387921Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72124148736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94565178368, + "active_bytes_peak_delta": 21114368, + "after": { + "active_bytes_current": 72124148736, + "active_bytes_peak": 94565178368, + "allocated_bytes_current": 72124148736, + "allocated_bytes_peak": 94565178368, + "device": 6, + "inactive_split_bytes_current": 2542851072, + "inactive_split_bytes_peak": 5492559872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98622767104, + "reserved_bytes_peak": 98622767104, + "segment_count_current": 1110, + "segment_count_peak": 1110, + "timestamp_utc": "2026-08-26T19:10:25.387529Z", + "world_size": 8 + }, + "allocated_bytes_current": 72124148736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94565178368, + "allocated_bytes_peak_delta": 21114368, + "before": { + "active_bytes_current": 72124148736, + "active_bytes_peak": 94544064000, + "allocated_bytes_current": 72124148736, + "allocated_bytes_peak": 94544064000, + "device": 6, + "inactive_split_bytes_current": 2542851072, + "inactive_split_bytes_peak": 4612917760, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 97385447424, + "reserved_bytes_peak": 97385447424, + "segment_count_current": 1108, + "segment_count_peak": 1108, + "timestamp_utc": "2026-08-26T19:10:25.097801Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2542851072, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5492559872, + "inactive_split_bytes_peak_delta": 879642112, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98622767104, + "reserved_bytes_current_delta": 1237319680, + "reserved_bytes_peak": 98622767104, + "reserved_bytes_peak_delta": 1237319680, + "segment_count_current": 1110, + "segment_count_current_delta": 2, + "segment_count_peak": 1110, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T191025387921Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.416016Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191025387921Z.pickle b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191025387921Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2696153b394ce81468b8aedc74db8190432cbad0 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191025387921Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ceefaa7654b9aba6f8c88981ccd4125b50619136f944ccff1834acc088bd7c6 +size 1689837 diff --git a/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191025709574Z.json b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191025709574Z.json new file mode 100644 index 0000000000000000000000000000000000000000..437c5968dddfe1226ae21c0c1749c3ade9b60840 --- /dev/null +++ b/echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191025709574Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72124148736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 94569142272, + "active_bytes_peak_delta": 3963904, + "after": { + "active_bytes_current": 72124148736, + "active_bytes_peak": 94569142272, + "allocated_bytes_current": 72124148736, + "allocated_bytes_peak": 94569142272, + "device": 6, + "inactive_split_bytes_current": 2542851072, + "inactive_split_bytes_peak": 5492559872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98622767104, + "reserved_bytes_peak": 98622767104, + "segment_count_current": 1110, + "segment_count_peak": 1110, + "timestamp_utc": "2026-08-26T19:10:25.709182Z", + "world_size": 8 + }, + "allocated_bytes_current": 72124148736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 94569142272, + "allocated_bytes_peak_delta": 3963904, + "before": { + "active_bytes_current": 72124148736, + "active_bytes_peak": 94565178368, + "allocated_bytes_current": 72124148736, + "allocated_bytes_peak": 94565178368, + "device": 6, + "inactive_split_bytes_current": 2542851072, + "inactive_split_bytes_peak": 5492559872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98622767104, + "reserved_bytes_peak": 98622767104, + "segment_count_current": 1110, + "segment_count_peak": 1110, + "timestamp_utc": "2026-08-26T19:10:25.419329Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2542851072, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5492559872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98622767104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 98622767104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1110, + "segment_count_current_delta": 0, + "segment_count_peak": 1110, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T191025709574Z.pickle", + "timestamp_utc": "2026-08-26T19:10:25.737104Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file