diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191359673365Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191359673365Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2727cc5454615f896330fce122668651b5d9cb1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191359673365Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9147284e646ca631a49016f8385f79c4d653380bd70f634e89082bb2bcf0789 +size 421723 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f036106a7dc961f0485f201e5da2150ce37b11f8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 3, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.985370Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 3, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.683225Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1688, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.994353Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bb4d048f5c645cff7e6b08d28f981b8b627983f3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:949994a4fd475feffaa977c358664b1931f7a967f3d2c3aa5f900c0fa78c79ab +size 421723 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.json new file mode 100644 index 0000000000000000000000000000000000000000..81ea1518388a43164718952b44ca131448961dfd --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77310399488, + "active_bytes_current_delta": 32789599232, + "active_bytes_peak": 84832202240, + "active_bytes_peak_delta": 40286236160, + "after": { + "active_bytes_current": 77310399488, + "active_bytes_peak": 84832202240, + "allocated_bytes_current": 77310399488, + "allocated_bytes_peak": 84832202240, + "device": 4, + "inactive_split_bytes_current": 1347480576, + "inactive_split_bytes_peak": 11552316416, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 90617937920, + "reserved_bytes_peak": 90617937920, + "segment_count_current": 1701, + "segment_count_peak": 1701, + "timestamp_utc": "2026-08-26T19:13:55.493909Z", + "world_size": 8 + }, + "allocated_bytes_current": 77310399488, + "allocated_bytes_current_delta": 32789599232, + "allocated_bytes_peak": 84832202240, + "allocated_bytes_peak_delta": 40286236160, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 4, + "inactive_split_bytes_current": 270172160, + "inactive_split_bytes_peak": 874151936, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 47949283328, + "reserved_bytes_peak": 47949283328, + "segment_count_current": 615, + "segment_count_peak": 615, + "timestamp_utc": "2026-08-26T19:13:28.769737Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1347480576, + "inactive_split_bytes_current_delta": 1077308416, + "inactive_split_bytes_peak": 11552316416, + "inactive_split_bytes_peak_delta": 10678164480, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 90617937920, + "reserved_bytes_current_delta": 42668654592, + "reserved_bytes_peak": 90617937920, + "reserved_bytes_peak_delta": 42668654592, + "segment_count_current": 1701, + "segment_count_current_delta": 1086, + "segment_count_peak": 1701, + "segment_count_peak_delta": 1086, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.pickle", + "timestamp_utc": "2026-08-26T19:13:55.534297Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..92cc55961f5e8d7e6d436905d66b317a21c4894c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c0e22735663331ab11cb4896069e2b1a9ff56157fe215c14818c650e9c5325a +size 2309736 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6380b6fd5f6b31712b06e2d65b4f238021dfbcfc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77310399488, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115718885376, + "active_bytes_peak_delta": 30886683136, + "after": { + "active_bytes_current": 77310399488, + "active_bytes_peak": 115718885376, + "allocated_bytes_current": 77310399488, + "allocated_bytes_peak": 115718885376, + "device": 4, + "inactive_split_bytes_current": 1347480576, + "inactive_split_bytes_peak": 11552316416, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117964800000, + "reserved_bytes_peak": 117964800000, + "segment_count_current": 1875, + "segment_count_peak": 1875, + "timestamp_utc": "2026-08-26T19:13:56.330153Z", + "world_size": 8 + }, + "allocated_bytes_current": 77310399488, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115718885376, + "allocated_bytes_peak_delta": 30886683136, + "before": { + "active_bytes_current": 77310399488, + "active_bytes_peak": 84832202240, + "allocated_bytes_current": 77310399488, + "allocated_bytes_peak": 84832202240, + "device": 4, + "inactive_split_bytes_current": 1347480576, + "inactive_split_bytes_peak": 11552316416, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 90617937920, + "reserved_bytes_peak": 90617937920, + "segment_count_current": 1701, + "segment_count_peak": 1701, + "timestamp_utc": "2026-08-26T19:13:55.539053Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1347480576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11552316416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117964800000, + "reserved_bytes_current_delta": 27346862080, + "reserved_bytes_peak": 117964800000, + "reserved_bytes_peak_delta": 27346862080, + "segment_count_current": 1875, + "segment_count_current_delta": 174, + "segment_count_peak": 1875, + "segment_count_peak_delta": 174, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.368692Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1b37d6ec75ae732702d2defdd736e4d97e724bec --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a433ebf6ad0b6fd862e8367d2a334be15e727899160d2cea2f2f6ef8cf5645bd +size 1832964 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c5a4c82803edab7ce17d27ca4e0ca972027b57e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77310399488, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115722308096, + "active_bytes_peak_delta": 3422720, + "after": { + "active_bytes_current": 77310399488, + "active_bytes_peak": 115722308096, + "allocated_bytes_current": 77310399488, + "allocated_bytes_peak": 115722308096, + "device": 4, + "inactive_split_bytes_current": 1347480576, + "inactive_split_bytes_peak": 11552316416, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117964800000, + "reserved_bytes_peak": 117964800000, + "segment_count_current": 1875, + "segment_count_peak": 1875, + "timestamp_utc": "2026-08-26T19:13:56.687840Z", + "world_size": 8 + }, + "allocated_bytes_current": 77310399488, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115722308096, + "allocated_bytes_peak_delta": 3422720, + "before": { + "active_bytes_current": 77310399488, + "active_bytes_peak": 115718885376, + "allocated_bytes_current": 77310399488, + "allocated_bytes_peak": 115718885376, + "device": 4, + "inactive_split_bytes_current": 1347480576, + "inactive_split_bytes_peak": 11552316416, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117964800000, + "reserved_bytes_peak": 117964800000, + "segment_count_current": 1875, + "segment_count_peak": 1875, + "timestamp_utc": "2026-08-26T19:13:56.372779Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1347480576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11552316416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117964800000, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117964800000, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1875, + "segment_count_current_delta": 0, + "segment_count_peak": 1875, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.719593Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..509395f6982ea941251c192742db637d6f9404f3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4339be419f1dfeea9b6f2341159cfa1ae4872d4b1f7db0f8bae6c4890a993b2c +size 1822365 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d62d3d95f90fbff5c7c582eb5b29e90c4ad8dc6d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77327442944, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 115722308096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77327442944, + "active_bytes_peak": 115722308096, + "allocated_bytes_current": 77327442944, + "allocated_bytes_peak": 115722308096, + "device": 4, + "inactive_split_bytes_current": 1355602944, + "inactive_split_bytes_peak": 11552316416, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 119114039296, + "reserved_bytes_peak": 119114039296, + "segment_count_current": 1713, + "segment_count_peak": 1875, + "timestamp_utc": "2026-08-26T19:13:59.326873Z", + "world_size": 8 + }, + "allocated_bytes_current": 77327442944, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115722308096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77310399488, + "active_bytes_peak": 115722308096, + "allocated_bytes_current": 77310399488, + "allocated_bytes_peak": 115722308096, + "device": 4, + "inactive_split_bytes_current": 1347480576, + "inactive_split_bytes_peak": 11552316416, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117964800000, + "reserved_bytes_peak": 117964800000, + "segment_count_current": 1875, + "segment_count_peak": 1875, + "timestamp_utc": "2026-08-26T19:13:56.723364Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1355602944, + "inactive_split_bytes_current_delta": 8122368, + "inactive_split_bytes_peak": 11552316416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 119114039296, + "reserved_bytes_current_delta": 1149239296, + "reserved_bytes_peak": 119114039296, + "reserved_bytes_peak_delta": 1149239296, + "segment_count_current": 1713, + "segment_count_current_delta": -162, + "segment_count_peak": 1875, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.364705Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e8911dfed58651edeeed3540811ad24eae299236 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81553d2001b26d037fb0ef08ecce377f2483d9911cde8478a990b930f5ca725f +size 2180760 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae10f27d208cc2f503303831b5734cf89961a21f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77327442944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115722308096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77327442944, + "active_bytes_peak": 115722308096, + "allocated_bytes_current": 77327442944, + "allocated_bytes_peak": 115722308096, + "device": 4, + "inactive_split_bytes_current": 1355602944, + "inactive_split_bytes_peak": 11552316416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 119114039296, + "reserved_bytes_peak": 119114039296, + "segment_count_current": 1713, + "segment_count_peak": 1875, + "timestamp_utc": "2026-08-26T19:13:59.672899Z", + "world_size": 8 + }, + "allocated_bytes_current": 77327442944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115722308096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77327442944, + "active_bytes_peak": 115722308096, + "allocated_bytes_current": 77327442944, + "allocated_bytes_peak": 115722308096, + "device": 4, + "inactive_split_bytes_current": 1355602944, + "inactive_split_bytes_peak": 11552316416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 119114039296, + "reserved_bytes_peak": 119114039296, + "segment_count_current": 1713, + "segment_count_peak": 1875, + "timestamp_utc": "2026-08-26T19:13:59.368796Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1355602944, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11552316416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 119114039296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119114039296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1713, + "segment_count_current_delta": 0, + "segment_count_peak": 1875, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.681876Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..35cf89954eafff93cea8189c8ee6b9eb19353dd2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78ca0e069ad76904f3dbf2aab22aa93fc2d6158cb98cf3f872b9ad469007c991 +size 427667 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e31f1cf15b7a6c70fd883d5f76ae1a5456a5934 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77327442944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115722308096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77327442944, + "active_bytes_peak": 115722308096, + "allocated_bytes_current": 77327442944, + "allocated_bytes_peak": 115722308096, + "device": 4, + "inactive_split_bytes_current": 1355602944, + "inactive_split_bytes_peak": 11552316416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 119114039296, + "reserved_bytes_peak": 119114039296, + "segment_count_current": 1713, + "segment_count_peak": 1875, + "timestamp_utc": "2026-08-26T19:13:59.985362Z", + "world_size": 8 + }, + "allocated_bytes_current": 77327442944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115722308096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77327442944, + "active_bytes_peak": 115722308096, + "allocated_bytes_current": 77327442944, + "allocated_bytes_peak": 115722308096, + "device": 4, + "inactive_split_bytes_current": 1355602944, + "inactive_split_bytes_peak": 11552316416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 119114039296, + "reserved_bytes_peak": 119114039296, + "segment_count_current": 1713, + "segment_count_peak": 1875, + "timestamp_utc": "2026-08-26T19:13:59.682999Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1355602944, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11552316416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 119114039296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119114039296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1713, + "segment_count_current_delta": 0, + "segment_count_peak": 1875, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.994375Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..209764ba9829561e6236db71d318277d6d94fe6e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0d5e64d3c98af45451f3e73d50e863218980461e14423705e2b3c59f0394ec4 +size 427667 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.json new file mode 100644 index 0000000000000000000000000000000000000000..be9ce62090a2bec0afda3f8cecafd588a8db7509 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 32780593664, + "active_bytes_peak": 84822148608, + "active_bytes_peak_delta": 40276182528, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 84822148608, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 84822148608, + "device": 5, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 90550829056, + "reserved_bytes_peak": 90550829056, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-26T19:13:55.493853Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 32780593664, + "allocated_bytes_peak": 84822148608, + "allocated_bytes_peak_delta": 40276182528, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 5, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:13:28.769726Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 849335808, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 10484395520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 90550829056, + "reserved_bytes_current_delta": 42299555840, + "reserved_bytes_peak": 90550829056, + "reserved_bytes_peak_delta": 42299555840, + "segment_count_current": 1692, + "segment_count_current_delta": 1071, + "segment_count_peak": 1692, + "segment_count_peak_delta": 1071, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.pickle", + "timestamp_utc": "2026-08-26T19:13:55.535234Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..34fed045544b0e3c1d1e1e0ae9d831ab0afa654d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c76f4addedaacbcbf678996fe2086b7ee58d9fa92ab2f9a784511a86de2d8943 +size 2301744 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d4112ba29955d2e6bbb5bf4db595c57f6a0398b0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115689589760, + "active_bytes_peak_delta": 30867441152, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689589760, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689589760, + "device": 5, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.330062Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115689589760, + "allocated_bytes_peak_delta": 30867441152, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 84822148608, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 84822148608, + "device": 5, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 90550829056, + "reserved_bytes_peak": 90550829056, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-26T19:13:55.539528Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118057074688, + "reserved_bytes_current_delta": 27506245632, + "reserved_bytes_peak": 118057074688, + "reserved_bytes_peak_delta": 27506245632, + "segment_count_current": 1871, + "segment_count_current_delta": 179, + "segment_count_peak": 1871, + "segment_count_peak_delta": 179, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.368425Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ec0b01ad457fc6e6a9cd798b7ab2b53cee9e178c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae3d144b7a7aa70ae100304915a5acf40b282558356bb7724a06362bcfe37d0c +size 1827611 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d9a62d5fbd1e5919c5e08bab1dc7a7df792ad908 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115689851904, + "active_bytes_peak_delta": 262144, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689851904, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689851904, + "device": 5, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.687752Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115689851904, + "allocated_bytes_peak_delta": 262144, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689589760, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689589760, + "device": 5, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.372485Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118057074688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118057074688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1871, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.719780Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3b059c951fa2b043c4b9d3536f14453b1e375d0b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8afceca630ac9709d43717867212431b78f6cd8c6e489d31b455d6b6405ba57f +size 1818571 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9abc6aa248bedcd32bdddc5345f7384c5926479a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 6557696, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 5, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.326814Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 6557696, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689851904, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689851904, + "device": 5, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.723479Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 8122368, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 970981376, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 970981376, + "segment_count_current": 1688, + "segment_count_current_delta": -183, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.365472Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ff20de45cbb95d9c79803f7ba2a7e41e9fb33259 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bb280f177e21f491febe7b9350f04063491f550be6141cedaa3bd07e6fd62e8 +size 2188971 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.json new file mode 100644 index 0000000000000000000000000000000000000000..44700ee8cb6fe74e347435cd4810e41fd99b340b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 5, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.672803Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 5, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.369653Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1688, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.681756Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ef9593855bfd8192508dd8521ac90af42d39e77 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc4f8e39145028b10440661429b72d0baa3b625ed880807aeba57acb39ef8785 +size 421723 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.json new file mode 100644 index 0000000000000000000000000000000000000000..548556f0161155979c6887addf4f42f6063e2f7c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 5, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.985251Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 5, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.682961Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1688, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.994017Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fdec24d20f94fbaa869a9b65d3d8b7b90749e480 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ee5b558e3780976e2049d345a43cd3c121fad600394fb1ca9cdc3230e8c8c23 +size 421723 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd5dce91f149b03f87901e9968a6b6e093e5d37b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 32780593664, + "active_bytes_peak": 84822148608, + "active_bytes_peak_delta": 40276182528, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 84822148608, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 84822148608, + "device": 6, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 90550829056, + "reserved_bytes_peak": 90550829056, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-26T19:13:55.493820Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 32780593664, + "allocated_bytes_peak": 84822148608, + "allocated_bytes_peak_delta": 40276182528, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 6, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:13:28.769718Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 849335808, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 10484395520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 90550829056, + "reserved_bytes_current_delta": 42299555840, + "reserved_bytes_peak": 90550829056, + "reserved_bytes_peak_delta": 42299555840, + "segment_count_current": 1692, + "segment_count_current_delta": 1071, + "segment_count_peak": 1692, + "segment_count_peak_delta": 1071, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.pickle", + "timestamp_utc": "2026-08-26T19:13:55.534205Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e30c5e1de39ada066bf9945659282d62b95849f9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7546d1ea9f3f0d79614678667f8c94abf0d3be2c7be5c9fc2bcae3181cc78087 +size 2301744 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.json new file mode 100644 index 0000000000000000000000000000000000000000..902187f53a91eb9af7b0d35cbeaadce153118380 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115689589760, + "active_bytes_peak_delta": 30867441152, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689589760, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689589760, + "device": 6, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.330030Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115689589760, + "allocated_bytes_peak_delta": 30867441152, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 84822148608, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 84822148608, + "device": 6, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 90550829056, + "reserved_bytes_peak": 90550829056, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-26T19:13:55.538393Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118057074688, + "reserved_bytes_current_delta": 27506245632, + "reserved_bytes_peak": 118057074688, + "reserved_bytes_peak_delta": 27506245632, + "segment_count_current": 1871, + "segment_count_current_delta": 179, + "segment_count_peak": 1871, + "segment_count_peak_delta": 179, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.368321Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ef6d954cab0149d9a54747f11b9e53f54759eb3d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e773a8d295d08939892c810ab7cedf3e6e3476f039cd840179cf2d3625501a4 +size 1827611 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ac2aa05a6bf0aeb020ca443f0dba341b97755bc8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115689851904, + "active_bytes_peak_delta": 262144, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689851904, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689851904, + "device": 6, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.687721Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115689851904, + "allocated_bytes_peak_delta": 262144, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689589760, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689589760, + "device": 6, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.372359Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118057074688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118057074688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1871, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.719841Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e6c811c4fdc631ee24a35753fdf79088ced2120 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:477b4999a2bd545526ad3f57e9928a1b11714caacafeec8d642e934dd20026ee +size 1818569 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.json new file mode 100644 index 0000000000000000000000000000000000000000..05fdc0b6b136252cbb1a9a6b48f4aff372bc4da7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 6557696, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 6, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.326767Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 6557696, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689851904, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689851904, + "device": 6, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.723546Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 8122368, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 970981376, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 970981376, + "segment_count_current": 1688, + "segment_count_current_delta": -183, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.365192Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..45f715ba7a44da8ab3a66d16ee56d23ade406bb8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9b11c7add56a973ef2608477b59db90684f3564f425375500655d32885ac3d2 +size 2188967 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.json new file mode 100644 index 0000000000000000000000000000000000000000..83830a3204292fad076e948498f4ca9d35856225 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 6, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.672754Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 6, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.370060Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1688, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.681579Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80f6d2767fde4000cd595b27e6f3b5551f1c22a6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46494cb4f75b92861470d7953cc41c7290a9ef1a4b8a8fbbc30d32287de3c77c +size 421719 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.json new file mode 100644 index 0000000000000000000000000000000000000000..890780f662ffcff7e9782adc034a567a81342726 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 6, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.985258Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 6, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.682681Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1688, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.994267Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3ebc3003ef6c74091ad4653be2ec9fdc6f95f1e7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8ed52e7fb1ba42cb3dfb7cfe137845aec85ea0965a531b8b3fcf3e7dd3f7e76 +size 421719 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e298aa7f8d992c77aa9b48e22b3e769fa1879b6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 32780593664, + "active_bytes_peak": 84822148608, + "active_bytes_peak_delta": 40276182528, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 84822148608, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 84822148608, + "device": 7, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 90550829056, + "reserved_bytes_peak": 90550829056, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-26T19:13:55.493830Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 32780593664, + "allocated_bytes_peak": 84822148608, + "allocated_bytes_peak_delta": 40276182528, + "before": { + "active_bytes_current": 44520800256, + "active_bytes_peak": 44545966080, + "allocated_bytes_current": 44520800256, + "allocated_bytes_peak": 44545966080, + "device": 7, + "inactive_split_bytes_current": 421167104, + "inactive_split_bytes_peak": 1025146880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:13:28.769712Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 849335808, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 10484395520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 90550829056, + "reserved_bytes_current_delta": 42299555840, + "reserved_bytes_peak": 90550829056, + "reserved_bytes_peak_delta": 42299555840, + "segment_count_current": 1692, + "segment_count_current_delta": 1071, + "segment_count_peak": 1692, + "segment_count_peak_delta": 1071, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.pickle", + "timestamp_utc": "2026-08-26T19:13:55.533437Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2304b9e42437de827cd4496aa725665713457b55 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3a4a4c320ca7cff52964878a75848b1c44cdceca40f57f8cf772c106d7ed2c3 +size 2301744 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.json new file mode 100644 index 0000000000000000000000000000000000000000..38734e90f0b75e5976a563b4eb319d17f99985d7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115689589760, + "active_bytes_peak_delta": 30867441152, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689589760, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689589760, + "device": 7, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.330084Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115689589760, + "allocated_bytes_peak_delta": 30867441152, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 84822148608, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 84822148608, + "device": 7, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 90550829056, + "reserved_bytes_peak": 90550829056, + "segment_count_current": 1692, + "segment_count_peak": 1692, + "timestamp_utc": "2026-08-26T19:13:55.537623Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118057074688, + "reserved_bytes_current_delta": 27506245632, + "reserved_bytes_peak": 118057074688, + "reserved_bytes_peak_delta": 27506245632, + "segment_count_current": 1871, + "segment_count_current_delta": 179, + "segment_count_peak": 1871, + "segment_count_peak_delta": 179, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.366764Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..daa15241289b473a2648b056f24df347f9ea808b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b039cf25681691e304df5a1e5986b58c7f46fdda2fa1a9e415857fc5f19d97c1 +size 1827611 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.json new file mode 100644 index 0000000000000000000000000000000000000000..78c8a68fe2880b5b485c4b0574951f0b5f4457c6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77301393920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115689851904, + "active_bytes_peak_delta": 262144, + "after": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689851904, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689851904, + "device": 7, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.687775Z", + "world_size": 8 + }, + "allocated_bytes_current": 77301393920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115689851904, + "allocated_bytes_peak_delta": 262144, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689589760, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689589760, + "device": 7, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.370614Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118057074688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118057074688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1871, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.pickle", + "timestamp_utc": "2026-08-26T19:13:56.718676Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..26c6ef89bc52fb8dade428b28285cab712762fac --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83a6c52cd07fca4b9503d38d0e115d42d3d8a779f0d8fd7be8553d584bcce60e +size 1818569 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d5d7557bf8fd321854b3ee79fb851d5dd1193649 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 6557696, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 7, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.326845Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 6557696, + "before": { + "active_bytes_current": 77301393920, + "active_bytes_peak": 115689851904, + "allocated_bytes_current": 77301393920, + "allocated_bytes_peak": 115689851904, + "device": 7, + "inactive_split_bytes_current": 1270502912, + "inactive_split_bytes_peak": 11509542400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118057074688, + "reserved_bytes_peak": 118057074688, + "segment_count_current": 1871, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:56.722150Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 8122368, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 970981376, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 970981376, + "segment_count_current": 1688, + "segment_count_current_delta": -183, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.365437Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e546d54521666d079c45b1e456800cb7492f29e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f769ed3ca5a81059504fdac9122eee3bc84daf6b28b36b3f4458951c58e6738 +size 2188969 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bf4a76269c143d42951c8158be798819084a510f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 7, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.672803Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 7, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.369446Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1688, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.681882Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9b5f308366c5081c90885299ad419e141e3246de --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2530246e9f8fdaeddf81c262d7186b4d97c1e7010a93d79ebdef32546b960e77 +size 421721 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191359985689Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191359985689Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d0d8ca2f2db4d0d14f6472bec7c1741438526627 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191359985689Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77318437376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115696409600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 7, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.985310Z", + "world_size": 8 + }, + "allocated_bytes_current": 77318437376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115696409600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77318437376, + "active_bytes_peak": 115696409600, + "allocated_bytes_current": 77318437376, + "allocated_bytes_peak": 115696409600, + "device": 7, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_peak": 11509542400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 119028056064, + "reserved_bytes_peak": 119028056064, + "segment_count_current": 1688, + "segment_count_peak": 1871, + "timestamp_utc": "2026-08-26T19:13:59.683052Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1278625280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11509542400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119028056064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119028056064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1688, + "segment_count_current_delta": 0, + "segment_count_peak": 1871, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T191359985689Z.pickle", + "timestamp_utc": "2026-08-26T19:13:59.994216Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191359985689Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191359985689Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3fed20e7bc526b8679fff3139aaf27e266b5c7a9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191359985689Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:649d6483ae08ffb0405d146df6b79b700fc6ebc4f650eb9760ce2499285f61e9 +size 421721 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..16be22419c44d8dd69ae40b31b7b1d6b242eae7e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-better-memory-dump-support +commit=c3a8a8a7 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=512 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..81a9a9c3107c5969b7c4983dbbab9ec5be9b1113 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log @@ -0,0 +1,1446 @@ +[19:12:59] WARN: ultraep-better-memory-dump-support lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / ultraep (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ c3a8a8a7 +============================================================== +W0826 19:13:00.637000 633046 torch/distributed/run.py:766] +W0826 19:13:00.637000 633046 torch/distributed/run.py:766] ***************************************** +W0826 19:13:00.637000 633046 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:13:00.637000 633046 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank [Gloo] Rank 3[Gloo] Rank is connected to 20 is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to [Gloo] Rank 7 is connected to peer ranks. 5Expected number of connected peer ranks is : is connected to 777 + peer ranks. [Gloo] Rank peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 7 is connected to 77 + + peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 201 is connected to is connected to is connected to 777 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77[Gloo] Rank 7 + + +3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +[Gloo] Rank 7 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W826 19:13:18.935398546 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:13:18.945728821 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:13:18.945752450 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:13:18.945791814 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:13:18.945880597 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:13:18.945968968 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:13:18.945981293 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:13:19.099917309 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 4.002 seconds +time to initialize megatron (seconds): 7.192 +[after megatron is initialized] datetime: 2026-08-26 19:13:25 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 8521882624 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:13:28 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:13:28 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3567.56, 3576.97) + train/valid/test-data-iterators-setup ..........: (73.79, 92.24) +[before the start of training step] datetime: 2026-08-26 19:13:28 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:13:55] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 26723.6 | throughput per GPU (TFLOP/s/GPU): 6.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.116001E+01 | loss scale: 1.0 | grad norm: 44.808 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67456 | min safe capacity: 1.0293 | +Number of parameters in transformer block in billions: 29.85 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 73717.49853515625 | max allocated: 80889.84912109375 | reserved: 86284.0 | max reserved: 86284.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 19:13:56] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 836.8 | throughput per GPU (TFLOP/s/GPU): 216.1 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117607E+01 | loss scale: 1.0 | grad norm: 44.340 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67712 | min safe capacity: 1.0332 | + [2026-08-26 19:13:56] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 357.6 | throughput per GPU (TFLOP/s/GPU): 505.6 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.685802E+00 | loss scale: 1.0 | grad norm: 62.813 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:13:59] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 2639.0 | throughput per GPU (TFLOP/s/GPU): 68.5 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338091E+01 | loss scale: 1.0 | grad norm: 20.303 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | + [2026-08-26 19:13:59] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 346.0 | throughput per GPU (TFLOP/s/GPU): 522.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.050091E+01 | loss scale: 1.0 | grad norm: 95.180 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | + [2026-08-26 19:13:59] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 312.4 | throughput per GPU (TFLOP/s/GPU): 578.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.041164E+01 | loss scale: 1.0 | grad norm: 50.092 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67840 | min safe capacity: 1.0352 | +[after training is done] datetime: 2026-08-26 19:14:00 +[rank0]:[W826 19:14:01.224461373 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 19:14:02.511398812 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 19:14:02.519705952 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 19:14:02.524106688 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 19:14:02.524284240 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 19:14:02.524442400 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 19:14:02.525602877 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 19:14:02.525799785 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:14:09.331000 633046 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:14:09.332000 633046 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 633139 closing signal SIGTERM +W0826 19:14:09.332000 633046 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 633141 closing signal SIGTERM +W0826 19:14:09.333000 633046 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 633144 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 633046 got signal: 15 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..481f546bfb57c5d707cbe4621c2bee29f3a8e350 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 2 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 27 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 819 +2, 821 +3, 821 +4, 819 +5, 821 +6, 821 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2551 +1, 2551 +2, 2551 +3, 2551 +4, 2551 +5, 2551 +6, 2551 +7, 2551 +0, 2959 +1, 2959 +2, 2959 +3, 2959 +4, 2959 +5, 2959 +6, 2959 +7, 2959 +0, 51537 +1, 51537 +2, 51537 +3, 51537 +4, 51249 +5, 51537 +6, 51537 +7, 51537 +0, 53015 +1, 53015 +2, 53015 +3, 53015 +4, 52727 +5, 53015 +6, 53015 +7, 53015 +0, 54649 +1, 54649 +2, 54649 +3, 54649 +4, 54361 +5, 54649 +6, 54649 +7, 54649 +0, 54649 +1, 54649 +2, 54649 +3, 54649 +4, 54361 +5, 54649 +6, 54649 +7, 54649 +0, 55175 +1, 55175 +2, 55175 +3, 55159 +4, 54887 +5, 55153 +6, 55175 +7, 55175 +0, 56903 +1, 56905 +2, 56905 +3, 56843 +4, 56615 +5, 56815 +6, 56903 +7, 56903 +0, 58129 +1, 57859 +2, 58137 +3, 58069 +4, 58331 +5, 58039 +6, 57825 +7, 57873 +0, 88251 +1, 88391 +2, 88365 +3, 88203 +4, 88301 +5, 88289 +6, 88273 +7, 88315 +0, 88843 +1, 88989 +2, 88961 +3, 88797 +4, 88893 +5, 88889 +6, 88859 +7, 88909 +0, 88843 +1, 88989 +2, 88961 +3, 88797 +4, 88893 +5, 88889 +6, 88859 +7, 88909 +0, 88843 +1, 88989 +2, 88961 +3, 88797 +4, 88893 +5, 88889 +6, 88859 +7, 88909 +0, 89247 +1, 89393 +2, 89365 +3, 89201 +4, 89297 +5, 89293 +6, 89263 +7, 89313 +0, 95347 +1, 95223 +2, 95497 +3, 95427 +4, 95613 +5, 95555 +6, 95533 +7, 95491 +0, 120453 +1, 120689 +2, 120695 +3, 120625 +4, 120701 +5, 120609 +6, 120533 +7, 120625 +0, 120455 +1, 120689 +2, 120697 +3, 120627 +4, 120703 +5, 120611 +6, 120535 +7, 120627 +0, 115479 +1, 115713 +2, 115721 +3, 115389 +4, 115727 +5, 115635 +6, 115559 +7, 115651 +0, 113731 +1, 113645 +2, 113653 +3, 113647 +4, 114305 +5, 113887 +6, 113555 +7, 114229 +0, 112377 +1, 4 +2, 112619 +3, 4 +4, 112625 +5, 4 +6, 4 +7, 112549 +0, 112374 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 4 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191510669713Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191510669713Z.json new file mode 100644 index 0000000000000000000000000000000000000000..95ca5d77c19e698c3cde5f37f87d119a741e9e02 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191510669713Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77344410624, + "active_bytes_current_delta": 32823616512, + "active_bytes_peak": 82722035712, + "active_bytes_peak_delta": 38176075776, + "after": { + "active_bytes_current": 77344410624, + "active_bytes_peak": 82722035712, + "allocated_bytes_current": 77344410624, + "allocated_bytes_peak": 82722035712, + "device": 0, + "inactive_split_bytes_current": 1110045696, + "inactive_split_bytes_peak": 13375292928, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 89911197696, + "reserved_bytes_peak": 89911197696, + "segment_count_current": 1673, + "segment_count_peak": 1673, + "timestamp_utc": "2026-08-26T19:15:10.669232Z", + "world_size": 8 + }, + "allocated_bytes_current": 77344410624, + "allocated_bytes_current_delta": 32823616512, + "allocated_bytes_peak": 82722035712, + "allocated_bytes_peak_delta": 38176075776, + "before": { + "active_bytes_current": 44520794112, + "active_bytes_peak": 44545959936, + "allocated_bytes_current": 44520794112, + "allocated_bytes_peak": 44545959936, + "device": 0, + "inactive_split_bytes_current": 446339072, + "inactive_split_bytes_peak": 1050318848, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:14:41.061734Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1110045696, + "inactive_split_bytes_current_delta": 663706624, + "inactive_split_bytes_peak": 13375292928, + "inactive_split_bytes_peak_delta": 12324974080, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 89911197696, + "reserved_bytes_current_delta": 41659924480, + "reserved_bytes_peak": 89911197696, + "reserved_bytes_peak_delta": 41659924480, + "segment_count_current": 1673, + "segment_count_current_delta": 1052, + "segment_count_peak": 1673, + "segment_count_peak_delta": 1052, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T191510669713Z.pickle", + "timestamp_utc": "2026-08-26T19:15:10.710363Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191510669713Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191510669713Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..246c6afac1b8f98c6830032156bac34f857913f1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191510669713Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80a614606ac49be5fcef6bf9e92dab0be3d46f08b0d0f5f2824bda88398f1fab +size 2315809 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191511400846Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191511400846Z.json new file mode 100644 index 0000000000000000000000000000000000000000..796d0cadb42c84603381654c6476f2cfcd3e2021 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191511400846Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77344410624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113608813056, + "active_bytes_peak_delta": 30886777344, + "after": { + "active_bytes_current": 77344410624, + "active_bytes_peak": 113608813056, + "allocated_bytes_current": 77344410624, + "allocated_bytes_peak": 113608813056, + "device": 0, + "inactive_split_bytes_current": 1110045696, + "inactive_split_bytes_peak": 13375292928, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115653738496, + "reserved_bytes_peak": 115653738496, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:11.400440Z", + "world_size": 8 + }, + "allocated_bytes_current": 77344410624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113608813056, + "allocated_bytes_peak_delta": 30886777344, + "before": { + "active_bytes_current": 77344410624, + "active_bytes_peak": 82722035712, + "allocated_bytes_current": 77344410624, + "allocated_bytes_peak": 82722035712, + "device": 0, + "inactive_split_bytes_current": 1110045696, + "inactive_split_bytes_peak": 13375292928, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 89911197696, + "reserved_bytes_peak": 89911197696, + "segment_count_current": 1673, + "segment_count_peak": 1673, + "timestamp_utc": "2026-08-26T19:15:10.714999Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1110045696, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13375292928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115653738496, + "reserved_bytes_current_delta": 25742540800, + "reserved_bytes_peak": 115653738496, + "reserved_bytes_peak_delta": 25742540800, + "segment_count_current": 1899, + "segment_count_current_delta": 226, + "segment_count_peak": 1899, + "segment_count_peak_delta": 226, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T191511400846Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.436583Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191511400846Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191511400846Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0263f461ce46e8354371ad866648bfdc630ae16d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191511400846Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5fa595be016dcf470f27fb6abda26211b24353be14912fc99f1080dec4fc156 +size 1910460 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191511811276Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191511811276Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d5ab751266c09a57c6563f615e24e41eb0c6ff0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191511811276Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77344410624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113670829056, + "active_bytes_peak_delta": 62016000, + "after": { + "active_bytes_current": 77344410624, + "active_bytes_peak": 113670829056, + "allocated_bytes_current": 77344410624, + "allocated_bytes_peak": 113670829056, + "device": 0, + "inactive_split_bytes_current": 1110045696, + "inactive_split_bytes_peak": 13375292928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115653738496, + "reserved_bytes_peak": 115653738496, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:11.810882Z", + "world_size": 8 + }, + "allocated_bytes_current": 77344410624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113670829056, + "allocated_bytes_peak_delta": 62016000, + "before": { + "active_bytes_current": 77344410624, + "active_bytes_peak": 113608813056, + "allocated_bytes_current": 77344410624, + "allocated_bytes_peak": 113608813056, + "device": 0, + "inactive_split_bytes_current": 1110045696, + "inactive_split_bytes_peak": 13375292928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115653738496, + "reserved_bytes_peak": 115653738496, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:11.440578Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1110045696, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13375292928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115653738496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115653738496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1899, + "segment_count_current_delta": 0, + "segment_count_peak": 1899, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T191511811276Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.842736Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191511811276Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191511811276Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f7e88bdb2a6660d3308b8738c50f315a60f18f76 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191511811276Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75e51af771a1bc4175eb800f4ba0b16dd1303737120ed34ea25d07176d06e62c +size 1901524 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191512211808Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191512211808Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a824c6ed7599dc47d8132f92041fdec1b7b20ef1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191512211808Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77344410624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113670829056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77344410624, + "active_bytes_peak": 113670829056, + "allocated_bytes_current": 77344410624, + "allocated_bytes_peak": 113670829056, + "device": 0, + "inactive_split_bytes_current": 1112142848, + "inactive_split_bytes_peak": 13375292928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115653738496, + "reserved_bytes_peak": 115653738496, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:12.211411Z", + "world_size": 8 + }, + "allocated_bytes_current": 77344410624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113670829056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77344410624, + "active_bytes_peak": 113670829056, + "allocated_bytes_current": 77344410624, + "allocated_bytes_peak": 113670829056, + "device": 0, + "inactive_split_bytes_current": 1110045696, + "inactive_split_bytes_peak": 13375292928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115653738496, + "reserved_bytes_peak": 115653738496, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:11.846428Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1112142848, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 13375292928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115653738496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115653738496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1899, + "segment_count_current_delta": 0, + "segment_count_peak": 1899, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T191512211808Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.242661Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191512211808Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191512211808Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ce413a531f7308d8724169c3510ce242a803a1df --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191512211808Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:516c45abd32d11daed64329866799e2aa1c0ccdabfea7e28e77e13f9a1f37a9d +size 1901271 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191512610990Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191512610990Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e17b85596b2ab449f400216c6a96cdfd1e056b5b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191512610990Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77344410624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113670829056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77344410624, + "active_bytes_peak": 113670829056, + "allocated_bytes_current": 77344410624, + "allocated_bytes_peak": 113670829056, + "device": 0, + "inactive_split_bytes_current": 1112142848, + "inactive_split_bytes_peak": 13375292928, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115653738496, + "reserved_bytes_peak": 115653738496, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:12.610603Z", + "world_size": 8 + }, + "allocated_bytes_current": 77344410624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113670829056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77344410624, + "active_bytes_peak": 113670829056, + "allocated_bytes_current": 77344410624, + "allocated_bytes_peak": 113670829056, + "device": 0, + "inactive_split_bytes_current": 1112142848, + "inactive_split_bytes_peak": 13375292928, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115653738496, + "reserved_bytes_peak": 115653738496, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:12.246190Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 1112142848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13375292928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115653738496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115653738496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1899, + "segment_count_current_delta": 0, + "segment_count_peak": 1899, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T191512610990Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.641275Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191512610990Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191512610990Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..812f66b40f2e913ab8277e19a25fcfc2ffcc4a81 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191512610990Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1020f39e381c3ffe92189bb39c8141e35421da7c809b100ec4a54307853eb7f0 +size 1901051 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191510669586Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191510669586Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4b08253d1d09644765e06c824fc0d8194f815a81 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191510669586Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77342477312, + "active_bytes_current_delta": 32821683200, + "active_bytes_peak": 82842547712, + "active_bytes_peak_delta": 38296587776, + "after": { + "active_bytes_current": 77342477312, + "active_bytes_peak": 82842547712, + "allocated_bytes_current": 77342477312, + "allocated_bytes_peak": 82842547712, + "device": 1, + "inactive_split_bytes_current": 1202156544, + "inactive_split_bytes_peak": 15444608512, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 89762299904, + "reserved_bytes_peak": 89762299904, + "segment_count_current": 1650, + "segment_count_peak": 1650, + "timestamp_utc": "2026-08-26T19:15:10.669033Z", + "world_size": 8 + }, + "allocated_bytes_current": 77342477312, + "allocated_bytes_current_delta": 32821683200, + "allocated_bytes_peak": 82842547712, + "allocated_bytes_peak_delta": 38296587776, + "before": { + "active_bytes_current": 44520794112, + "active_bytes_peak": 44545959936, + "allocated_bytes_current": 44520794112, + "allocated_bytes_peak": 44545959936, + "device": 1, + "inactive_split_bytes_current": 421173248, + "inactive_split_bytes_peak": 1025153024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:14:41.061552Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1202156544, + "inactive_split_bytes_current_delta": 780983296, + "inactive_split_bytes_peak": 15444608512, + "inactive_split_bytes_peak_delta": 14419455488, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 89762299904, + "reserved_bytes_current_delta": 41511026688, + "reserved_bytes_peak": 89762299904, + "reserved_bytes_peak_delta": 41511026688, + "segment_count_current": 1650, + "segment_count_current_delta": 1029, + "segment_count_peak": 1650, + "segment_count_peak_delta": 1029, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T191510669586Z.pickle", + "timestamp_utc": "2026-08-26T19:15:10.709706Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191510669586Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191510669586Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9012473d5adb1bbb57d4381dffb5f18a3d24c54b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191510669586Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7c8548415175b64c6e2e72e0d823f931165e491df75531446c5cdaa364005f3 +size 2299497 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191511400753Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191511400753Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4171f1962d0954060e9bb8a1c7418b4b94134b53 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191511400753Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77342477312, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113626272256, + "active_bytes_peak_delta": 30783724544, + "after": { + "active_bytes_current": 77342477312, + "active_bytes_peak": 113626272256, + "allocated_bytes_current": 77342477312, + "allocated_bytes_peak": 113626272256, + "device": 1, + "inactive_split_bytes_current": 1204253696, + "inactive_split_bytes_peak": 15444608512, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115899105280, + "reserved_bytes_peak": 115899105280, + "segment_count_current": 1870, + "segment_count_peak": 1870, + "timestamp_utc": "2026-08-26T19:15:11.400349Z", + "world_size": 8 + }, + "allocated_bytes_current": 77342477312, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113626272256, + "allocated_bytes_peak_delta": 30783724544, + "before": { + "active_bytes_current": 77342477312, + "active_bytes_peak": 82842547712, + "allocated_bytes_current": 77342477312, + "allocated_bytes_peak": 82842547712, + "device": 1, + "inactive_split_bytes_current": 1202156544, + "inactive_split_bytes_peak": 15444608512, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 89762299904, + "reserved_bytes_peak": 89762299904, + "segment_count_current": 1650, + "segment_count_peak": 1650, + "timestamp_utc": "2026-08-26T19:15:10.714280Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1204253696, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 15444608512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115899105280, + "reserved_bytes_current_delta": 26136805376, + "reserved_bytes_peak": 115899105280, + "reserved_bytes_peak_delta": 26136805376, + "segment_count_current": 1870, + "segment_count_current_delta": 220, + "segment_count_peak": 1870, + "segment_count_peak_delta": 220, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T191511400753Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.435864Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191511400753Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191511400753Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d7426eeab2af835591cfd804e148cbc5f6f72203 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191511400753Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57837dd51cead1317638feb826624dffa41da197a6312f17d4aa55c7e403007c +size 1893720 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191511811247Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191511811247Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7bd0c6893e86429c1ed18842ef8fa15996fb32d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191511811247Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77342477312, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113639584256, + "active_bytes_peak_delta": 13312000, + "after": { + "active_bytes_current": 77342477312, + "active_bytes_peak": 113639584256, + "allocated_bytes_current": 77342477312, + "allocated_bytes_peak": 113639584256, + "device": 1, + "inactive_split_bytes_current": 1202156544, + "inactive_split_bytes_peak": 15444608512, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115899105280, + "reserved_bytes_peak": 115899105280, + "segment_count_current": 1870, + "segment_count_peak": 1870, + "timestamp_utc": "2026-08-26T19:15:11.810836Z", + "world_size": 8 + }, + "allocated_bytes_current": 77342477312, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113639584256, + "allocated_bytes_peak_delta": 13312000, + "before": { + "active_bytes_current": 77342477312, + "active_bytes_peak": 113626272256, + "allocated_bytes_current": 77342477312, + "allocated_bytes_peak": 113626272256, + "device": 1, + "inactive_split_bytes_current": 1204253696, + "inactive_split_bytes_peak": 15444608512, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 115899105280, + "reserved_bytes_peak": 115899105280, + "segment_count_current": 1870, + "segment_count_peak": 1870, + "timestamp_utc": "2026-08-26T19:15:11.439700Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1202156544, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 15444608512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115899105280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115899105280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1870, + "segment_count_current_delta": 0, + "segment_count_peak": 1870, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T191511811247Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.843762Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191511811247Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191511811247Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..298e8d2782aac18d4f8238148641216eba0aef68 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191511811247Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:070ebe975d8141a08f86209b68711d7b8912c02eccf6e86deeefe6d4d6968869 +size 1887742 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191512211664Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191512211664Z.json new file mode 100644 index 0000000000000000000000000000000000000000..306e7f590dd516669ae6910a0e443766fdf18976 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191512211664Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77342477312, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113639584256, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77342477312, + "active_bytes_peak": 113639584256, + "allocated_bytes_current": 77342477312, + "allocated_bytes_peak": 113639584256, + "device": 1, + "inactive_split_bytes_current": 1202156544, + "inactive_split_bytes_peak": 15444608512, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115899105280, + "reserved_bytes_peak": 115899105280, + "segment_count_current": 1870, + "segment_count_peak": 1870, + "timestamp_utc": "2026-08-26T19:15:12.211280Z", + "world_size": 8 + }, + "allocated_bytes_current": 77342477312, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113639584256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77342477312, + "active_bytes_peak": 113639584256, + "allocated_bytes_current": 77342477312, + "allocated_bytes_peak": 113639584256, + "device": 1, + "inactive_split_bytes_current": 1202156544, + "inactive_split_bytes_peak": 15444608512, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 115899105280, + "reserved_bytes_peak": 115899105280, + "segment_count_current": 1870, + "segment_count_peak": 1870, + "timestamp_utc": "2026-08-26T19:15:11.847224Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1202156544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15444608512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115899105280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115899105280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1870, + "segment_count_current_delta": 0, + "segment_count_peak": 1870, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T191512211664Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.243180Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191512211664Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191512211664Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..139cec273667c5ccc5b111a9737d9a153f2e181f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191512211664Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a10ed88312bae4cd8989d0893bd0d8740336bf3025568d03db17348287f192a0 +size 1887405 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191512610844Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191512610844Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e01ee5b143627e587e6f5e2ef86b725d08410617 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191512610844Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77342477312, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113639584256, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77342477312, + "active_bytes_peak": 113639584256, + "allocated_bytes_current": 77342477312, + "allocated_bytes_peak": 113639584256, + "device": 1, + "inactive_split_bytes_current": 1202156544, + "inactive_split_bytes_peak": 15444608512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115899105280, + "reserved_bytes_peak": 115899105280, + "segment_count_current": 1870, + "segment_count_peak": 1870, + "timestamp_utc": "2026-08-26T19:15:12.610454Z", + "world_size": 8 + }, + "allocated_bytes_current": 77342477312, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113639584256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77342477312, + "active_bytes_peak": 113639584256, + "allocated_bytes_current": 77342477312, + "allocated_bytes_peak": 113639584256, + "device": 1, + "inactive_split_bytes_current": 1202156544, + "inactive_split_bytes_peak": 15444608512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 115899105280, + "reserved_bytes_peak": 115899105280, + "segment_count_current": 1870, + "segment_count_peak": 1870, + "timestamp_utc": "2026-08-26T19:15:12.246708Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1202156544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15444608512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115899105280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115899105280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1870, + "segment_count_current_delta": 0, + "segment_count_peak": 1870, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T191512610844Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.641758Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191512610844Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191512610844Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d608ebad85b1e48c57b9498ca134cc50a0451f33 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191512610844Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e861eee6c452bcd919b9e202e81432fdef565f1758b443a7adac1e537200c38d +size 1887283 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191510669641Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191510669641Z.json new file mode 100644 index 0000000000000000000000000000000000000000..82265125447b70e23ab213d781cd3bcb568949a8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191510669641Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345627136, + "active_bytes_current_delta": 32824833024, + "active_bytes_peak": 82853709312, + "active_bytes_peak_delta": 38307749376, + "after": { + "active_bytes_current": 77345627136, + "active_bytes_peak": 82853709312, + "allocated_bytes_current": 77345627136, + "allocated_bytes_peak": 82853709312, + "device": 2, + "inactive_split_bytes_current": 1117217792, + "inactive_split_bytes_peak": 13053774848, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 90009763840, + "reserved_bytes_peak": 90009763840, + "segment_count_current": 1681, + "segment_count_peak": 1681, + "timestamp_utc": "2026-08-26T19:15:10.669142Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345627136, + "allocated_bytes_current_delta": 32824833024, + "allocated_bytes_peak": 82853709312, + "allocated_bytes_peak_delta": 38307749376, + "before": { + "active_bytes_current": 44520794112, + "active_bytes_peak": 44545959936, + "allocated_bytes_current": 44520794112, + "allocated_bytes_peak": 44545959936, + "device": 2, + "inactive_split_bytes_current": 421173248, + "inactive_split_bytes_peak": 1025153024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:14:41.061837Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1117217792, + "inactive_split_bytes_current_delta": 696044544, + "inactive_split_bytes_peak": 13053774848, + "inactive_split_bytes_peak_delta": 12028621824, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 90009763840, + "reserved_bytes_current_delta": 41758490624, + "reserved_bytes_peak": 90009763840, + "reserved_bytes_peak_delta": 41758490624, + "segment_count_current": 1681, + "segment_count_current_delta": 1060, + "segment_count_peak": 1681, + "segment_count_peak_delta": 1060, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T191510669641Z.pickle", + "timestamp_utc": "2026-08-26T19:15:10.709987Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191510669641Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191510669641Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..943413e385a272c2d376153346c3006af9183521 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191510669641Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fa8629fb60c7b79ce172814d8dad67d9e354150e0640b6a15bfa6bbfd0fec60 +size 2304409 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191511400899Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191511400899Z.json new file mode 100644 index 0000000000000000000000000000000000000000..64efc870e380f319b09092b5f7a789a5123bc3e3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191511400899Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345627136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113699828224, + "active_bytes_peak_delta": 30846118912, + "after": { + "active_bytes_current": 77345627136, + "active_bytes_peak": 113699828224, + "allocated_bytes_current": 77345627136, + "allocated_bytes_peak": 113699828224, + "device": 2, + "inactive_split_bytes_current": 1115120640, + "inactive_split_bytes_peak": 13053774848, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115907493888, + "reserved_bytes_peak": 115907493888, + "segment_count_current": 1891, + "segment_count_peak": 1891, + "timestamp_utc": "2026-08-26T19:15:11.400466Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345627136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113699828224, + "allocated_bytes_peak_delta": 30846118912, + "before": { + "active_bytes_current": 77345627136, + "active_bytes_peak": 82853709312, + "allocated_bytes_current": 77345627136, + "allocated_bytes_peak": 82853709312, + "device": 2, + "inactive_split_bytes_current": 1117217792, + "inactive_split_bytes_peak": 13053774848, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 90009763840, + "reserved_bytes_peak": 90009763840, + "segment_count_current": 1681, + "segment_count_peak": 1681, + "timestamp_utc": "2026-08-26T19:15:10.714652Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1115120640, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 13053774848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115907493888, + "reserved_bytes_current_delta": 25897730048, + "reserved_bytes_peak": 115907493888, + "reserved_bytes_peak_delta": 25897730048, + "segment_count_current": 1891, + "segment_count_current_delta": 210, + "segment_count_peak": 1891, + "segment_count_peak_delta": 210, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T191511400899Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.437042Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191511400899Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191511400899Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e5d6accd4f38112f48f13a0d7d7007b51b9fea3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191511400899Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddf8af55c253910a013c3408b526190254f78c7ed574af5e5efb2720d0bd7cb4 +size 1897393 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191511811328Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191511811328Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8a5c4c38ca119cd0c86ccdc5fc663e9e51e52e9a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191511811328Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345627136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113699828224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77345627136, + "active_bytes_peak": 113699828224, + "allocated_bytes_current": 77345627136, + "allocated_bytes_peak": 113699828224, + "device": 2, + "inactive_split_bytes_current": 1115120640, + "inactive_split_bytes_peak": 13751313920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115907493888, + "reserved_bytes_peak": 115907493888, + "segment_count_current": 1891, + "segment_count_peak": 1891, + "timestamp_utc": "2026-08-26T19:15:11.810910Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345627136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113699828224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77345627136, + "active_bytes_peak": 113699828224, + "allocated_bytes_current": 77345627136, + "allocated_bytes_peak": 113699828224, + "device": 2, + "inactive_split_bytes_current": 1115120640, + "inactive_split_bytes_peak": 13053774848, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 115907493888, + "reserved_bytes_peak": 115907493888, + "segment_count_current": 1891, + "segment_count_peak": 1891, + "timestamp_utc": "2026-08-26T19:15:11.441211Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1115120640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13751313920, + "inactive_split_bytes_peak_delta": 697539072, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115907493888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115907493888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1891, + "segment_count_current_delta": 0, + "segment_count_peak": 1891, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T191511811328Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.844423Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191511811328Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191511811328Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..43a4cce17fcd2924fde47df08da6395df12e38ff --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191511811328Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3de6fe7c1fc36a7d8164bdce95eda8d0b46da0ae3de3209c041de832d2cd20ff +size 1891991 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191512211825Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191512211825Z.json new file mode 100644 index 0000000000000000000000000000000000000000..83af81d2578d6c5d948f2d9c44bbbff8d151c638 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191512211825Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345627136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113699828224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77345627136, + "active_bytes_peak": 113699828224, + "allocated_bytes_current": 77345627136, + "allocated_bytes_peak": 113699828224, + "device": 2, + "inactive_split_bytes_current": 1115120640, + "inactive_split_bytes_peak": 13751313920, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115907493888, + "reserved_bytes_peak": 115907493888, + "segment_count_current": 1891, + "segment_count_peak": 1891, + "timestamp_utc": "2026-08-26T19:15:12.211433Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345627136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113699828224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77345627136, + "active_bytes_peak": 113699828224, + "allocated_bytes_current": 77345627136, + "allocated_bytes_peak": 113699828224, + "device": 2, + "inactive_split_bytes_current": 1115120640, + "inactive_split_bytes_peak": 13751313920, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 115907493888, + "reserved_bytes_peak": 115907493888, + "segment_count_current": 1891, + "segment_count_peak": 1891, + "timestamp_utc": "2026-08-26T19:15:11.848256Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1115120640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13751313920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115907493888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115907493888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1891, + "segment_count_current_delta": 0, + "segment_count_peak": 1891, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T191512211825Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.244081Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191512211825Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191512211825Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3d2aa5b096a1d22f9cc56250bcf965f60c298dc7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191512211825Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c88690ae24c6b5a8d995ca679c37a8e4f28ced78c5abc230843f9b2b15d5330 +size 1891991 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191512610986Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191512610986Z.json new file mode 100644 index 0000000000000000000000000000000000000000..18a5f899fe604458a60779674fce8aa1c16b6f0a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191512610986Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345627136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113699828224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77345627136, + "active_bytes_peak": 113699828224, + "allocated_bytes_current": 77345627136, + "allocated_bytes_peak": 113699828224, + "device": 2, + "inactive_split_bytes_current": 1115120640, + "inactive_split_bytes_peak": 13751313920, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115907493888, + "reserved_bytes_peak": 115907493888, + "segment_count_current": 1891, + "segment_count_peak": 1891, + "timestamp_utc": "2026-08-26T19:15:12.610589Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345627136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113699828224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77345627136, + "active_bytes_peak": 113699828224, + "allocated_bytes_current": 77345627136, + "allocated_bytes_peak": 113699828224, + "device": 2, + "inactive_split_bytes_current": 1115120640, + "inactive_split_bytes_peak": 13751313920, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 115907493888, + "reserved_bytes_peak": 115907493888, + "segment_count_current": 1891, + "segment_count_peak": 1891, + "timestamp_utc": "2026-08-26T19:15:12.247849Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1115120640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13751313920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115907493888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115907493888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1891, + "segment_count_current_delta": 0, + "segment_count_peak": 1891, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T191512610986Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.643283Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191512610986Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191512610986Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..604ca517b0b628f908ecf3222ee391475e845d9e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191512610986Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0b650155f29ca69728cc7a6a537b06d6ef5d723c71ac9c842a0ae7a96d9aa7e +size 1891994 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191510669687Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191510669687Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d7ee4e201c1bf7e089d527e5eb5775b0c780a272 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191510669687Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345983488, + "active_bytes_current_delta": 32825189376, + "active_bytes_peak": 82707948544, + "active_bytes_peak_delta": 38161988608, + "after": { + "active_bytes_current": 77345983488, + "active_bytes_peak": 82707948544, + "allocated_bytes_current": 77345983488, + "allocated_bytes_peak": 82707948544, + "device": 3, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_peak": 13385925120, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 89988792320, + "reserved_bytes_peak": 89988792320, + "segment_count_current": 1676, + "segment_count_peak": 1676, + "timestamp_utc": "2026-08-26T19:15:10.669188Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345983488, + "allocated_bytes_current_delta": 32825189376, + "allocated_bytes_peak": 82707948544, + "allocated_bytes_peak_delta": 38161988608, + "before": { + "active_bytes_current": 44520794112, + "active_bytes_peak": 44545959936, + "allocated_bytes_current": 44520794112, + "allocated_bytes_peak": 44545959936, + "device": 3, + "inactive_split_bytes_current": 421173248, + "inactive_split_bytes_peak": 1025153024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:14:41.061628Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_current_delta": 735534080, + "inactive_split_bytes_peak": 13385925120, + "inactive_split_bytes_peak_delta": 12360772096, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 89988792320, + "reserved_bytes_current_delta": 41737519104, + "reserved_bytes_peak": 89988792320, + "reserved_bytes_peak_delta": 41737519104, + "segment_count_current": 1676, + "segment_count_current_delta": 1055, + "segment_count_peak": 1676, + "segment_count_peak_delta": 1055, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T191510669687Z.pickle", + "timestamp_utc": "2026-08-26T19:15:10.710974Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191510669687Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191510669687Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4059594c24aa315909d8c6d5cf523d3752306c51 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191510669687Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f31c57c33a4df62e1348952ec62bb39f89cdae7060a7b049c75274da253f927d +size 2303382 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191511401005Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191511401005Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96813f3490ce6425c845d45c2db20e4ec922335c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191511401005Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345983488, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113620341248, + "active_bytes_peak_delta": 30912392704, + "after": { + "active_bytes_current": 77345983488, + "active_bytes_peak": 113620341248, + "allocated_bytes_current": 77345983488, + "allocated_bytes_peak": 113620341248, + "device": 3, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_peak": 13385925120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:11.400572Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345983488, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113620341248, + "allocated_bytes_peak_delta": 30912392704, + "before": { + "active_bytes_current": 77345983488, + "active_bytes_peak": 82707948544, + "allocated_bytes_current": 77345983488, + "allocated_bytes_peak": 82707948544, + "device": 3, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_peak": 13385925120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 89988792320, + "reserved_bytes_peak": 89988792320, + "segment_count_current": 1676, + "segment_count_peak": 1676, + "timestamp_utc": "2026-08-26T19:15:10.715791Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13385925120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 115834093568, + "reserved_bytes_current_delta": 25845301248, + "reserved_bytes_peak": 115834093568, + "reserved_bytes_peak_delta": 25845301248, + "segment_count_current": 1899, + "segment_count_current_delta": 223, + "segment_count_peak": 1899, + "segment_count_peak_delta": 223, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T191511401005Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.437902Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191511401005Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191511401005Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1a55aa9ff179d7399a8bcc58ce8e5e6b5e797dc2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191511401005Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fab5e19a1b2146310c6d4a088bb26c4c9b2f262bec61c6f198b7b2456b07d8f +size 1897699 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191511811341Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191511811341Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e93af788587811f7dc25f75fbc42eb3ea6c7b13d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191511811341Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345983488, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113661862912, + "active_bytes_peak_delta": 41521664, + "after": { + "active_bytes_current": 77345983488, + "active_bytes_peak": 113661862912, + "allocated_bytes_current": 77345983488, + "allocated_bytes_peak": 113661862912, + "device": 3, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_peak": 13385925120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:11.810926Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345983488, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113661862912, + "allocated_bytes_peak_delta": 41521664, + "before": { + "active_bytes_current": 77345983488, + "active_bytes_peak": 113620341248, + "allocated_bytes_current": 77345983488, + "allocated_bytes_peak": 113620341248, + "device": 3, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_peak": 13385925120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:11.442036Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13385925120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 115834093568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115834093568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1899, + "segment_count_current_delta": 0, + "segment_count_peak": 1899, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T191511811341Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.844162Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191511811341Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191511811341Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8e1f95f63c47d07e56889d2e0176ff40f7e7f210 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191511811341Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e218713dc97d922c24d3d748525a2f67263bb36c48dce2991491fd0f63421f4 +size 1890702 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191512211852Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191512211852Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7fc03cb0c16b4a9b14bf2c206952753b9e97e73f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191512211852Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345983488, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113661862912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77345983488, + "active_bytes_peak": 113661862912, + "allocated_bytes_current": 77345983488, + "allocated_bytes_peak": 113661862912, + "device": 3, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_peak": 13385925120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:12.211449Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345983488, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113661862912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77345983488, + "active_bytes_peak": 113661862912, + "allocated_bytes_current": 77345983488, + "allocated_bytes_peak": 113661862912, + "device": 3, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_peak": 13385925120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:11.847897Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13385925120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 115834093568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115834093568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1899, + "segment_count_current_delta": 0, + "segment_count_peak": 1899, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T191512211852Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.244231Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191512211852Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191512211852Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f67fbbd8dc609e3d4a7bdf6e08ce8bb634181fa --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191512211852Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd816af4b8063ed096fe117516e590ae11f9666a71149d18e551a4084c3475b2 +size 1890252 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191512610994Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191512610994Z.json new file mode 100644 index 0000000000000000000000000000000000000000..605471462e6f7cf9bfaf4a849ee96bb767735b0a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191512610994Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345983488, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113661862912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77345983488, + "active_bytes_peak": 113661862912, + "allocated_bytes_current": 77345983488, + "allocated_bytes_peak": 113661862912, + "device": 3, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_peak": 13385925120, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:12.610595Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345983488, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113661862912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77345983488, + "active_bytes_peak": 113661862912, + "allocated_bytes_current": 77345983488, + "allocated_bytes_peak": 113661862912, + "device": 3, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_peak": 13385925120, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1899, + "segment_count_peak": 1899, + "timestamp_utc": "2026-08-26T19:15:12.247872Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 1156707328, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13385925120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 115834093568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115834093568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1899, + "segment_count_current_delta": 0, + "segment_count_peak": 1899, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T191512610994Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.643352Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191512610994Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191512610994Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a4c4af43fcb2ec75c889b90e2060575327636c3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191512610994Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b112b0cce96aadc5b3d8ea630c28e16970d9ad373336096d1f22f90e97dad80 +size 1890252 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191510669682Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191510669682Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3cb069bfb8bd93fb11f3737cc6806589dbe5f8eb --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191510669682Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345752576, + "active_bytes_current_delta": 32824958464, + "active_bytes_peak": 82752384512, + "active_bytes_peak_delta": 38206424576, + "after": { + "active_bytes_current": 77345752576, + "active_bytes_peak": 82752384512, + "allocated_bytes_current": 77345752576, + "allocated_bytes_peak": 82752384512, + "device": 4, + "inactive_split_bytes_current": 1389722112, + "inactive_split_bytes_peak": 13874745344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 90190118912, + "reserved_bytes_peak": 90190118912, + "segment_count_current": 1696, + "segment_count_peak": 1696, + "timestamp_utc": "2026-08-26T19:15:10.669160Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345752576, + "allocated_bytes_current_delta": 32824958464, + "allocated_bytes_peak": 82752384512, + "allocated_bytes_peak_delta": 38206424576, + "before": { + "active_bytes_current": 44520794112, + "active_bytes_peak": 44545959936, + "allocated_bytes_current": 44520794112, + "allocated_bytes_peak": 44545959936, + "device": 4, + "inactive_split_bytes_current": 270178304, + "inactive_split_bytes_peak": 874158080, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 47949283328, + "reserved_bytes_peak": 47949283328, + "segment_count_current": 615, + "segment_count_peak": 615, + "timestamp_utc": "2026-08-26T19:14:41.061444Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1389722112, + "inactive_split_bytes_current_delta": 1119543808, + "inactive_split_bytes_peak": 13874745344, + "inactive_split_bytes_peak_delta": 13000587264, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 90190118912, + "reserved_bytes_current_delta": 42240835584, + "reserved_bytes_peak": 90190118912, + "reserved_bytes_peak_delta": 42240835584, + "segment_count_current": 1696, + "segment_count_current_delta": 1081, + "segment_count_peak": 1696, + "segment_count_peak_delta": 1081, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T191510669682Z.pickle", + "timestamp_utc": "2026-08-26T19:15:10.709392Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191510669682Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191510669682Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8cf36dc6f6c6372c238eefe8aae2c83a83f78fbf --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191510669682Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f212c5d27e033a70b41272d84a24838d9c181b5cc7804598fa0eaf86fb23b42 +size 2313836 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191511400950Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191511400950Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3aba9e915426a9911df528df082e2835a6ff7923 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191511400950Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345752576, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113709009408, + "active_bytes_peak_delta": 30956624896, + "after": { + "active_bytes_current": 77345752576, + "active_bytes_peak": 113709009408, + "allocated_bytes_current": 77345752576, + "allocated_bytes_peak": 113709009408, + "device": 4, + "inactive_split_bytes_current": 1389722112, + "inactive_split_bytes_peak": 13874745344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115913785344, + "reserved_bytes_peak": 115913785344, + "segment_count_current": 1916, + "segment_count_peak": 1916, + "timestamp_utc": "2026-08-26T19:15:11.400519Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345752576, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113709009408, + "allocated_bytes_peak_delta": 30956624896, + "before": { + "active_bytes_current": 77345752576, + "active_bytes_peak": 82752384512, + "allocated_bytes_current": 77345752576, + "allocated_bytes_peak": 82752384512, + "device": 4, + "inactive_split_bytes_current": 1389722112, + "inactive_split_bytes_peak": 13874745344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 90190118912, + "reserved_bytes_peak": 90190118912, + "segment_count_current": 1696, + "segment_count_peak": 1696, + "timestamp_utc": "2026-08-26T19:15:10.714114Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1389722112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13874745344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115913785344, + "reserved_bytes_current_delta": 25723666432, + "reserved_bytes_peak": 115913785344, + "reserved_bytes_peak_delta": 25723666432, + "segment_count_current": 1916, + "segment_count_current_delta": 220, + "segment_count_peak": 1916, + "segment_count_peak_delta": 220, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T191511400950Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.436414Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191511400950Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191511400950Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a234ad449b9fa7d16faeb001d915622ff4c01f07 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191511400950Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80999b565c805cf1c6ffba649eb275daedb083cf6dced33ce6952a914dae337c +size 1904241 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191511811326Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191511811326Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3bd0643422d018c4bcd4354ac5a6746386952b40 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191511811326Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345752576, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113709009408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77345752576, + "active_bytes_peak": 113709009408, + "allocated_bytes_current": 77345752576, + "allocated_bytes_peak": 113709009408, + "device": 4, + "inactive_split_bytes_current": 1389722112, + "inactive_split_bytes_peak": 14235176960, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115913785344, + "reserved_bytes_peak": 115913785344, + "segment_count_current": 1916, + "segment_count_peak": 1916, + "timestamp_utc": "2026-08-26T19:15:11.810913Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345752576, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113709009408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77345752576, + "active_bytes_peak": 113709009408, + "allocated_bytes_current": 77345752576, + "allocated_bytes_peak": 113709009408, + "device": 4, + "inactive_split_bytes_current": 1389722112, + "inactive_split_bytes_peak": 13874745344, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115913785344, + "reserved_bytes_peak": 115913785344, + "segment_count_current": 1916, + "segment_count_peak": 1916, + "timestamp_utc": "2026-08-26T19:15:11.440440Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1389722112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14235176960, + "inactive_split_bytes_peak_delta": 360431616, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115913785344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115913785344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1916, + "segment_count_current_delta": 0, + "segment_count_peak": 1916, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T191511811326Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.843065Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191511811326Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191511811326Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0226c64f503ef671f3f5d93b0d50607835cf5955 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191511811326Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65f8a66333706bb9ac5d5adde1fc1fa3cc13b51dbfab87e66b0f2e8e29cfd482 +size 1895232 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191512211840Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191512211840Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d492249c645538233d4942c0b4999c80eb70721 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191512211840Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345752576, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113709009408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77345752576, + "active_bytes_peak": 113709009408, + "allocated_bytes_current": 77345752576, + "allocated_bytes_peak": 113709009408, + "device": 4, + "inactive_split_bytes_current": 1389722112, + "inactive_split_bytes_peak": 14235176960, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115913785344, + "reserved_bytes_peak": 115913785344, + "segment_count_current": 1916, + "segment_count_peak": 1916, + "timestamp_utc": "2026-08-26T19:15:12.211436Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345752576, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113709009408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77345752576, + "active_bytes_peak": 113709009408, + "allocated_bytes_current": 77345752576, + "allocated_bytes_peak": 113709009408, + "device": 4, + "inactive_split_bytes_current": 1389722112, + "inactive_split_bytes_peak": 14235176960, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115913785344, + "reserved_bytes_peak": 115913785344, + "segment_count_current": 1916, + "segment_count_peak": 1916, + "timestamp_utc": "2026-08-26T19:15:11.846788Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1389722112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14235176960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115913785344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115913785344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1916, + "segment_count_current_delta": 0, + "segment_count_peak": 1916, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T191512211840Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.243170Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191512211840Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191512211840Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f778b7863ee52f7cde8af2a84c01708341d6e7fe --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191512211840Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00b430b851e502e466150dcf4a9ab111b495be6095dad7ce79b198222e1b610c +size 1895023 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191512611011Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191512611011Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3d12323310b4a0f197a844a762dd5a89698589b5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191512611011Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77345752576, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113709009408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77345752576, + "active_bytes_peak": 113709009408, + "allocated_bytes_current": 77345752576, + "allocated_bytes_peak": 113709009408, + "device": 4, + "inactive_split_bytes_current": 1391819264, + "inactive_split_bytes_peak": 14235176960, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115913785344, + "reserved_bytes_peak": 115913785344, + "segment_count_current": 1916, + "segment_count_peak": 1916, + "timestamp_utc": "2026-08-26T19:15:12.610602Z", + "world_size": 8 + }, + "allocated_bytes_current": 77345752576, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113709009408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77345752576, + "active_bytes_peak": 113709009408, + "allocated_bytes_current": 77345752576, + "allocated_bytes_peak": 113709009408, + "device": 4, + "inactive_split_bytes_current": 1389722112, + "inactive_split_bytes_peak": 14235176960, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115913785344, + "reserved_bytes_peak": 115913785344, + "segment_count_current": 1916, + "segment_count_peak": 1916, + "timestamp_utc": "2026-08-26T19:15:12.246806Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 1391819264, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 14235176960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115913785344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115913785344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1916, + "segment_count_current_delta": 0, + "segment_count_peak": 1916, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T191512611011Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.643002Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191512611011Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191512611011Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..590baca3753fbaf8996fd66f8790e497ad4d1b2f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191512611011Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45e4d05392e42fedf5494e35fcaa45271bd5192b17d2bee490b04e5a43e3f587 +size 1895055 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191510669588Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191510669588Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2b36e3aa2d1927451edb1d6201ce0c69527c2d74 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191510669588Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77343242752, + "active_bytes_current_delta": 32822448640, + "active_bytes_peak": 82714992640, + "active_bytes_peak_delta": 38169032704, + "after": { + "active_bytes_current": 77343242752, + "active_bytes_peak": 82714992640, + "allocated_bytes_current": 77343242752, + "allocated_bytes_peak": 82714992640, + "device": 5, + "inactive_split_bytes_current": 1188808192, + "inactive_split_bytes_peak": 13422829568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 90060095488, + "reserved_bytes_peak": 90060095488, + "segment_count_current": 1682, + "segment_count_peak": 1682, + "timestamp_utc": "2026-08-26T19:15:10.669036Z", + "world_size": 8 + }, + "allocated_bytes_current": 77343242752, + "allocated_bytes_current_delta": 32822448640, + "allocated_bytes_peak": 82714992640, + "allocated_bytes_peak_delta": 38169032704, + "before": { + "active_bytes_current": 44520794112, + "active_bytes_peak": 44545959936, + "allocated_bytes_current": 44520794112, + "allocated_bytes_peak": 44545959936, + "device": 5, + "inactive_split_bytes_current": 421173248, + "inactive_split_bytes_peak": 1025153024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:14:41.061599Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1188808192, + "inactive_split_bytes_current_delta": 767634944, + "inactive_split_bytes_peak": 13422829568, + "inactive_split_bytes_peak_delta": 12397676544, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 90060095488, + "reserved_bytes_current_delta": 41808822272, + "reserved_bytes_peak": 90060095488, + "reserved_bytes_peak_delta": 41808822272, + "segment_count_current": 1682, + "segment_count_current_delta": 1061, + "segment_count_peak": 1682, + "segment_count_peak_delta": 1061, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T191510669588Z.pickle", + "timestamp_utc": "2026-08-26T19:15:10.709549Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191510669588Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191510669588Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e05f04dc4b1ae9a3778e6dfd00865d865dbfd0f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191510669588Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:692813114d73390bed6a7f723a479660e6d3043ead14bca85d3e462b07536106 +size 2304448 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191511400807Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191511400807Z.json new file mode 100644 index 0000000000000000000000000000000000000000..14e8405d49a50cca584649ce7dd0b2b427706262 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191511400807Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77343242752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113578297856, + "active_bytes_peak_delta": 30863305216, + "after": { + "active_bytes_current": 77343242752, + "active_bytes_peak": 113578297856, + "allocated_bytes_current": 77343242752, + "allocated_bytes_peak": 113578297856, + "device": 5, + "inactive_split_bytes_current": 1188808192, + "inactive_split_bytes_peak": 13422829568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115815219200, + "reserved_bytes_peak": 115815219200, + "segment_count_current": 1891, + "segment_count_peak": 1891, + "timestamp_utc": "2026-08-26T19:15:11.400392Z", + "world_size": 8 + }, + "allocated_bytes_current": 77343242752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113578297856, + "allocated_bytes_peak_delta": 30863305216, + "before": { + "active_bytes_current": 77343242752, + "active_bytes_peak": 82714992640, + "allocated_bytes_current": 77343242752, + "allocated_bytes_peak": 82714992640, + "device": 5, + "inactive_split_bytes_current": 1188808192, + "inactive_split_bytes_peak": 13422829568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 90060095488, + "reserved_bytes_peak": 90060095488, + "segment_count_current": 1682, + "segment_count_peak": 1682, + "timestamp_utc": "2026-08-26T19:15:10.714077Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1188808192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13422829568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115815219200, + "reserved_bytes_current_delta": 25755123712, + "reserved_bytes_peak": 115815219200, + "reserved_bytes_peak_delta": 25755123712, + "segment_count_current": 1891, + "segment_count_current_delta": 209, + "segment_count_peak": 1891, + "segment_count_peak_delta": 209, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T191511400807Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.435110Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191511400807Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191511400807Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5c0fe1cef82cbeed01394a07aae351db6842069b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191511400807Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51bb2c3771674fd2b7b604e47c603ec733a10f46d335c558c1430ec77efbf2fa +size 1897044 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191511811184Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191511811184Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e51ea1f574668ffbc32f7c5421ed9f1d09d7ca4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191511811184Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77343242752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113592105472, + "active_bytes_peak_delta": 13807616, + "after": { + "active_bytes_current": 77343242752, + "active_bytes_peak": 113592105472, + "allocated_bytes_current": 77343242752, + "allocated_bytes_peak": 113592105472, + "device": 5, + "inactive_split_bytes_current": 1190905344, + "inactive_split_bytes_peak": 14358206976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115817316352, + "reserved_bytes_peak": 115817316352, + "segment_count_current": 1892, + "segment_count_peak": 1892, + "timestamp_utc": "2026-08-26T19:15:11.810796Z", + "world_size": 8 + }, + "allocated_bytes_current": 77343242752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113592105472, + "allocated_bytes_peak_delta": 13807616, + "before": { + "active_bytes_current": 77343242752, + "active_bytes_peak": 113578297856, + "allocated_bytes_current": 77343242752, + "allocated_bytes_peak": 113578297856, + "device": 5, + "inactive_split_bytes_current": 1188808192, + "inactive_split_bytes_peak": 13422829568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 115815219200, + "reserved_bytes_peak": 115815219200, + "segment_count_current": 1891, + "segment_count_peak": 1891, + "timestamp_utc": "2026-08-26T19:15:11.438955Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1190905344, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 14358206976, + "inactive_split_bytes_peak_delta": 935377408, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115817316352, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 115817316352, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 1892, + "segment_count_current_delta": 1, + "segment_count_peak": 1892, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T191511811184Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.842752Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191511811184Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191511811184Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cc95685109ec2ab1fd6f1b3af4dd9e60534ec558 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191511811184Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c726d84bfcaf694ac3cae346ac791cc8c41b3a8c2761c6fff4c70769f1c5f6eb +size 1891129 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191512211698Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191512211698Z.json new file mode 100644 index 0000000000000000000000000000000000000000..41becf84c97579fb0ecd0de680021a9149fc2391 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191512211698Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77343242752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113629871616, + "active_bytes_peak_delta": 37766144, + "after": { + "active_bytes_current": 77343242752, + "active_bytes_peak": 113629871616, + "allocated_bytes_current": 77343242752, + "allocated_bytes_peak": 113629871616, + "device": 5, + "inactive_split_bytes_current": 1188808192, + "inactive_split_bytes_peak": 14358206976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115817316352, + "reserved_bytes_peak": 115817316352, + "segment_count_current": 1892, + "segment_count_peak": 1892, + "timestamp_utc": "2026-08-26T19:15:12.211309Z", + "world_size": 8 + }, + "allocated_bytes_current": 77343242752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113629871616, + "allocated_bytes_peak_delta": 37766144, + "before": { + "active_bytes_current": 77343242752, + "active_bytes_peak": 113592105472, + "allocated_bytes_current": 77343242752, + "allocated_bytes_peak": 113592105472, + "device": 5, + "inactive_split_bytes_current": 1190905344, + "inactive_split_bytes_peak": 14358206976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 115817316352, + "reserved_bytes_peak": 115817316352, + "segment_count_current": 1892, + "segment_count_peak": 1892, + "timestamp_utc": "2026-08-26T19:15:11.846356Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1188808192, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 14358206976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115817316352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115817316352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1892, + "segment_count_current_delta": 0, + "segment_count_peak": 1892, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T191512211698Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.242075Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191512211698Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191512211698Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d641669ae299a050fd86a0b8d4b9eda2c673c3d8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191512211698Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95220fc987048db6df6cfab8ad7740f7d39be5cf39d4ac8e7bc97b8c62d4d677 +size 1890839 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191512610886Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191512610886Z.json new file mode 100644 index 0000000000000000000000000000000000000000..acd19d2c989f7e88a367566b46485ec154dcc76b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191512610886Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77343242752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113629871616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77343242752, + "active_bytes_peak": 113629871616, + "allocated_bytes_current": 77343242752, + "allocated_bytes_peak": 113629871616, + "device": 5, + "inactive_split_bytes_current": 1188808192, + "inactive_split_bytes_peak": 14358206976, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115817316352, + "reserved_bytes_peak": 115817316352, + "segment_count_current": 1892, + "segment_count_peak": 1892, + "timestamp_utc": "2026-08-26T19:15:12.610492Z", + "world_size": 8 + }, + "allocated_bytes_current": 77343242752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113629871616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77343242752, + "active_bytes_peak": 113629871616, + "allocated_bytes_current": 77343242752, + "allocated_bytes_peak": 113629871616, + "device": 5, + "inactive_split_bytes_current": 1188808192, + "inactive_split_bytes_peak": 14358206976, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 115817316352, + "reserved_bytes_peak": 115817316352, + "segment_count_current": 1892, + "segment_count_peak": 1892, + "timestamp_utc": "2026-08-26T19:15:12.245563Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 1188808192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14358206976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115817316352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115817316352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1892, + "segment_count_current_delta": 0, + "segment_count_peak": 1892, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T191512610886Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.641871Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191512610886Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191512610886Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..22b38559b5ab9ae880c1d169ee1c06d26cc62e4c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191512610886Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16f94db1dba250603efb744cc2acdc7fd5094746f31d66a8d543c3b40892eef9 +size 1890548 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191510669573Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191510669573Z.json new file mode 100644 index 0000000000000000000000000000000000000000..55aa27f99fcec6b0319ea67a57f1642cd2bd86e5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191510669573Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77341434880, + "active_bytes_current_delta": 32820640768, + "active_bytes_peak": 82746957824, + "active_bytes_peak_delta": 38200997888, + "after": { + "active_bytes_current": 77341434880, + "active_bytes_peak": 82746957824, + "allocated_bytes_current": 77341434880, + "allocated_bytes_peak": 82746957824, + "device": 6, + "inactive_split_bytes_current": 1121410048, + "inactive_split_bytes_peak": 15735567872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 90053804032, + "reserved_bytes_peak": 90053804032, + "segment_count_current": 1684, + "segment_count_peak": 1684, + "timestamp_utc": "2026-08-26T19:15:10.669039Z", + "world_size": 8 + }, + "allocated_bytes_current": 77341434880, + "allocated_bytes_current_delta": 32820640768, + "allocated_bytes_peak": 82746957824, + "allocated_bytes_peak_delta": 38200997888, + "before": { + "active_bytes_current": 44520794112, + "active_bytes_peak": 44545959936, + "allocated_bytes_current": 44520794112, + "allocated_bytes_peak": 44545959936, + "device": 6, + "inactive_split_bytes_current": 421173248, + "inactive_split_bytes_peak": 1025153024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:14:41.061565Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1121410048, + "inactive_split_bytes_current_delta": 700236800, + "inactive_split_bytes_peak": 15735567872, + "inactive_split_bytes_peak_delta": 14710414848, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 90053804032, + "reserved_bytes_current_delta": 41802530816, + "reserved_bytes_peak": 90053804032, + "reserved_bytes_peak_delta": 41802530816, + "segment_count_current": 1684, + "segment_count_current_delta": 1063, + "segment_count_peak": 1684, + "segment_count_peak_delta": 1063, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T191510669573Z.pickle", + "timestamp_utc": "2026-08-26T19:15:10.709313Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191510669573Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191510669573Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4d2cf5b02814ee9a8e8428b640ee34ad8d8706f8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191510669573Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:848932be93ab3a634558ed4524bb2a41c135e2b33c6fe658b8646367d20aa3b9 +size 2304680 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191511400795Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191511400795Z.json new file mode 100644 index 0000000000000000000000000000000000000000..719f6b1340369249d5fa35db489b2c64d7a65dc5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191511400795Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77341434880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113612270592, + "active_bytes_peak_delta": 30865312768, + "after": { + "active_bytes_current": 77341434880, + "active_bytes_peak": 113612270592, + "allocated_bytes_current": 77341434880, + "allocated_bytes_peak": 113612270592, + "device": 6, + "inactive_split_bytes_current": 1121410048, + "inactive_split_bytes_peak": 15735567872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 115737624576, + "reserved_bytes_peak": 115737624576, + "segment_count_current": 1897, + "segment_count_peak": 1897, + "timestamp_utc": "2026-08-26T19:15:11.400384Z", + "world_size": 8 + }, + "allocated_bytes_current": 77341434880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113612270592, + "allocated_bytes_peak_delta": 30865312768, + "before": { + "active_bytes_current": 77341434880, + "active_bytes_peak": 82746957824, + "allocated_bytes_current": 77341434880, + "allocated_bytes_peak": 82746957824, + "device": 6, + "inactive_split_bytes_current": 1121410048, + "inactive_split_bytes_peak": 15735567872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 90053804032, + "reserved_bytes_peak": 90053804032, + "segment_count_current": 1684, + "segment_count_peak": 1684, + "timestamp_utc": "2026-08-26T19:15:10.713942Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1121410048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15735567872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 115737624576, + "reserved_bytes_current_delta": 25683820544, + "reserved_bytes_peak": 115737624576, + "reserved_bytes_peak_delta": 25683820544, + "segment_count_current": 1897, + "segment_count_current_delta": 213, + "segment_count_peak": 1897, + "segment_count_peak_delta": 213, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T191511400795Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.436608Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191511400795Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191511400795Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0dd755898d6ec466cb40258f59ce3c29a2027a07 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191511400795Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d8343367d4981bd195274034a633ff7715e70f2b4b0b2e6e65bbd5daa262bfc +size 1911405 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191511811209Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191511811209Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76241e7ea04ca357346470888d6f9cb954bf9f40 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191511811209Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77341434880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113612270592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77341434880, + "active_bytes_peak": 113612270592, + "allocated_bytes_current": 77341434880, + "allocated_bytes_peak": 113612270592, + "device": 6, + "inactive_split_bytes_current": 1121410048, + "inactive_split_bytes_peak": 15735567872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 115737624576, + "reserved_bytes_peak": 115737624576, + "segment_count_current": 1897, + "segment_count_peak": 1897, + "timestamp_utc": "2026-08-26T19:15:11.810810Z", + "world_size": 8 + }, + "allocated_bytes_current": 77341434880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113612270592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77341434880, + "active_bytes_peak": 113612270592, + "allocated_bytes_current": 77341434880, + "allocated_bytes_peak": 113612270592, + "device": 6, + "inactive_split_bytes_current": 1121410048, + "inactive_split_bytes_peak": 15735567872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 115737624576, + "reserved_bytes_peak": 115737624576, + "segment_count_current": 1897, + "segment_count_peak": 1897, + "timestamp_utc": "2026-08-26T19:15:11.440703Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1121410048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15735567872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 115737624576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115737624576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1897, + "segment_count_current_delta": 0, + "segment_count_peak": 1897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T191511811209Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.843906Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191511811209Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191511811209Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..733289591b063abac2b27ee4cf001d996ba99e75 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191511811209Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61bc8a9a426d1b91e4da52928f5f89e5f32a7d0e683399063b4d34455fc333fb +size 1906004 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191512211715Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191512211715Z.json new file mode 100644 index 0000000000000000000000000000000000000000..87055bd998f1ad92b4b52ea95c13fed773cd31bb --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191512211715Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77341434880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113612270592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77341434880, + "active_bytes_peak": 113612270592, + "allocated_bytes_current": 77341434880, + "allocated_bytes_peak": 113612270592, + "device": 6, + "inactive_split_bytes_current": 1121410048, + "inactive_split_bytes_peak": 15735567872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 115737624576, + "reserved_bytes_peak": 115737624576, + "segment_count_current": 1897, + "segment_count_peak": 1897, + "timestamp_utc": "2026-08-26T19:15:12.211324Z", + "world_size": 8 + }, + "allocated_bytes_current": 77341434880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113612270592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77341434880, + "active_bytes_peak": 113612270592, + "allocated_bytes_current": 77341434880, + "allocated_bytes_peak": 113612270592, + "device": 6, + "inactive_split_bytes_current": 1121410048, + "inactive_split_bytes_peak": 15735567872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 115737624576, + "reserved_bytes_peak": 115737624576, + "segment_count_current": 1897, + "segment_count_peak": 1897, + "timestamp_utc": "2026-08-26T19:15:11.847661Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1121410048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15735567872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 115737624576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115737624576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1897, + "segment_count_current_delta": 0, + "segment_count_peak": 1897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T191512211715Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.243874Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191512211715Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191512211715Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..18ff28e009bb57accf6e42c1799da2c1482cc793 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191512211715Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c54b51b0a72774b004cfe465e9844a02c14e7eb602361adaa9209901b620abc2 +size 1905443 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191512610858Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191512610858Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cc39314af226127eb0cac9bef2c78148e7834911 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191512610858Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77341434880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113612270592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77341434880, + "active_bytes_peak": 113612270592, + "allocated_bytes_current": 77341434880, + "allocated_bytes_peak": 113612270592, + "device": 6, + "inactive_split_bytes_current": 1123507200, + "inactive_split_bytes_peak": 15735567872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 115737624576, + "reserved_bytes_peak": 115737624576, + "segment_count_current": 1897, + "segment_count_peak": 1897, + "timestamp_utc": "2026-08-26T19:15:12.610473Z", + "world_size": 8 + }, + "allocated_bytes_current": 77341434880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113612270592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77341434880, + "active_bytes_peak": 113612270592, + "allocated_bytes_current": 77341434880, + "allocated_bytes_peak": 113612270592, + "device": 6, + "inactive_split_bytes_current": 1121410048, + "inactive_split_bytes_peak": 15735567872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 115737624576, + "reserved_bytes_peak": 115737624576, + "segment_count_current": 1897, + "segment_count_peak": 1897, + "timestamp_utc": "2026-08-26T19:15:12.247496Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 1123507200, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 15735567872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 115737624576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115737624576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1897, + "segment_count_current_delta": 0, + "segment_count_peak": 1897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T191512610858Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.642840Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191512610858Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191512610858Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..310817d242589a408d108199d837e69866db6ba1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191512610858Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afd5359e603c86b22d2e299187511813235269b9394b91220a660e3d211bbe88 +size 1905480 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191510669588Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191510669588Z.json new file mode 100644 index 0000000000000000000000000000000000000000..765768260922b0a53541017e57e79195fc9a32d3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191510669588Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77343633920, + "active_bytes_current_delta": 32822839808, + "active_bytes_peak": 82781058048, + "active_bytes_peak_delta": 38235098112, + "after": { + "active_bytes_current": 77343633920, + "active_bytes_peak": 82781058048, + "allocated_bytes_current": 77343633920, + "allocated_bytes_peak": 82781058048, + "device": 7, + "inactive_split_bytes_current": 1098239488, + "inactive_split_bytes_peak": 13979639808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 90005569536, + "reserved_bytes_peak": 90005569536, + "segment_count_current": 1684, + "segment_count_peak": 1684, + "timestamp_utc": "2026-08-26T19:15:10.669056Z", + "world_size": 8 + }, + "allocated_bytes_current": 77343633920, + "allocated_bytes_current_delta": 32822839808, + "allocated_bytes_peak": 82781058048, + "allocated_bytes_peak_delta": 38235098112, + "before": { + "active_bytes_current": 44520794112, + "active_bytes_peak": 44545959936, + "allocated_bytes_current": 44520794112, + "allocated_bytes_peak": 44545959936, + "device": 7, + "inactive_split_bytes_current": 421173248, + "inactive_split_bytes_peak": 1025153024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 48251273216, + "reserved_bytes_peak": 48251273216, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-26T19:14:41.061745Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1098239488, + "inactive_split_bytes_current_delta": 677066240, + "inactive_split_bytes_peak": 13979639808, + "inactive_split_bytes_peak_delta": 12954486784, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 90005569536, + "reserved_bytes_current_delta": 41754296320, + "reserved_bytes_peak": 90005569536, + "reserved_bytes_peak_delta": 41754296320, + "segment_count_current": 1684, + "segment_count_current_delta": 1063, + "segment_count_peak": 1684, + "segment_count_peak_delta": 1063, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T191510669588Z.pickle", + "timestamp_utc": "2026-08-26T19:15:10.709999Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191510669588Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191510669588Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f8f570c57a9c53476d28ff89c39177ae834ef877 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191510669588Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c7e3eec92b84afbc41acfb5295d118c87856e659b262687430c2fbd9ce891da +size 2304695 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191511400805Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191511400805Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e7fd3e64668f3a4e98ad64cd2694bfb79576e1c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191511400805Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77343633920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113625560064, + "active_bytes_peak_delta": 30844502016, + "after": { + "active_bytes_current": 77343633920, + "active_bytes_peak": 113625560064, + "allocated_bytes_current": 77343633920, + "allocated_bytes_peak": 113625560064, + "device": 7, + "inactive_split_bytes_current": 1100336640, + "inactive_split_bytes_peak": 13979639808, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1896, + "segment_count_peak": 1896, + "timestamp_utc": "2026-08-26T19:15:11.400401Z", + "world_size": 8 + }, + "allocated_bytes_current": 77343633920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113625560064, + "allocated_bytes_peak_delta": 30844502016, + "before": { + "active_bytes_current": 77343633920, + "active_bytes_peak": 82781058048, + "allocated_bytes_current": 77343633920, + "allocated_bytes_peak": 82781058048, + "device": 7, + "inactive_split_bytes_current": 1098239488, + "inactive_split_bytes_peak": 13979639808, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 90005569536, + "reserved_bytes_peak": 90005569536, + "segment_count_current": 1684, + "segment_count_peak": 1684, + "timestamp_utc": "2026-08-26T19:15:10.714762Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1100336640, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 13979639808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115834093568, + "reserved_bytes_current_delta": 25828524032, + "reserved_bytes_peak": 115834093568, + "reserved_bytes_peak_delta": 25828524032, + "segment_count_current": 1896, + "segment_count_current_delta": 212, + "segment_count_peak": 1896, + "segment_count_peak_delta": 212, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T191511400805Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.436368Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191511400805Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191511400805Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..48329f9c08afe7d3d03d38b40d5e8f3b59a6d55f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191511400805Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ada3f5c7b19c688deb40626a491b004ef545b6ff485c3241462c3ea93d74ef5 +size 1898306 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191511811244Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191511811244Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f395c681f1d68ebc2e91ce7b71f1e6cc31636645 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191511811244Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77343633920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113713497600, + "active_bytes_peak_delta": 87937536, + "after": { + "active_bytes_current": 77343633920, + "active_bytes_peak": 113713497600, + "allocated_bytes_current": 77343633920, + "allocated_bytes_peak": 113713497600, + "device": 7, + "inactive_split_bytes_current": 1098239488, + "inactive_split_bytes_peak": 14218493952, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1896, + "segment_count_peak": 1896, + "timestamp_utc": "2026-08-26T19:15:11.810845Z", + "world_size": 8 + }, + "allocated_bytes_current": 77343633920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113713497600, + "allocated_bytes_peak_delta": 87937536, + "before": { + "active_bytes_current": 77343633920, + "active_bytes_peak": 113625560064, + "allocated_bytes_current": 77343633920, + "allocated_bytes_peak": 113625560064, + "device": 7, + "inactive_split_bytes_current": 1100336640, + "inactive_split_bytes_peak": 13979639808, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1896, + "segment_count_peak": 1896, + "timestamp_utc": "2026-08-26T19:15:11.440353Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1098239488, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 14218493952, + "inactive_split_bytes_peak_delta": 238854144, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115834093568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115834093568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1896, + "segment_count_current_delta": 0, + "segment_count_peak": 1896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T191511811244Z.pickle", + "timestamp_utc": "2026-08-26T19:15:11.843916Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191511811244Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191511811244Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..48a9e8094dfc06dec44df3cb9510c3850b734621 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191511811244Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3447220514e5590a1d3745b76395a70bb45511c544f81fda4996a609728a82fc +size 1893070 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191512211731Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191512211731Z.json new file mode 100644 index 0000000000000000000000000000000000000000..02e5df7e412d7bb48edb901296cd0c7f6252249f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191512211731Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77343633920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113713497600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77343633920, + "active_bytes_peak": 113713497600, + "allocated_bytes_current": 77343633920, + "allocated_bytes_peak": 113713497600, + "device": 7, + "inactive_split_bytes_current": 1100336640, + "inactive_split_bytes_peak": 14218493952, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1896, + "segment_count_peak": 1896, + "timestamp_utc": "2026-08-26T19:15:12.211335Z", + "world_size": 8 + }, + "allocated_bytes_current": 77343633920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113713497600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77343633920, + "active_bytes_peak": 113713497600, + "allocated_bytes_current": 77343633920, + "allocated_bytes_peak": 113713497600, + "device": 7, + "inactive_split_bytes_current": 1098239488, + "inactive_split_bytes_peak": 14218493952, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1896, + "segment_count_peak": 1896, + "timestamp_utc": "2026-08-26T19:15:11.847579Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1100336640, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 14218493952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115834093568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115834093568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1896, + "segment_count_current_delta": 0, + "segment_count_peak": 1896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T191512211731Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.243378Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191512211731Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191512211731Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..59d0bcbd61d71a6979de964a9bd2211fd03c05bf --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191512211731Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b67cc3f632eb1581dd1ac52a40c19e7b8aff0849a3fad8f3d4230dc20444d0bb +size 1892925 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191512610909Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191512610909Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dad96722b8b2864515bed2da39d0143e7581f08e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191512610909Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 77343633920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113713497600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 77343633920, + "active_bytes_peak": 113713497600, + "allocated_bytes_current": 77343633920, + "allocated_bytes_peak": 113713497600, + "device": 7, + "inactive_split_bytes_current": 1098239488, + "inactive_split_bytes_peak": 14218493952, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1896, + "segment_count_peak": 1896, + "timestamp_utc": "2026-08-26T19:15:12.610521Z", + "world_size": 8 + }, + "allocated_bytes_current": 77343633920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113713497600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 77343633920, + "active_bytes_peak": 113713497600, + "allocated_bytes_current": 77343633920, + "allocated_bytes_peak": 113713497600, + "device": 7, + "inactive_split_bytes_current": 1100336640, + "inactive_split_bytes_peak": 14218493952, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 115834093568, + "reserved_bytes_peak": 115834093568, + "segment_count_current": 1896, + "segment_count_peak": 1896, + "timestamp_utc": "2026-08-26T19:15:12.247027Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 1098239488, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 14218493952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115834093568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115834093568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1896, + "segment_count_current_delta": 0, + "segment_count_peak": 1896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T191512610909Z.pickle", + "timestamp_utc": "2026-08-26T19:15:12.642807Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191512610909Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191512610909Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b4c5afe303a2ae077671d518ffaf488c4bb0a47 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191512610909Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e34d40deb0b88f1226dcc29298a03ac816dffa47f18a4faca72bf41f3fef9c96 +size 1892608 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..3f9541b4b1edf17ec01b2ddff80451a7d0f90708 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-better-memory-dump-support +commit=c3a8a8a7 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=0 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=dynamic +capacity=dynamic +cuda_graph=0 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=5 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=512 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..752e0c90269f47060e1aa4d7e2d4a38b2ad4b088 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log @@ -0,0 +1,1475 @@ +[19:14:11] WARN: ultraep-better-memory-dump-support lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / ultraep (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=dynamic cap=dynamic cg=0 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ c3a8a8a7 +============================================================== +W0826 19:14:12.857000 636111 torch/distributed/run.py:766] +W0826 19:14:12.857000 636111 torch/distributed/run.py:766] ***************************************** +W0826 19:14:12.857000 636111 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:14:12.857000 636111 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ False + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... False + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 1 is connected to 74 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0[Gloo] Rank +0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W826 19:14:31.430100084 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:14:31.430100805 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:14:31.431645140 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:14:31.431653582 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:14:31.431754469 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:14:31.436583452 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:14:31.438291851 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:14:31.584654809 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.583 seconds +time to initialize megatron (seconds): 7.271 +[after megatron is initialized] datetime: 2026-08-26 19:14:37 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 8521882624 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.k_layernorm.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:14:40 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 40 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(40, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from ddd6755186c6b13339ab7bd4e0fae553-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from ddd6755186c6b13339ab7bd4e0fae553-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from ddd6755186c6b13339ab7bd4e0fae553-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:14:40 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3814.77, 3823.43) + train/valid/test-data-iterators-setup ..........: (73.84, 90.95) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:14:41 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:15:10] iteration 1/ 5 | consumed samples: 8 | elapsed time per iteration (ms): 29607.0 | throughput per GPU (TFLOP/s/GPU): 6.1 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.116001E+01 | loss scale: 1.0 | grad norm: 44.807 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.85 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 73761.3779296875 | max allocated: 78889.880859375 | reserved: 85746.0 | max reserved: 85746.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:15:11] iteration 2/ 5 | consumed samples: 16 | elapsed time per iteration (ms): 732.0 | throughput per GPU (TFLOP/s/GPU): 247.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117607E+01 | loss scale: 1.0 | grad norm: 44.339 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:15:11] iteration 3/ 5 | consumed samples: 24 | elapsed time per iteration (ms): 410.4 | throughput per GPU (TFLOP/s/GPU): 440.6 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.685557E+00 | loss scale: 1.0 | grad norm: 62.732 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:15:12] iteration 4/ 5 | consumed samples: 32 | elapsed time per iteration (ms): 400.4 | throughput per GPU (TFLOP/s/GPU): 451.6 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338093E+01 | loss scale: 1.0 | grad norm: 20.309 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:15:12] iteration 5/ 5 | consumed samples: 40 | elapsed time per iteration (ms): 399.1 | throughput per GPU (TFLOP/s/GPU): 453.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.051497E+01 | loss scale: 1.0 | grad norm: 95.281 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 19:15:12 +[rank0]:[W826 19:15:13.888895667 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 19:15:15.045872297 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 19:15:15.046048586 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 19:15:15.047736643 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 19:15:15.048979265 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 19:15:15.300147857 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 19:15:15.419994829 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 19:15:15.638680902 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:15:21.548000 636111 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:15:21.549000 636111 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 636202 closing signal SIGTERM +W0826 19:15:21.549000 636111 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 636204 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 636111 got signal: 15 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..1c6c3e1c97044c9481591ad625b11de87c1bdbfc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 629 +4, 4 +5, 4 +6, 4 +7, 4 +0, 981 +1, 973 +2, 957 +3, 969 +4, 963 +5, 945 +6, 955 +7, 973 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 2149 +1, 2133 +2, 2149 +3, 2147 +4, 2165 +5, 2141 +6, 2139 +7, 2143 +0, 11431 +1, 11031 +2, 11431 +3, 11031 +4, 11131 +5, 11431 +6, 11531 +7, 11431 +0, 47013 +1, 47011 +2, 46985 +3, 46987 +4, 47001 +5, 47007 +6, 46989 +7, 47057 +0, 48193 +1, 48185 +2, 48185 +3, 48207 +4, 48201 +5, 48215 +6, 48155 +7, 48195 +0, 48647 +1, 48649 +2, 48659 +3, 48621 +4, 48645 +5, 48659 +6, 48649 +7, 48659 +0, 48775 +1, 48777 +2, 48787 +3, 48749 +4, 48773 +5, 48787 +6, 48777 +7, 48787 +0, 49355 +1, 49379 +2, 49363 +3, 49353 +4, 49351 +5, 49363 +6, 49337 +7, 49363 +0, 56439 +1, 56445 +2, 56569 +3, 56455 +4, 56565 +5, 56633 +6, 56635 +7, 56635 +0, 83853 +1, 83879 +2, 83859 +3, 83877 +4, 83915 +5, 83841 +6, 83841 +7, 83841 +0, 84311 +1, 84335 +2, 84313 +3, 84329 +4, 84327 +5, 84313 +6, 84313 +7, 84313 +0, 84311 +1, 84335 +2, 84313 +3, 84329 +4, 84327 +5, 84313 +6, 84313 +7, 84313 +0, 84507 +1, 84529 +2, 84565 +3, 84543 +4, 84541 +5, 84559 +6, 84539 +7, 84535 +0, 85571 +1, 85625 +2, 85625 +3, 85645 +4, 85593 +5, 85597 +6, 85587 +7, 85613 +0, 92637 +1, 92427 +2, 92355 +3, 92707 +4, 92711 +5, 92315 +6, 92835 +7, 92395 +0, 115531 +1, 115557 +2, 115553 +3, 115525 +4, 115573 +5, 115553 +6, 115533 +7, 115553 +0, 104841 +1, 104625 +2, 104921 +3, 104495 +4, 104025 +5, 103899 +6, 103995 +7, 103619 +0, 118839 +1, 118725 +2, 118695 +3, 118725 +4, 118717 +5, 118695 +6, 118709 +7, 118695 +0, 118017 +1, 117903 +2, 117873 +3, 117903 +4, 117895 +5, 117873 +6, 117887 +7, 117873 +0, 114801 +1, 114687 +2, 114657 +3, 114687 +4, 114679 +5, 114657 +6, 114671 +7, 114657 +0, 112733 +1, 112619 +2, 112977 +3, 112771 +4, 112999 +5, 112589 +6, 112991 +7, 112589 +0, 1 +1, 4 +2, 4 +3, 4 +4, 112999 +5, 4 +6, 4 +7, 4 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191620696074Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191620696074Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d464f8d7f03d5e2c7bb21feeb129c5298bd44ea4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191620696074Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75881334784, + "active_bytes_current_delta": 32329081344, + "active_bytes_peak": 81402461696, + "active_bytes_peak_delta": 37843916800, + "after": { + "active_bytes_current": 75881334784, + "active_bytes_peak": 81402461696, + "allocated_bytes_current": 75881334784, + "allocated_bytes_peak": 81402461696, + "device": 0, + "inactive_split_bytes_current": 4057905152, + "inactive_split_bytes_peak": 15084731392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 81738596352, + "reserved_bytes_peak": 81738596352, + "segment_count_current": 2100, + "segment_count_peak": 2100, + "timestamp_utc": "2026-08-26T19:16:20.695654Z", + "world_size": 8 + }, + "allocated_bytes_current": 75881334784, + "allocated_bytes_current_delta": 32329081344, + "allocated_bytes_peak": 81402461696, + "allocated_bytes_peak_delta": 37843916800, + "before": { + "active_bytes_current": 43552253440, + "active_bytes_peak": 43558544896, + "allocated_bytes_current": 43552253440, + "allocated_bytes_peak": 43558544896, + "device": 0, + "inactive_split_bytes_current": 1255496192, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 44822429696, + "reserved_bytes_peak": 44822429696, + "segment_count_current": 885, + "segment_count_peak": 885, + "timestamp_utc": "2026-08-26T19:15:53.657809Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4057905152, + "inactive_split_bytes_current_delta": 2802408960, + "inactive_split_bytes_peak": 15084731392, + "inactive_split_bytes_peak_delta": 13472470016, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 81738596352, + "reserved_bytes_current_delta": 36916166656, + "reserved_bytes_peak": 81738596352, + "reserved_bytes_peak_delta": 36916166656, + "segment_count_current": 2100, + "segment_count_current_delta": 1215, + "segment_count_peak": 2100, + "segment_count_peak_delta": 1215, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T191620696074Z.pickle", + "timestamp_utc": "2026-08-26T19:16:20.830603Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191620696074Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191620696074Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..036ebff94b31d8839ccf21a45dff57c8d82dd692 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191620696074Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5bf2612fd24e5e121d3de2ff6d62639d67f1602387ad59615945e3e54d7275b +size 7007310 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191622179582Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191622179582Z.json new file mode 100644 index 0000000000000000000000000000000000000000..03f20feba6d68934e44728bb1b55f3fb3b9285e9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191622179582Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75881334784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112192527360, + "active_bytes_peak_delta": 30790065664, + "after": { + "active_bytes_current": 75881334784, + "active_bytes_peak": 112192527360, + "allocated_bytes_current": 75881334784, + "allocated_bytes_peak": 112192527360, + "device": 0, + "inactive_split_bytes_current": 4060002304, + "inactive_split_bytes_peak": 15084731392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112560439296, + "reserved_bytes_peak": 112560439296, + "segment_count_current": 2878, + "segment_count_peak": 2878, + "timestamp_utc": "2026-08-26T19:16:22.179159Z", + "world_size": 8 + }, + "allocated_bytes_current": 75881334784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112192527360, + "allocated_bytes_peak_delta": 30790065664, + "before": { + "active_bytes_current": 75881334784, + "active_bytes_peak": 81402461696, + "allocated_bytes_current": 75881334784, + "allocated_bytes_peak": 81402461696, + "device": 0, + "inactive_split_bytes_current": 4057905152, + "inactive_split_bytes_peak": 15084731392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 81738596352, + "reserved_bytes_peak": 81738596352, + "segment_count_current": 2100, + "segment_count_peak": 2100, + "timestamp_utc": "2026-08-26T19:16:20.850951Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4060002304, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 15084731392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112560439296, + "reserved_bytes_current_delta": 30821842944, + "reserved_bytes_peak": 112560439296, + "reserved_bytes_peak_delta": 30821842944, + "segment_count_current": 2878, + "segment_count_current_delta": 778, + "segment_count_peak": 2878, + "segment_count_peak_delta": 778, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T191622179582Z.pickle", + "timestamp_utc": "2026-08-26T19:16:22.308344Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191622179582Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191622179582Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b5d31ec9e8bae866701a5d2b7e7f4ace32ce6e48 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191622179582Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c92889d9a53a5e41ebf1c0399368378806d39fafd049ad979079d7bac4c7ddbf +size 7115203 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191623321541Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191623321541Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c084a3dc8c394c70f7d0a7efa88fdb509ddddbec --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191623321541Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75881334784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112218545152, + "active_bytes_peak_delta": 26017792, + "after": { + "active_bytes_current": 75881334784, + "active_bytes_peak": 112218545152, + "allocated_bytes_current": 75881334784, + "allocated_bytes_peak": 112218545152, + "device": 0, + "inactive_split_bytes_current": 4060002304, + "inactive_split_bytes_peak": 15084731392, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112560439296, + "reserved_bytes_peak": 112560439296, + "segment_count_current": 2878, + "segment_count_peak": 2878, + "timestamp_utc": "2026-08-26T19:16:23.321103Z", + "world_size": 8 + }, + "allocated_bytes_current": 75881334784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112218545152, + "allocated_bytes_peak_delta": 26017792, + "before": { + "active_bytes_current": 75881334784, + "active_bytes_peak": 112192527360, + "allocated_bytes_current": 75881334784, + "allocated_bytes_peak": 112192527360, + "device": 0, + "inactive_split_bytes_current": 4060002304, + "inactive_split_bytes_peak": 15084731392, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112560439296, + "reserved_bytes_peak": 112560439296, + "segment_count_current": 2878, + "segment_count_peak": 2878, + "timestamp_utc": "2026-08-26T19:16:22.324570Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4060002304, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15084731392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112560439296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112560439296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2878, + "segment_count_current_delta": 0, + "segment_count_peak": 2878, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T191623321541Z.pickle", + "timestamp_utc": "2026-08-26T19:16:23.450000Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191623321541Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191623321541Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a3bfc947dda716b8609aa09fd2bb74e747e6e1a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191623321541Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d463973b16ab2756de3f6e68756bb59fae85ba943dbefd7241d18324fbaf48 +size 7078606 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191626684089Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191626684089Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dd3b8c1d0a73a77b7996f6fa975cf07762a721f7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191626684089Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75898378240, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 112218545152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75898378240, + "active_bytes_peak": 112218545152, + "allocated_bytes_current": 75898378240, + "allocated_bytes_peak": 112218545152, + "device": 0, + "inactive_split_bytes_current": 4133136384, + "inactive_split_bytes_peak": 15084731392, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115951534080, + "reserved_bytes_peak": 115951534080, + "segment_count_current": 3092, + "segment_count_peak": 3092, + "timestamp_utc": "2026-08-26T19:16:26.683582Z", + "world_size": 8 + }, + "allocated_bytes_current": 75898378240, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 112218545152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75881334784, + "active_bytes_peak": 112218545152, + "allocated_bytes_current": 75881334784, + "allocated_bytes_peak": 112218545152, + "device": 0, + "inactive_split_bytes_current": 4060002304, + "inactive_split_bytes_peak": 15084731392, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112560439296, + "reserved_bytes_peak": 112560439296, + "segment_count_current": 2878, + "segment_count_peak": 2878, + "timestamp_utc": "2026-08-26T19:16:23.465606Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4133136384, + "inactive_split_bytes_current_delta": 73134080, + "inactive_split_bytes_peak": 15084731392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115951534080, + "reserved_bytes_current_delta": 3391094784, + "reserved_bytes_peak": 115951534080, + "reserved_bytes_peak_delta": 3391094784, + "segment_count_current": 3092, + "segment_count_current_delta": 214, + "segment_count_peak": 3092, + "segment_count_peak_delta": 214, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T191626684089Z.pickle", + "timestamp_utc": "2026-08-26T19:16:26.823911Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191626684089Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191626684089Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fc533182de2e487702b5414b4e176c0478f2ad64 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191626684089Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d47c91d61bab7295aa9ae9a85e332365c062cee9ef6164751cbd77c74f862cef +size 7504696 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191627204396Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191627204396Z.json new file mode 100644 index 0000000000000000000000000000000000000000..df38c36a16fcad49ee044d4fa33e256da459cc3b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191627204396Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75898378240, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112218545152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75898378240, + "active_bytes_peak": 112218545152, + "allocated_bytes_current": 75898378240, + "allocated_bytes_peak": 112218545152, + "device": 0, + "inactive_split_bytes_current": 4133136384, + "inactive_split_bytes_peak": 15084731392, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115951534080, + "reserved_bytes_peak": 115951534080, + "segment_count_current": 3092, + "segment_count_peak": 3092, + "timestamp_utc": "2026-08-26T19:16:27.203958Z", + "world_size": 8 + }, + "allocated_bytes_current": 75898378240, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112218545152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75898378240, + "active_bytes_peak": 112218545152, + "allocated_bytes_current": 75898378240, + "allocated_bytes_peak": 112218545152, + "device": 0, + "inactive_split_bytes_current": 4133136384, + "inactive_split_bytes_peak": 15084731392, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115951534080, + "reserved_bytes_peak": 115951534080, + "segment_count_current": 3092, + "segment_count_peak": 3092, + "timestamp_utc": "2026-08-26T19:16:26.840397Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4133136384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15084731392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115951534080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115951534080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3092, + "segment_count_current_delta": 0, + "segment_count_peak": 3092, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T191627204396Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.224398Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191627204396Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191627204396Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72702aa9db1d8aebba6e0eb4fe73a0bd9938675d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191627204396Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bef36c0cb9dac74672042864db0883ecc581e52b479258fc1e9dbb31216aaf64 +size 890682 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191627580153Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191627580153Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d4bb9ccbf1acd69a7c7d14b3df5f7317d1b535dc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191627580153Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75898378240, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112218545152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75898378240, + "active_bytes_peak": 112218545152, + "allocated_bytes_current": 75898378240, + "allocated_bytes_peak": 112218545152, + "device": 0, + "inactive_split_bytes_current": 4133136384, + "inactive_split_bytes_peak": 15084731392, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115951534080, + "reserved_bytes_peak": 115951534080, + "segment_count_current": 3092, + "segment_count_peak": 3092, + "timestamp_utc": "2026-08-26T19:16:27.579693Z", + "world_size": 8 + }, + "allocated_bytes_current": 75898378240, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112218545152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75898378240, + "active_bytes_peak": 112218545152, + "allocated_bytes_current": 75898378240, + "allocated_bytes_peak": 112218545152, + "device": 0, + "inactive_split_bytes_current": 4133136384, + "inactive_split_bytes_peak": 15084731392, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115951534080, + "reserved_bytes_peak": 115951534080, + "segment_count_current": 3092, + "segment_count_peak": 3092, + "timestamp_utc": "2026-08-26T19:16:27.226005Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4133136384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15084731392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115951534080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115951534080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3092, + "segment_count_current_delta": 0, + "segment_count_peak": 3092, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T191627580153Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.598537Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191627580153Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191627580153Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a164ae3239b0c2b41aab8ce48acb38a10b6f4c20 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191627580153Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aefedf64e1d0fec86e2c8ae7f81d4bb04b9f83f2bb70bce3a3e85963e184a1ce +size 890682 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191620696219Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191620696219Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d916da560ed5a322fb63276f0da04e28f0fda646 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191620696219Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75897122816, + "active_bytes_current_delta": 32343126528, + "active_bytes_peak": 81411558400, + "active_bytes_peak_delta": 37851270656, + "after": { + "active_bytes_current": 75897122816, + "active_bytes_peak": 81411558400, + "allocated_bytes_current": 75897122816, + "allocated_bytes_peak": 81411558400, + "device": 1, + "inactive_split_bytes_current": 3922579456, + "inactive_split_bytes_peak": 14693553152, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 81765859328, + "reserved_bytes_peak": 81765859328, + "segment_count_current": 2106, + "segment_count_peak": 2106, + "timestamp_utc": "2026-08-26T19:16:20.695782Z", + "world_size": 8 + }, + "allocated_bytes_current": 75897122816, + "allocated_bytes_current_delta": 32343126528, + "allocated_bytes_peak": 81411558400, + "allocated_bytes_peak_delta": 37851270656, + "before": { + "active_bytes_current": 43553996288, + "active_bytes_peak": 43560287744, + "allocated_bytes_current": 43553996288, + "allocated_bytes_peak": 43560287744, + "device": 1, + "inactive_split_bytes_current": 1255850496, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T19:15:53.657841Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3922579456, + "inactive_split_bytes_current_delta": 2666728960, + "inactive_split_bytes_peak": 14693553152, + "inactive_split_bytes_peak_delta": 13081291776, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 81765859328, + "reserved_bytes_current_delta": 36941332480, + "reserved_bytes_peak": 81765859328, + "reserved_bytes_peak_delta": 36941332480, + "segment_count_current": 2106, + "segment_count_current_delta": 1216, + "segment_count_peak": 2106, + "segment_count_peak_delta": 1216, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T191620696219Z.pickle", + "timestamp_utc": "2026-08-26T19:16:20.832601Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191620696219Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191620696219Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5396e079d7952cd2d17a3cd6c39300aa675a3c50 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191620696219Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2a57201cb08746fa129741d28f7e38ddeae75c9fb7938eb671da888a7d928b3 +size 7010157 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191622179696Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191622179696Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b0d0c3bfefcc08315f466a6b0bd95219d439af85 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191622179696Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75897122816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112210289152, + "active_bytes_peak_delta": 30798730752, + "after": { + "active_bytes_current": 75897122816, + "active_bytes_peak": 112210289152, + "allocated_bytes_current": 75897122816, + "allocated_bytes_peak": 112210289152, + "device": 1, + "inactive_split_bytes_current": 3924676608, + "inactive_split_bytes_peak": 14693553152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112587702272, + "reserved_bytes_peak": 112587702272, + "segment_count_current": 2889, + "segment_count_peak": 2889, + "timestamp_utc": "2026-08-26T19:16:22.179271Z", + "world_size": 8 + }, + "allocated_bytes_current": 75897122816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112210289152, + "allocated_bytes_peak_delta": 30798730752, + "before": { + "active_bytes_current": 75897122816, + "active_bytes_peak": 81411558400, + "allocated_bytes_current": 75897122816, + "allocated_bytes_peak": 81411558400, + "device": 1, + "inactive_split_bytes_current": 3922579456, + "inactive_split_bytes_peak": 14693553152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 81765859328, + "reserved_bytes_peak": 81765859328, + "segment_count_current": 2106, + "segment_count_peak": 2106, + "timestamp_utc": "2026-08-26T19:16:20.852978Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3924676608, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 14693553152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112587702272, + "reserved_bytes_current_delta": 30821842944, + "reserved_bytes_peak": 112587702272, + "reserved_bytes_peak_delta": 30821842944, + "segment_count_current": 2889, + "segment_count_current_delta": 783, + "segment_count_peak": 2889, + "segment_count_peak_delta": 783, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T191622179696Z.pickle", + "timestamp_utc": "2026-08-26T19:16:22.309207Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191622179696Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191622179696Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f9bd354b0bf5d356a25f9a51b33bb495363bc40c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191622179696Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86e425075344cc00ed4ef4ad7f254770b426b424ce043768d2510c439c36e101 +size 6988330 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191623321668Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191623321668Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bb4860b37bf8b7db9da36ed49addac9712818cc5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191623321668Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75897122816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112210289152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75897122816, + "active_bytes_peak": 112210289152, + "allocated_bytes_current": 75897122816, + "allocated_bytes_peak": 112210289152, + "device": 1, + "inactive_split_bytes_current": 3924676608, + "inactive_split_bytes_peak": 14693553152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112587702272, + "reserved_bytes_peak": 112587702272, + "segment_count_current": 2889, + "segment_count_peak": 2889, + "timestamp_utc": "2026-08-26T19:16:23.321221Z", + "world_size": 8 + }, + "allocated_bytes_current": 75897122816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112210289152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75897122816, + "active_bytes_peak": 112210289152, + "allocated_bytes_current": 75897122816, + "allocated_bytes_peak": 112210289152, + "device": 1, + "inactive_split_bytes_current": 3924676608, + "inactive_split_bytes_peak": 14693553152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112587702272, + "reserved_bytes_peak": 112587702272, + "segment_count_current": 2889, + "segment_count_peak": 2889, + "timestamp_utc": "2026-08-26T19:16:22.332427Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3924676608, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14693553152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112587702272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112587702272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2889, + "segment_count_current_delta": 0, + "segment_count_peak": 2889, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T191623321668Z.pickle", + "timestamp_utc": "2026-08-26T19:16:23.451085Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191623321668Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191623321668Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cc34b10f71f4e2a943f2897e8d11a2c9849b4fc6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191623321668Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:443b6463624e976d6299f96c475e34c5a942905a68f2f63c8f601aca5caa9858 +size 6950925 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191626684236Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191626684236Z.json new file mode 100644 index 0000000000000000000000000000000000000000..56dd87c26f4b382ee63e108bd8de26f1dd8dbd1e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191626684236Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75914166272, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 112210289152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75914166272, + "active_bytes_peak": 112210289152, + "allocated_bytes_current": 75914166272, + "allocated_bytes_peak": 112210289152, + "device": 1, + "inactive_split_bytes_current": 3997810688, + "inactive_split_bytes_peak": 14693553152, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115831996416, + "reserved_bytes_peak": 115831996416, + "segment_count_current": 3091, + "segment_count_peak": 3091, + "timestamp_utc": "2026-08-26T19:16:26.683719Z", + "world_size": 8 + }, + "allocated_bytes_current": 75914166272, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 112210289152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75897122816, + "active_bytes_peak": 112210289152, + "allocated_bytes_current": 75897122816, + "allocated_bytes_peak": 112210289152, + "device": 1, + "inactive_split_bytes_current": 3924676608, + "inactive_split_bytes_peak": 14693553152, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112587702272, + "reserved_bytes_peak": 112587702272, + "segment_count_current": 2889, + "segment_count_peak": 2889, + "timestamp_utc": "2026-08-26T19:16:23.466182Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3997810688, + "inactive_split_bytes_current_delta": 73134080, + "inactive_split_bytes_peak": 14693553152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115831996416, + "reserved_bytes_current_delta": 3244294144, + "reserved_bytes_peak": 115831996416, + "reserved_bytes_peak_delta": 3244294144, + "segment_count_current": 3091, + "segment_count_current_delta": 202, + "segment_count_peak": 3091, + "segment_count_peak_delta": 202, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T191626684236Z.pickle", + "timestamp_utc": "2026-08-26T19:16:26.823835Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191626684236Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191626684236Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..884071a9e197b416316b3e3b62dea6f3953c12c4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191626684236Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e0e181478e8be37afc29f595373ef444b0be974be83ae2310753c67e4d964fc +size 7508864 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191627204486Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191627204486Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a9685986d7b05c6fa01312f69f3e4aa5c11c1a1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191627204486Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75914166272, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112210289152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75914166272, + "active_bytes_peak": 112210289152, + "allocated_bytes_current": 75914166272, + "allocated_bytes_peak": 112210289152, + "device": 1, + "inactive_split_bytes_current": 3997810688, + "inactive_split_bytes_peak": 14693553152, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115831996416, + "reserved_bytes_peak": 115831996416, + "segment_count_current": 3091, + "segment_count_peak": 3091, + "timestamp_utc": "2026-08-26T19:16:27.204058Z", + "world_size": 8 + }, + "allocated_bytes_current": 75914166272, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112210289152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75914166272, + "active_bytes_peak": 112210289152, + "allocated_bytes_current": 75914166272, + "allocated_bytes_peak": 112210289152, + "device": 1, + "inactive_split_bytes_current": 3997810688, + "inactive_split_bytes_peak": 14693553152, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 115831996416, + "reserved_bytes_peak": 115831996416, + "segment_count_current": 3091, + "segment_count_peak": 3091, + "timestamp_utc": "2026-08-26T19:16:26.843936Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3997810688, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14693553152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115831996416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115831996416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3091, + "segment_count_current_delta": 0, + "segment_count_peak": 3091, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T191627204486Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.224232Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191627204486Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191627204486Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3e5ce8849b6e3d2bcb99ffd30c524d50fee9cbc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191627204486Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3acf1b12cd7824e0a29bac1224295a4e3cb5d2629e27ad80abef418b7627ed02 +size 893855 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191627580170Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191627580170Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b3f30d1e979fcac21b8b6829e6e112c64a924aad --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191627580170Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75914166272, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112210289152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75914166272, + "active_bytes_peak": 112210289152, + "allocated_bytes_current": 75914166272, + "allocated_bytes_peak": 112210289152, + "device": 1, + "inactive_split_bytes_current": 3997810688, + "inactive_split_bytes_peak": 14693553152, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115831996416, + "reserved_bytes_peak": 115831996416, + "segment_count_current": 3091, + "segment_count_peak": 3091, + "timestamp_utc": "2026-08-26T19:16:27.579784Z", + "world_size": 8 + }, + "allocated_bytes_current": 75914166272, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112210289152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75914166272, + "active_bytes_peak": 112210289152, + "allocated_bytes_current": 75914166272, + "allocated_bytes_peak": 112210289152, + "device": 1, + "inactive_split_bytes_current": 3997810688, + "inactive_split_bytes_peak": 14693553152, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 115831996416, + "reserved_bytes_peak": 115831996416, + "segment_count_current": 3091, + "segment_count_peak": 3091, + "timestamp_utc": "2026-08-26T19:16:27.225827Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3997810688, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14693553152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115831996416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115831996416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3091, + "segment_count_current_delta": 0, + "segment_count_peak": 3091, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T191627580170Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.598475Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191627580170Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191627580170Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2ea3a28cd2be2a4e163dbb116b022fe2747df16 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191627580170Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f32dbda498e5af06c03b624b5c4f71feb0aad0e5caa82cb71e3073214fa1394 +size 893855 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191620696133Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191620696133Z.json new file mode 100644 index 0000000000000000000000000000000000000000..342445abe3903cfb0ad64cbca4f02e297e8bd5a9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191620696133Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75897316864, + "active_bytes_current_delta": 32343122432, + "active_bytes_peak": 81403138048, + "active_bytes_peak_delta": 37842652160, + "after": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 81403138048, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 81403138048, + "device": 2, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_peak": 14871617024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 81740693504, + "reserved_bytes_peak": 81740693504, + "segment_count_current": 2093, + "segment_count_peak": 2093, + "timestamp_utc": "2026-08-26T19:16:20.695712Z", + "world_size": 8 + }, + "allocated_bytes_current": 75897316864, + "allocated_bytes_current_delta": 32343122432, + "allocated_bytes_peak": 81403138048, + "allocated_bytes_peak_delta": 37842652160, + "before": { + "active_bytes_current": 43554194432, + "active_bytes_peak": 43560485888, + "allocated_bytes_current": 43554194432, + "allocated_bytes_peak": 43560485888, + "device": 2, + "inactive_split_bytes_current": 1255652352, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T19:15:53.657886Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_current_delta": 2635275776, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 13259355648, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 81740693504, + "reserved_bytes_current_delta": 36905680896, + "reserved_bytes_peak": 81740693504, + "reserved_bytes_peak_delta": 36905680896, + "segment_count_current": 2093, + "segment_count_current_delta": 1210, + "segment_count_peak": 2093, + "segment_count_peak_delta": 1210, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T191620696133Z.pickle", + "timestamp_utc": "2026-08-26T19:16:20.831270Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191620696133Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191620696133Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a96b55e73db5b9a2822a666c4c95ccd8a25ab94 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191620696133Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6382f4ba6a2ee080ad53a694e63e096c3650dd851fe6bd3593b477fee65fd80b +size 6992691 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191622179750Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191622179750Z.json new file mode 100644 index 0000000000000000000000000000000000000000..128567f9418661de79c16e9dc0c8055665f2b9a7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191622179750Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75897316864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112208827904, + "active_bytes_peak_delta": 30805689856, + "after": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 112208827904, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 112208827904, + "device": 2, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_peak": 14871617024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112583507968, + "reserved_bytes_peak": 112583507968, + "segment_count_current": 2863, + "segment_count_peak": 2863, + "timestamp_utc": "2026-08-26T19:16:22.179316Z", + "world_size": 8 + }, + "allocated_bytes_current": 75897316864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112208827904, + "allocated_bytes_peak_delta": 30805689856, + "before": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 81403138048, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 81403138048, + "device": 2, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_peak": 14871617024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 81740693504, + "reserved_bytes_peak": 81740693504, + "segment_count_current": 2093, + "segment_count_peak": 2093, + "timestamp_utc": "2026-08-26T19:16:20.850574Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112583507968, + "reserved_bytes_current_delta": 30842814464, + "reserved_bytes_peak": 112583507968, + "reserved_bytes_peak_delta": 30842814464, + "segment_count_current": 2863, + "segment_count_current_delta": 770, + "segment_count_peak": 2863, + "segment_count_peak_delta": 770, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T191622179750Z.pickle", + "timestamp_utc": "2026-08-26T19:16:22.310320Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191622179750Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191622179750Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d056dd6a21fc4f1d1eb0e4a62ade44018f16fc10 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191622179750Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eee22d1e7e747bc22bd9001a71c8637a4b4fe1d8955896d46af1f23f214acc4c +size 6748998 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191623321713Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191623321713Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9f82aa013c695adbd760f0f2126a02682218fbad --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191623321713Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75897316864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112219198976, + "active_bytes_peak_delta": 10371072, + "after": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 112219198976, + "device": 2, + "inactive_split_bytes_current": 3893025280, + "inactive_split_bytes_peak": 14871617024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112583507968, + "reserved_bytes_peak": 112583507968, + "segment_count_current": 2863, + "segment_count_peak": 2863, + "timestamp_utc": "2026-08-26T19:16:23.321253Z", + "world_size": 8 + }, + "allocated_bytes_current": 75897316864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112219198976, + "allocated_bytes_peak_delta": 10371072, + "before": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 112208827904, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 112208827904, + "device": 2, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_peak": 14871617024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112583507968, + "reserved_bytes_peak": 112583507968, + "segment_count_current": 2863, + "segment_count_peak": 2863, + "timestamp_utc": "2026-08-26T19:16:22.326392Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3893025280, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112583507968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112583507968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2863, + "segment_count_current_delta": 0, + "segment_count_peak": 2863, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T191623321713Z.pickle", + "timestamp_utc": "2026-08-26T19:16:23.451247Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191623321713Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191623321713Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29286ba568e2067bcbdf5bd7654d13e64de8e72e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191623321713Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e06e02c65e021a7e9b58eade2692f656d5675b97f05c1a9f6cff538e4dccb9d +size 6711483 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191626684246Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191626684246Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e67dfc40f52761d41bc27d185a066961bd0573eb --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191626684246Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75914360320, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 112219198976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 2, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:26.683745Z", + "world_size": 8 + }, + "allocated_bytes_current": 75914360320, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 112219198976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 112219198976, + "device": 2, + "inactive_split_bytes_current": 3893025280, + "inactive_split_bytes_peak": 14871617024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112583507968, + "reserved_bytes_peak": 112583507968, + "segment_count_current": 2863, + "segment_count_peak": 2863, + "timestamp_utc": "2026-08-26T19:16:23.466820Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_current_delta": 73134080, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115800539136, + "reserved_bytes_current_delta": 3217031168, + "reserved_bytes_peak": 115800539136, + "reserved_bytes_peak_delta": 3217031168, + "segment_count_current": 3075, + "segment_count_current_delta": 212, + "segment_count_peak": 3075, + "segment_count_peak_delta": 212, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T191626684246Z.pickle", + "timestamp_utc": "2026-08-26T19:16:26.824055Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191626684246Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191626684246Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..556fc0b457532a821e412b9a1726d5bcb1cc787a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191626684246Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6f8fed0939e5c02fd8dc4bd9568492ae0ebf69219b1666964b6521a429207e9 +size 7494048 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191627204512Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191627204512Z.json new file mode 100644 index 0000000000000000000000000000000000000000..056b79edaf00a046fc0d5aabd04b7ce92f029ccc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191627204512Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75914360320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112219198976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 2, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:27.204082Z", + "world_size": 8 + }, + "allocated_bytes_current": 75914360320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112219198976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 2, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:26.841097Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115800539136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115800539136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3075, + "segment_count_current_delta": 0, + "segment_count_peak": 3075, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T191627204512Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.224195Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191627204512Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191627204512Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a8fea55b7490fc135abb50f8c15d761f259167c0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191627204512Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cc746b2abbfd1c643727b398349f57f9089ee77b89a170f4bd7047036bd0031 +size 879602 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191627580234Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191627580234Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ebb7cb3d291148614b77a76589d14dc96336bc4f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191627580234Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75914360320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112219198976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 2, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:27.579840Z", + "world_size": 8 + }, + "allocated_bytes_current": 75914360320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112219198976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 2, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:27.225766Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115800539136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115800539136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3075, + "segment_count_current_delta": 0, + "segment_count_peak": 3075, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T191627580234Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.598521Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191627580234Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191627580234Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..943927084b017c96fe0bb32a8daf5368110a610f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191627580234Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b92e4cacf3ad4ab8d0cf091ab4d3955aa1ea93df3427e62f04b1655f07aede6a +size 879602 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191620696130Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191620696130Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6af20c7c5cfbf04475493e9662eb40db196c6c46 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191620696130Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75912380416, + "active_bytes_current_delta": 32354906624, + "active_bytes_peak": 81403872768, + "active_bytes_peak_delta": 37840107520, + "after": { + "active_bytes_current": 75912380416, + "active_bytes_peak": 81403872768, + "allocated_bytes_current": 75912380416, + "allocated_bytes_peak": 81403872768, + "device": 3, + "inactive_split_bytes_current": 3909419008, + "inactive_split_bytes_peak": 14554563584, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 81757470720, + "reserved_bytes_peak": 81757470720, + "segment_count_current": 2104, + "segment_count_peak": 2104, + "timestamp_utc": "2026-08-26T19:16:20.695694Z", + "world_size": 8 + }, + "allocated_bytes_current": 75912380416, + "allocated_bytes_current_delta": 32354906624, + "allocated_bytes_peak": 81403872768, + "allocated_bytes_peak_delta": 37840107520, + "before": { + "active_bytes_current": 43557473792, + "active_bytes_peak": 43563765248, + "allocated_bytes_current": 43557473792, + "allocated_bytes_peak": 43563765248, + "device": 3, + "inactive_split_bytes_current": 1223012864, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 44795166720, + "reserved_bytes_peak": 44795166720, + "segment_count_current": 887, + "segment_count_peak": 887, + "timestamp_utc": "2026-08-26T19:15:53.657865Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3909419008, + "inactive_split_bytes_current_delta": 2686406144, + "inactive_split_bytes_peak": 14554563584, + "inactive_split_bytes_peak_delta": 12942302208, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 81757470720, + "reserved_bytes_current_delta": 36962304000, + "reserved_bytes_peak": 81757470720, + "reserved_bytes_peak_delta": 36962304000, + "segment_count_current": 2104, + "segment_count_current_delta": 1217, + "segment_count_peak": 2104, + "segment_count_peak_delta": 1217, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T191620696130Z.pickle", + "timestamp_utc": "2026-08-26T19:16:20.831295Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191620696130Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191620696130Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8107252c0c3abc7802ca787adab3baa89ad4ec40 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191620696130Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ce50fb22ee8a210ab845e80fc9a9c15012afb3a6ba50caa977c6b60f0586f4d +size 7008809 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191622179723Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191622179723Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ee3a2b87c80be1e9b51a66874ef2a5306e83fbc3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191622179723Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75912380416, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112172755968, + "active_bytes_peak_delta": 30768883200, + "after": { + "active_bytes_current": 75912380416, + "active_bytes_peak": 112172755968, + "allocated_bytes_current": 75912380416, + "allocated_bytes_peak": 112172755968, + "device": 3, + "inactive_split_bytes_current": 3911516160, + "inactive_split_bytes_peak": 14554563584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112554147840, + "reserved_bytes_peak": 112554147840, + "segment_count_current": 2888, + "segment_count_peak": 2888, + "timestamp_utc": "2026-08-26T19:16:22.179292Z", + "world_size": 8 + }, + "allocated_bytes_current": 75912380416, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112172755968, + "allocated_bytes_peak_delta": 30768883200, + "before": { + "active_bytes_current": 75912380416, + "active_bytes_peak": 81403872768, + "allocated_bytes_current": 75912380416, + "allocated_bytes_peak": 81403872768, + "device": 3, + "inactive_split_bytes_current": 3909419008, + "inactive_split_bytes_peak": 14554563584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 81757470720, + "reserved_bytes_peak": 81757470720, + "segment_count_current": 2104, + "segment_count_peak": 2104, + "timestamp_utc": "2026-08-26T19:16:20.851455Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3911516160, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 14554563584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112554147840, + "reserved_bytes_current_delta": 30796677120, + "reserved_bytes_peak": 112554147840, + "reserved_bytes_peak_delta": 30796677120, + "segment_count_current": 2888, + "segment_count_current_delta": 784, + "segment_count_peak": 2888, + "segment_count_peak_delta": 784, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T191622179723Z.pickle", + "timestamp_utc": "2026-08-26T19:16:22.309855Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191622179723Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191622179723Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..25b7dd86823f9c079f9d62c83eb3877c3e31659a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191622179723Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ceffe5f4ef98ea76f4d044c40db3609b08251baf61122129126fad257353d65 +size 7030919 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191623321676Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191623321676Z.json new file mode 100644 index 0000000000000000000000000000000000000000..819a3565c68788d18d3f795366613d40613b51bb --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191623321676Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75912380416, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112191499264, + "active_bytes_peak_delta": 18743296, + "after": { + "active_bytes_current": 75912380416, + "active_bytes_peak": 112191499264, + "allocated_bytes_current": 75912380416, + "allocated_bytes_peak": 112191499264, + "device": 3, + "inactive_split_bytes_current": 3909419008, + "inactive_split_bytes_peak": 14554563584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112554147840, + "reserved_bytes_peak": 112554147840, + "segment_count_current": 2888, + "segment_count_peak": 2888, + "timestamp_utc": "2026-08-26T19:16:23.321231Z", + "world_size": 8 + }, + "allocated_bytes_current": 75912380416, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112191499264, + "allocated_bytes_peak_delta": 18743296, + "before": { + "active_bytes_current": 75912380416, + "active_bytes_peak": 112172755968, + "allocated_bytes_current": 75912380416, + "allocated_bytes_peak": 112172755968, + "device": 3, + "inactive_split_bytes_current": 3911516160, + "inactive_split_bytes_peak": 14554563584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112554147840, + "reserved_bytes_peak": 112554147840, + "segment_count_current": 2888, + "segment_count_peak": 2888, + "timestamp_utc": "2026-08-26T19:16:22.326052Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3909419008, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 14554563584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112554147840, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112554147840, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2888, + "segment_count_current_delta": 0, + "segment_count_peak": 2888, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T191623321676Z.pickle", + "timestamp_utc": "2026-08-26T19:16:23.452379Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191623321676Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191623321676Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5cf8dddfd09e5debb5558998e3dc3a00fc3d1af0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191623321676Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b5a3ea58c69e800373b0c0abe89e4e20efafb6af9902ee66cdbe4cd0ccc2cae +size 6993669 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191626684261Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191626684261Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7feb0109acce6e9920cfbea4619713b3b61554c2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191626684261Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75929423872, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 112191499264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75929423872, + "active_bytes_peak": 112191499264, + "allocated_bytes_current": 75929423872, + "allocated_bytes_peak": 112191499264, + "device": 3, + "inactive_split_bytes_current": 3984650240, + "inactive_split_bytes_peak": 14554563584, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 115831996416, + "reserved_bytes_peak": 115831996416, + "segment_count_current": 3093, + "segment_count_peak": 3093, + "timestamp_utc": "2026-08-26T19:16:26.683680Z", + "world_size": 8 + }, + "allocated_bytes_current": 75929423872, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 112191499264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75912380416, + "active_bytes_peak": 112191499264, + "allocated_bytes_current": 75912380416, + "allocated_bytes_peak": 112191499264, + "device": 3, + "inactive_split_bytes_current": 3909419008, + "inactive_split_bytes_peak": 14554563584, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112554147840, + "reserved_bytes_peak": 112554147840, + "segment_count_current": 2888, + "segment_count_peak": 2888, + "timestamp_utc": "2026-08-26T19:16:23.467372Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3984650240, + "inactive_split_bytes_current_delta": 75231232, + "inactive_split_bytes_peak": 14554563584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 115831996416, + "reserved_bytes_current_delta": 3277848576, + "reserved_bytes_peak": 115831996416, + "reserved_bytes_peak_delta": 3277848576, + "segment_count_current": 3093, + "segment_count_current_delta": 205, + "segment_count_peak": 3093, + "segment_count_peak_delta": 205, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T191626684261Z.pickle", + "timestamp_utc": "2026-08-26T19:16:26.823940Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191626684261Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191626684261Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cb2483fcac928f609c09dec1e166acd02b53355a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191626684261Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd7a22a2d50dd6e263f8dd4f50bcf0fcbe2e4379d112557b59b44e8a0a340c05 +size 7507925 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191627204504Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191627204504Z.json new file mode 100644 index 0000000000000000000000000000000000000000..591d4fe90960e88315ffecdd1ab55907cc205bd6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191627204504Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75929423872, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112191499264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75929423872, + "active_bytes_peak": 112191499264, + "allocated_bytes_current": 75929423872, + "allocated_bytes_peak": 112191499264, + "device": 3, + "inactive_split_bytes_current": 3984650240, + "inactive_split_bytes_peak": 14554563584, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 115831996416, + "reserved_bytes_peak": 115831996416, + "segment_count_current": 3093, + "segment_count_peak": 3093, + "timestamp_utc": "2026-08-26T19:16:27.204080Z", + "world_size": 8 + }, + "allocated_bytes_current": 75929423872, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112191499264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75929423872, + "active_bytes_peak": 112191499264, + "allocated_bytes_current": 75929423872, + "allocated_bytes_peak": 112191499264, + "device": 3, + "inactive_split_bytes_current": 3984650240, + "inactive_split_bytes_peak": 14554563584, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 115831996416, + "reserved_bytes_peak": 115831996416, + "segment_count_current": 3093, + "segment_count_peak": 3093, + "timestamp_utc": "2026-08-26T19:16:26.841025Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3984650240, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14554563584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 115831996416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115831996416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3093, + "segment_count_current_delta": 0, + "segment_count_peak": 3093, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T191627204504Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.224727Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191627204504Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191627204504Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe9d7e85c0c5fd7fde9b2d1fe63bef9706567527 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191627204504Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3af4392a1f18af03cb7b96ff73392a52c708a4404839ff2669ffbe7eb60c31bd +size 893072 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191627580184Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191627580184Z.json new file mode 100644 index 0000000000000000000000000000000000000000..86e86409631391c912f0a3a99372f4abcce4e703 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191627580184Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75929423872, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112191499264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75929423872, + "active_bytes_peak": 112191499264, + "allocated_bytes_current": 75929423872, + "allocated_bytes_peak": 112191499264, + "device": 3, + "inactive_split_bytes_current": 3984650240, + "inactive_split_bytes_peak": 14554563584, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 115831996416, + "reserved_bytes_peak": 115831996416, + "segment_count_current": 3093, + "segment_count_peak": 3093, + "timestamp_utc": "2026-08-26T19:16:27.579800Z", + "world_size": 8 + }, + "allocated_bytes_current": 75929423872, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112191499264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75929423872, + "active_bytes_peak": 112191499264, + "allocated_bytes_current": 75929423872, + "allocated_bytes_peak": 112191499264, + "device": 3, + "inactive_split_bytes_current": 3984650240, + "inactive_split_bytes_peak": 14554563584, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 115831996416, + "reserved_bytes_peak": 115831996416, + "segment_count_current": 3093, + "segment_count_peak": 3093, + "timestamp_utc": "2026-08-26T19:16:27.226323Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3984650240, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14554563584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 115831996416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115831996416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3093, + "segment_count_current_delta": 0, + "segment_count_peak": 3093, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T191627580184Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.598815Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191627580184Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191627580184Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7020d5a376cd3a32074c5a11fa50530bcc752ce --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191627580184Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e64c67c1f6da75531a4ef2ce74f3fb754f462d488bb9984e438df2366e6f6d46 +size 893072 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191620696059Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191620696059Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e94e5f206307d68b8f482d4adb042d2d9acad53 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191620696059Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75910481920, + "active_bytes_current_delta": 32359153152, + "active_bytes_peak": 81404331520, + "active_bytes_peak_delta": 37846711296, + "after": { + "active_bytes_current": 75910481920, + "active_bytes_peak": 81404331520, + "allocated_bytes_current": 75910481920, + "allocated_bytes_peak": 81404331520, + "device": 4, + "inactive_split_bytes_current": 3900831744, + "inactive_split_bytes_peak": 14613085184, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 81757470720, + "reserved_bytes_peak": 81757470720, + "segment_count_current": 2105, + "segment_count_peak": 2105, + "timestamp_utc": "2026-08-26T19:16:20.695597Z", + "world_size": 8 + }, + "allocated_bytes_current": 75910481920, + "allocated_bytes_current_delta": 32359153152, + "allocated_bytes_peak": 81404331520, + "allocated_bytes_peak_delta": 37846711296, + "before": { + "active_bytes_current": 43551328768, + "active_bytes_peak": 43557620224, + "allocated_bytes_current": 43551328768, + "allocated_bytes_peak": 43557620224, + "device": 4, + "inactive_split_bytes_current": 1254323712, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 44820332544, + "reserved_bytes_peak": 44820332544, + "segment_count_current": 889, + "segment_count_peak": 889, + "timestamp_utc": "2026-08-26T19:15:53.657680Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3900831744, + "inactive_split_bytes_current_delta": 2646508032, + "inactive_split_bytes_peak": 14613085184, + "inactive_split_bytes_peak_delta": 13000823808, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 81757470720, + "reserved_bytes_current_delta": 36937138176, + "reserved_bytes_peak": 81757470720, + "reserved_bytes_peak_delta": 36937138176, + "segment_count_current": 2105, + "segment_count_current_delta": 1216, + "segment_count_peak": 2105, + "segment_count_peak_delta": 1216, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T191620696059Z.pickle", + "timestamp_utc": "2026-08-26T19:16:20.830566Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191620696059Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191620696059Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94bc44cb98be7dcc416e5f9c90779021713ddfab --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191620696059Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51b7b2c7ff8562e8046b06d1588a645ed3c97be95f6ab11502931467ffabfb19 +size 7010523 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191622179586Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191622179586Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aa08fa780d640e27469ae8f50308bba6ff18bcb9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191622179586Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75910481920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112245314048, + "active_bytes_peak_delta": 30840982528, + "after": { + "active_bytes_current": 75910481920, + "active_bytes_peak": 112245314048, + "allocated_bytes_current": 75910481920, + "allocated_bytes_peak": 112245314048, + "device": 4, + "inactive_split_bytes_current": 3902928896, + "inactive_split_bytes_peak": 14613085184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112604479488, + "reserved_bytes_peak": 112604479488, + "segment_count_current": 2886, + "segment_count_peak": 2886, + "timestamp_utc": "2026-08-26T19:16:22.179157Z", + "world_size": 8 + }, + "allocated_bytes_current": 75910481920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112245314048, + "allocated_bytes_peak_delta": 30840982528, + "before": { + "active_bytes_current": 75910481920, + "active_bytes_peak": 81404331520, + "allocated_bytes_current": 75910481920, + "allocated_bytes_peak": 81404331520, + "device": 4, + "inactive_split_bytes_current": 3900831744, + "inactive_split_bytes_peak": 14613085184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 81757470720, + "reserved_bytes_peak": 81757470720, + "segment_count_current": 2105, + "segment_count_peak": 2105, + "timestamp_utc": "2026-08-26T19:16:20.850803Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3902928896, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 14613085184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112604479488, + "reserved_bytes_current_delta": 30847008768, + "reserved_bytes_peak": 112604479488, + "reserved_bytes_peak_delta": 30847008768, + "segment_count_current": 2886, + "segment_count_current_delta": 781, + "segment_count_peak": 2886, + "segment_count_peak_delta": 781, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T191622179586Z.pickle", + "timestamp_utc": "2026-08-26T19:16:22.308193Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191622179586Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191622179586Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11088a8c73a8661f0aeccb489604364a2c99d80e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191622179586Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f4117892d0e4db41a9af0eed39c750219ed5ab14e1ff8f3e960ff6697bfebc7 +size 6988372 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191623321512Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191623321512Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cef5d6b17fbf43e4daa41996680f60be1d3a431f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191623321512Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75910481920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112245314048, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75910481920, + "active_bytes_peak": 112245314048, + "allocated_bytes_current": 75910481920, + "allocated_bytes_peak": 112245314048, + "device": 4, + "inactive_split_bytes_current": 3902928896, + "inactive_split_bytes_peak": 14613085184, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112604479488, + "reserved_bytes_peak": 112604479488, + "segment_count_current": 2886, + "segment_count_peak": 2886, + "timestamp_utc": "2026-08-26T19:16:23.321091Z", + "world_size": 8 + }, + "allocated_bytes_current": 75910481920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112245314048, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75910481920, + "active_bytes_peak": 112245314048, + "allocated_bytes_current": 75910481920, + "allocated_bytes_peak": 112245314048, + "device": 4, + "inactive_split_bytes_current": 3902928896, + "inactive_split_bytes_peak": 14613085184, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112604479488, + "reserved_bytes_peak": 112604479488, + "segment_count_current": 2886, + "segment_count_peak": 2886, + "timestamp_utc": "2026-08-26T19:16:22.324161Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3902928896, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14613085184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112604479488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112604479488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2886, + "segment_count_current_delta": 0, + "segment_count_peak": 2886, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T191623321512Z.pickle", + "timestamp_utc": "2026-08-26T19:16:23.449500Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191623321512Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191623321512Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e90095f058ec338920df11a62369458dae08e80 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191623321512Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf4ba342074c6b120f5baee69aa5e68b1bf5758d321efc6118562b23914ef972 +size 6951496 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191626684090Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191626684090Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a373eee1301ac4ccf3d6440247a54a05e561f352 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191626684090Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75927525376, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 112245314048, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75927525376, + "active_bytes_peak": 112245314048, + "allocated_bytes_current": 75927525376, + "allocated_bytes_peak": 112245314048, + "device": 4, + "inactive_split_bytes_current": 3976062976, + "inactive_split_bytes_peak": 14613085184, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115823607808, + "reserved_bytes_peak": 115823607808, + "segment_count_current": 3090, + "segment_count_peak": 3090, + "timestamp_utc": "2026-08-26T19:16:26.683584Z", + "world_size": 8 + }, + "allocated_bytes_current": 75927525376, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 112245314048, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75910481920, + "active_bytes_peak": 112245314048, + "allocated_bytes_current": 75910481920, + "allocated_bytes_peak": 112245314048, + "device": 4, + "inactive_split_bytes_current": 3902928896, + "inactive_split_bytes_peak": 14613085184, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112604479488, + "reserved_bytes_peak": 112604479488, + "segment_count_current": 2886, + "segment_count_peak": 2886, + "timestamp_utc": "2026-08-26T19:16:23.464762Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3976062976, + "inactive_split_bytes_current_delta": 73134080, + "inactive_split_bytes_peak": 14613085184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115823607808, + "reserved_bytes_current_delta": 3219128320, + "reserved_bytes_peak": 115823607808, + "reserved_bytes_peak_delta": 3219128320, + "segment_count_current": 3090, + "segment_count_current_delta": 204, + "segment_count_peak": 3090, + "segment_count_peak_delta": 204, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T191626684090Z.pickle", + "timestamp_utc": "2026-08-26T19:16:26.823398Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191626684090Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191626684090Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..59b4b7126db1c68cd14296fa796e01a7d8c44326 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191626684090Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28939ddac1ba961b3ef13640161a03f052225cedfe256a4afbd644e319d2cc6c +size 7509618 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191627204381Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191627204381Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ea9c96de53ad96a75a66386b19981f7c597aed56 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191627204381Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75927525376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112245314048, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75927525376, + "active_bytes_peak": 112245314048, + "allocated_bytes_current": 75927525376, + "allocated_bytes_peak": 112245314048, + "device": 4, + "inactive_split_bytes_current": 3976062976, + "inactive_split_bytes_peak": 14613085184, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115823607808, + "reserved_bytes_peak": 115823607808, + "segment_count_current": 3090, + "segment_count_peak": 3090, + "timestamp_utc": "2026-08-26T19:16:27.203944Z", + "world_size": 8 + }, + "allocated_bytes_current": 75927525376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112245314048, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75927525376, + "active_bytes_peak": 112245314048, + "allocated_bytes_current": 75927525376, + "allocated_bytes_peak": 112245314048, + "device": 4, + "inactive_split_bytes_current": 3976062976, + "inactive_split_bytes_peak": 14613085184, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115823607808, + "reserved_bytes_peak": 115823607808, + "segment_count_current": 3090, + "segment_count_peak": 3090, + "timestamp_utc": "2026-08-26T19:16:26.839934Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3976062976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14613085184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115823607808, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115823607808, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3090, + "segment_count_current_delta": 0, + "segment_count_peak": 3090, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T191627204381Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.224621Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191627204381Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191627204381Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8d0a20875a5bf537e10da6ec6aa644f39e42066e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191627204381Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9638b2cadc7f06981b4bef5f9091ca53da98bcd7e5e59f620672c87fbbb6504d +size 894212 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191627580105Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191627580105Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ab4f88d0d6ce15dc8527cf7fd03f142cc548e6ff --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191627580105Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75927525376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112245314048, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75927525376, + "active_bytes_peak": 112245314048, + "allocated_bytes_current": 75927525376, + "allocated_bytes_peak": 112245314048, + "device": 4, + "inactive_split_bytes_current": 3976062976, + "inactive_split_bytes_peak": 14613085184, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115823607808, + "reserved_bytes_peak": 115823607808, + "segment_count_current": 3090, + "segment_count_peak": 3090, + "timestamp_utc": "2026-08-26T19:16:27.579721Z", + "world_size": 8 + }, + "allocated_bytes_current": 75927525376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112245314048, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75927525376, + "active_bytes_peak": 112245314048, + "allocated_bytes_current": 75927525376, + "allocated_bytes_peak": 112245314048, + "device": 4, + "inactive_split_bytes_current": 3976062976, + "inactive_split_bytes_peak": 14613085184, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115823607808, + "reserved_bytes_peak": 115823607808, + "segment_count_current": 3090, + "segment_count_peak": 3090, + "timestamp_utc": "2026-08-26T19:16:27.226217Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3976062976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14613085184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115823607808, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115823607808, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3090, + "segment_count_current_delta": 0, + "segment_count_peak": 3090, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T191627580105Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.598559Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191627580105Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191627580105Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2b3ea93770b796c2f2671c76e508844466039a4b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191627580105Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb9479c274c5021c5bcbbc54a69113e365539f7cd119823f19dcf8f685e0d9f5 +size 894212 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191620696130Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191620696130Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f97af0937e4bb9715d364578fbd0ca0f60da40e3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191620696130Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75897316864, + "active_bytes_current_delta": 32343122432, + "active_bytes_peak": 81403138048, + "active_bytes_peak_delta": 37842652160, + "after": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 81403138048, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 81403138048, + "device": 5, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_peak": 14871617024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 81740693504, + "reserved_bytes_peak": 81740693504, + "segment_count_current": 2093, + "segment_count_peak": 2093, + "timestamp_utc": "2026-08-26T19:16:20.695702Z", + "world_size": 8 + }, + "allocated_bytes_current": 75897316864, + "allocated_bytes_current_delta": 32343122432, + "allocated_bytes_peak": 81403138048, + "allocated_bytes_peak_delta": 37842652160, + "before": { + "active_bytes_current": 43554194432, + "active_bytes_peak": 43560485888, + "allocated_bytes_current": 43554194432, + "allocated_bytes_peak": 43560485888, + "device": 5, + "inactive_split_bytes_current": 1255652352, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T19:15:53.657794Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_current_delta": 2635275776, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 13259355648, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 81740693504, + "reserved_bytes_current_delta": 36905680896, + "reserved_bytes_peak": 81740693504, + "reserved_bytes_peak_delta": 36905680896, + "segment_count_current": 2093, + "segment_count_current_delta": 1210, + "segment_count_peak": 2093, + "segment_count_peak_delta": 1210, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T191620696130Z.pickle", + "timestamp_utc": "2026-08-26T19:16:20.831852Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191620696130Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191620696130Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2100d9947904d6b9c11fb2be32862f3ed5f4d32f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191620696130Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1691b996cee24687f95c91ff75bc689ecd4bab1feab30649beb83d8bf7c34df +size 6992691 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191622179757Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191622179757Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8db5860130e3c06d5218a3c7601c1c62e4d26c19 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191622179757Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75897316864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112208827904, + "active_bytes_peak_delta": 30805689856, + "after": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 112208827904, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 112208827904, + "device": 5, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_peak": 14871617024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112583507968, + "reserved_bytes_peak": 112583507968, + "segment_count_current": 2863, + "segment_count_peak": 2863, + "timestamp_utc": "2026-08-26T19:16:22.179298Z", + "world_size": 8 + }, + "allocated_bytes_current": 75897316864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112208827904, + "allocated_bytes_peak_delta": 30805689856, + "before": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 81403138048, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 81403138048, + "device": 5, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_peak": 14871617024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 81740693504, + "reserved_bytes_peak": 81740693504, + "segment_count_current": 2093, + "segment_count_peak": 2093, + "timestamp_utc": "2026-08-26T19:16:20.852502Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112583507968, + "reserved_bytes_current_delta": 30842814464, + "reserved_bytes_peak": 112583507968, + "reserved_bytes_peak_delta": 30842814464, + "segment_count_current": 2863, + "segment_count_current_delta": 770, + "segment_count_peak": 2863, + "segment_count_peak_delta": 770, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T191622179757Z.pickle", + "timestamp_utc": "2026-08-26T19:16:22.309516Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191622179757Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191622179757Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..282074f5956bcce108caa3c1c015cd89709265b3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191622179757Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40fc949668fe92e5aa4eafc275c41d01ea32a3ea5b5cea8826099865e3e5f931 +size 6748998 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191623321688Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191623321688Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1b0c1e91b80e3985a6112db290a7d475310398f7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191623321688Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75897316864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112219198976, + "active_bytes_peak_delta": 10371072, + "after": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 112219198976, + "device": 5, + "inactive_split_bytes_current": 3893025280, + "inactive_split_bytes_peak": 14871617024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112583507968, + "reserved_bytes_peak": 112583507968, + "segment_count_current": 2863, + "segment_count_peak": 2863, + "timestamp_utc": "2026-08-26T19:16:23.321237Z", + "world_size": 8 + }, + "allocated_bytes_current": 75897316864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112219198976, + "allocated_bytes_peak_delta": 10371072, + "before": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 112208827904, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 112208827904, + "device": 5, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_peak": 14871617024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112583507968, + "reserved_bytes_peak": 112583507968, + "segment_count_current": 2863, + "segment_count_peak": 2863, + "timestamp_utc": "2026-08-26T19:16:22.325853Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3893025280, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112583507968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112583507968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2863, + "segment_count_current_delta": 0, + "segment_count_peak": 2863, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T191623321688Z.pickle", + "timestamp_utc": "2026-08-26T19:16:23.452470Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191623321688Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191623321688Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c0c364851c1c2dcff052c9f37239bd8d9a836830 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191623321688Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c618117173af0c7d0873fa642f4c307815f8cb07d12ed29cbfb86b0ef2ea2af5 +size 6711480 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191626684182Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191626684182Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1b3fb67a968835de0ae280b4a856f7541f5123fa --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191626684182Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75914360320, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 112219198976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 5, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:26.683685Z", + "world_size": 8 + }, + "allocated_bytes_current": 75914360320, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 112219198976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 112219198976, + "device": 5, + "inactive_split_bytes_current": 3893025280, + "inactive_split_bytes_peak": 14871617024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112583507968, + "reserved_bytes_peak": 112583507968, + "segment_count_current": 2863, + "segment_count_peak": 2863, + "timestamp_utc": "2026-08-26T19:16:23.468124Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_current_delta": 73134080, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115800539136, + "reserved_bytes_current_delta": 3217031168, + "reserved_bytes_peak": 115800539136, + "reserved_bytes_peak_delta": 3217031168, + "segment_count_current": 3075, + "segment_count_current_delta": 212, + "segment_count_peak": 3075, + "segment_count_peak_delta": 212, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T191626684182Z.pickle", + "timestamp_utc": "2026-08-26T19:16:26.823458Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191626684182Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191626684182Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fdb7e69fdc668cf354d852def861a3e4c4925bd7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191626684182Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbcd85085f7ed1f961872d354fdffb6e531cf61073b8e7dc5ce078f77ebc2b0c +size 7494045 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191627204500Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191627204500Z.json new file mode 100644 index 0000000000000000000000000000000000000000..149d28652694702859796d1888afb0f40a77395a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191627204500Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75914360320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112219198976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 5, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:27.204064Z", + "world_size": 8 + }, + "allocated_bytes_current": 75914360320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112219198976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 5, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:26.839740Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115800539136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115800539136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3075, + "segment_count_current_delta": 0, + "segment_count_peak": 3075, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T191627204500Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.224540Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191627204500Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191627204500Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..917add841ed826ee8caa0aaefce3c267e70b4b3b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191627204500Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12a4e87aa7e943a249f9d05266a72114b9752f5aa6a42038e528b1ea6c2dd631 +size 879599 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191627580178Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191627580178Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96dc896ff475876c0fe0871215a063ea809c5861 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191627580178Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75914360320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112219198976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 5, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:27.579801Z", + "world_size": 8 + }, + "allocated_bytes_current": 75914360320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112219198976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 5, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:27.226116Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 115800539136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115800539136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3075, + "segment_count_current_delta": 0, + "segment_count_peak": 3075, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T191627580178Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.598430Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191627580178Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191627580178Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d7cfa5d610b9b9b5cb28664c3df7850a8dd7a55 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191627580178Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a37e3ae4c3e4752a53353913fd0da34bb3374284a810cb558dc348231cf2297b +size 879599 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191620696046Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191620696046Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7ed607ac0f593499635c4b25f1152c87cee0c258 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191620696046Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75891794944, + "active_bytes_current_delta": 32337733120, + "active_bytes_peak": 81401857536, + "active_bytes_peak_delta": 37841504256, + "after": { + "active_bytes_current": 75891794944, + "active_bytes_peak": 81401857536, + "allocated_bytes_current": 75891794944, + "allocated_bytes_peak": 81401857536, + "device": 6, + "inactive_split_bytes_current": 3913227264, + "inactive_split_bytes_peak": 14831001600, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 81740693504, + "reserved_bytes_peak": 81740693504, + "segment_count_current": 2105, + "segment_count_peak": 2105, + "timestamp_utc": "2026-08-26T19:16:20.695572Z", + "world_size": 8 + }, + "allocated_bytes_current": 75891794944, + "allocated_bytes_current_delta": 32337733120, + "allocated_bytes_peak": 81401857536, + "allocated_bytes_peak_delta": 37841504256, + "before": { + "active_bytes_current": 43554061824, + "active_bytes_peak": 43560353280, + "allocated_bytes_current": 43554061824, + "allocated_bytes_peak": 43560353280, + "device": 6, + "inactive_split_bytes_current": 1255784960, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T19:15:53.657981Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3913227264, + "inactive_split_bytes_current_delta": 2657442304, + "inactive_split_bytes_peak": 14831001600, + "inactive_split_bytes_peak_delta": 13218740224, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 81740693504, + "reserved_bytes_current_delta": 36916166656, + "reserved_bytes_peak": 81740693504, + "reserved_bytes_peak_delta": 36916166656, + "segment_count_current": 2105, + "segment_count_current_delta": 1215, + "segment_count_peak": 2105, + "segment_count_peak_delta": 1215, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T191620696046Z.pickle", + "timestamp_utc": "2026-08-26T19:16:20.831999Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191620696046Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191620696046Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..90ebe56b2a2dad1bf534a69727c44737a4b89279 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191620696046Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37c9deb6d7b70cb0ac2affeaf6ed71bd89085f0972f937ae6c3052c5602a4def +size 7010929 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191622179570Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191622179570Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dce50996dfc00797309035963aa6fd14ee7a3c4b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191622179570Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75891794944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112202653184, + "active_bytes_peak_delta": 30800795648, + "after": { + "active_bytes_current": 75891794944, + "active_bytes_peak": 112202653184, + "allocated_bytes_current": 75891794944, + "allocated_bytes_peak": 112202653184, + "device": 6, + "inactive_split_bytes_current": 3915324416, + "inactive_split_bytes_peak": 14831001600, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112562536448, + "reserved_bytes_peak": 112562536448, + "segment_count_current": 2887, + "segment_count_peak": 2887, + "timestamp_utc": "2026-08-26T19:16:22.179150Z", + "world_size": 8 + }, + "allocated_bytes_current": 75891794944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112202653184, + "allocated_bytes_peak_delta": 30800795648, + "before": { + "active_bytes_current": 75891794944, + "active_bytes_peak": 81401857536, + "allocated_bytes_current": 75891794944, + "allocated_bytes_peak": 81401857536, + "device": 6, + "inactive_split_bytes_current": 3913227264, + "inactive_split_bytes_peak": 14831001600, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 81740693504, + "reserved_bytes_peak": 81740693504, + "segment_count_current": 2105, + "segment_count_peak": 2105, + "timestamp_utc": "2026-08-26T19:16:20.853076Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3915324416, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 14831001600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112562536448, + "reserved_bytes_current_delta": 30821842944, + "reserved_bytes_peak": 112562536448, + "reserved_bytes_peak_delta": 30821842944, + "segment_count_current": 2887, + "segment_count_current_delta": 782, + "segment_count_peak": 2887, + "segment_count_peak_delta": 782, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T191622179570Z.pickle", + "timestamp_utc": "2026-08-26T19:16:22.307781Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191622179570Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191622179570Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf9b3fe47ab51ae23d084ff0127a31d49a58f6c2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191622179570Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52d74a1d04b28d5fbb3525edd5b58ba6a29fca8d3dbe2c79e95a6d54f580bfee +size 7032461 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191623321511Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191623321511Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c19115cfdd84ff2c0b45e06255cabc51c679b458 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191623321511Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75891794944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112202653184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75891794944, + "active_bytes_peak": 112202653184, + "allocated_bytes_current": 75891794944, + "allocated_bytes_peak": 112202653184, + "device": 6, + "inactive_split_bytes_current": 3913227264, + "inactive_split_bytes_peak": 14831001600, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112562536448, + "reserved_bytes_peak": 112562536448, + "segment_count_current": 2887, + "segment_count_peak": 2887, + "timestamp_utc": "2026-08-26T19:16:23.321083Z", + "world_size": 8 + }, + "allocated_bytes_current": 75891794944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112202653184, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75891794944, + "active_bytes_peak": 112202653184, + "allocated_bytes_current": 75891794944, + "allocated_bytes_peak": 112202653184, + "device": 6, + "inactive_split_bytes_current": 3915324416, + "inactive_split_bytes_peak": 14831001600, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112562536448, + "reserved_bytes_peak": 112562536448, + "segment_count_current": 2887, + "segment_count_peak": 2887, + "timestamp_utc": "2026-08-26T19:16:22.324002Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3913227264, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 14831001600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112562536448, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112562536448, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2887, + "segment_count_current_delta": 0, + "segment_count_peak": 2887, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T191623321511Z.pickle", + "timestamp_utc": "2026-08-26T19:16:23.450784Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191623321511Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191623321511Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..edea53afa02ce535982d2096960257a11e271926 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191623321511Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6df8c2d76901b07297cba672b54b5d3f283d3a4a63d1e9c0aee4c61ec646ed0c +size 6995473 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191626684090Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191626684090Z.json new file mode 100644 index 0000000000000000000000000000000000000000..df6c6d6cc62fa982401a10d71ea7a77798fa0b04 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191626684090Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75908838400, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 112202653184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75908838400, + "active_bytes_peak": 112202653184, + "allocated_bytes_current": 75908838400, + "allocated_bytes_peak": 112202653184, + "device": 6, + "inactive_split_bytes_current": 3988458496, + "inactive_split_bytes_peak": 14831001600, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 115815219200, + "reserved_bytes_peak": 115815219200, + "segment_count_current": 3094, + "segment_count_peak": 3094, + "timestamp_utc": "2026-08-26T19:16:26.683583Z", + "world_size": 8 + }, + "allocated_bytes_current": 75908838400, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 112202653184, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75891794944, + "active_bytes_peak": 112202653184, + "allocated_bytes_current": 75891794944, + "allocated_bytes_peak": 112202653184, + "device": 6, + "inactive_split_bytes_current": 3913227264, + "inactive_split_bytes_peak": 14831001600, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112562536448, + "reserved_bytes_peak": 112562536448, + "segment_count_current": 2887, + "segment_count_peak": 2887, + "timestamp_utc": "2026-08-26T19:16:23.466298Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3988458496, + "inactive_split_bytes_current_delta": 75231232, + "inactive_split_bytes_peak": 14831001600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 115815219200, + "reserved_bytes_current_delta": 3252682752, + "reserved_bytes_peak": 115815219200, + "reserved_bytes_peak_delta": 3252682752, + "segment_count_current": 3094, + "segment_count_current_delta": 207, + "segment_count_peak": 3094, + "segment_count_peak_delta": 207, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T191626684090Z.pickle", + "timestamp_utc": "2026-08-26T19:16:26.823000Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191626684090Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191626684090Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d728bcc96ad8926f977300bd496bebe3608f3c49 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191626684090Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80caf1bb527732161c54e33cfdec04e9dfa86a4f80a98d4398b391b4fdbb7226 +size 7509822 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191627204371Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191627204371Z.json new file mode 100644 index 0000000000000000000000000000000000000000..366d278a5904a549dd5938ff33cad4918b3ca21e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191627204371Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75908838400, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112202653184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75908838400, + "active_bytes_peak": 112202653184, + "allocated_bytes_current": 75908838400, + "allocated_bytes_peak": 112202653184, + "device": 6, + "inactive_split_bytes_current": 3988458496, + "inactive_split_bytes_peak": 14831001600, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 115815219200, + "reserved_bytes_peak": 115815219200, + "segment_count_current": 3094, + "segment_count_peak": 3094, + "timestamp_utc": "2026-08-26T19:16:27.203938Z", + "world_size": 8 + }, + "allocated_bytes_current": 75908838400, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112202653184, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75908838400, + "active_bytes_peak": 112202653184, + "allocated_bytes_current": 75908838400, + "allocated_bytes_peak": 112202653184, + "device": 6, + "inactive_split_bytes_current": 3988458496, + "inactive_split_bytes_peak": 14831001600, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 115815219200, + "reserved_bytes_peak": 115815219200, + "segment_count_current": 3094, + "segment_count_peak": 3094, + "timestamp_utc": "2026-08-26T19:16:26.840188Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3988458496, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14831001600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 115815219200, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115815219200, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3094, + "segment_count_current_delta": 0, + "segment_count_peak": 3094, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T191627204371Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.224661Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191627204371Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191627204371Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2e6a4138f024aa6242bca09d1bb36352acbed48 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191627204371Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21bb9f41f42854cef040973603506716e2cf92e710b90df5c9e70e3eb0208b45 +size 894906 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191627580059Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191627580059Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b06c1efeb1d6dd0c28b80ada9bd7989151233ac0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191627580059Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75908838400, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112202653184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75908838400, + "active_bytes_peak": 112202653184, + "allocated_bytes_current": 75908838400, + "allocated_bytes_peak": 112202653184, + "device": 6, + "inactive_split_bytes_current": 3988458496, + "inactive_split_bytes_peak": 14831001600, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 115815219200, + "reserved_bytes_peak": 115815219200, + "segment_count_current": 3094, + "segment_count_peak": 3094, + "timestamp_utc": "2026-08-26T19:16:27.579677Z", + "world_size": 8 + }, + "allocated_bytes_current": 75908838400, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112202653184, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75908838400, + "active_bytes_peak": 112202653184, + "allocated_bytes_current": 75908838400, + "allocated_bytes_peak": 112202653184, + "device": 6, + "inactive_split_bytes_current": 3988458496, + "inactive_split_bytes_peak": 14831001600, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 115815219200, + "reserved_bytes_peak": 115815219200, + "segment_count_current": 3094, + "segment_count_peak": 3094, + "timestamp_utc": "2026-08-26T19:16:27.226253Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3988458496, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14831001600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 115815219200, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115815219200, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3094, + "segment_count_current_delta": 0, + "segment_count_peak": 3094, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T191627580059Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.598321Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191627580059Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191627580059Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7484ae99083ef2b307262a4e0783abd62436cd0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191627580059Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dd444e24c6c32afd765cca6812acb09af8b2751d65bcbdfe26f271703f98b0f +size 894906 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191620696058Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191620696058Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e39a53946c0a8e2a9f3818ff27d531eea58ff8fa --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191620696058Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75897316864, + "active_bytes_current_delta": 32343122432, + "active_bytes_peak": 81403138048, + "active_bytes_peak_delta": 37842652160, + "after": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 81403138048, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 81403138048, + "device": 7, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_peak": 14871617024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 81740693504, + "reserved_bytes_peak": 81740693504, + "segment_count_current": 2093, + "segment_count_peak": 2093, + "timestamp_utc": "2026-08-26T19:16:20.695612Z", + "world_size": 8 + }, + "allocated_bytes_current": 75897316864, + "allocated_bytes_current_delta": 32343122432, + "allocated_bytes_peak": 81403138048, + "allocated_bytes_peak_delta": 37842652160, + "before": { + "active_bytes_current": 43554194432, + "active_bytes_peak": 43560485888, + "allocated_bytes_current": 43554194432, + "allocated_bytes_peak": 43560485888, + "device": 7, + "inactive_split_bytes_current": 1255652352, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 44835012608, + "reserved_bytes_peak": 44835012608, + "segment_count_current": 883, + "segment_count_peak": 883, + "timestamp_utc": "2026-08-26T19:15:53.657945Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_current_delta": 2635275776, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 13259355648, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 81740693504, + "reserved_bytes_current_delta": 36905680896, + "reserved_bytes_peak": 81740693504, + "reserved_bytes_peak_delta": 36905680896, + "segment_count_current": 2093, + "segment_count_current_delta": 1210, + "segment_count_peak": 2093, + "segment_count_peak_delta": 1210, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T191620696058Z.pickle", + "timestamp_utc": "2026-08-26T19:16:20.829037Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191620696058Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191620696058Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..609b9d3a42ed4bfe3bcc7ba2d86bffc4bb112802 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191620696058Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d50d7c72eabb01841c53659f9d32901fd741c3626d9c060c79b5388eaa93b87 +size 6992691 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191622179602Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191622179602Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b3382a629449882f8bf9a07d23325d0bd8823bca --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191622179602Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75897316864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112208827904, + "active_bytes_peak_delta": 30805689856, + "after": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 112208827904, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 112208827904, + "device": 7, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_peak": 14871617024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112583507968, + "reserved_bytes_peak": 112583507968, + "segment_count_current": 2863, + "segment_count_peak": 2863, + "timestamp_utc": "2026-08-26T19:16:22.179183Z", + "world_size": 8 + }, + "allocated_bytes_current": 75897316864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112208827904, + "allocated_bytes_peak_delta": 30805689856, + "before": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 81403138048, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 81403138048, + "device": 7, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_peak": 14871617024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 81740693504, + "reserved_bytes_peak": 81740693504, + "segment_count_current": 2093, + "segment_count_peak": 2093, + "timestamp_utc": "2026-08-26T19:16:20.849812Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112583507968, + "reserved_bytes_current_delta": 30842814464, + "reserved_bytes_peak": 112583507968, + "reserved_bytes_peak_delta": 30842814464, + "segment_count_current": 2863, + "segment_count_current_delta": 770, + "segment_count_peak": 2863, + "segment_count_peak_delta": 770, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T191622179602Z.pickle", + "timestamp_utc": "2026-08-26T19:16:22.305985Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191622179602Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191622179602Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cb441d88cf8d148c2db95af2f66e94c6d55bee14 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191622179602Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7073e2527393c28fd4fece23cfb11a56ef450e7c4f2ee948360d8f586bd9979f +size 6748996 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191623321537Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191623321537Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dabe0be67cde72cf6a68e4204255f6469abd183b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191623321537Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75897316864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112219198976, + "active_bytes_peak_delta": 10371072, + "after": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 112219198976, + "device": 7, + "inactive_split_bytes_current": 3893025280, + "inactive_split_bytes_peak": 14871617024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112583507968, + "reserved_bytes_peak": 112583507968, + "segment_count_current": 2863, + "segment_count_peak": 2863, + "timestamp_utc": "2026-08-26T19:16:23.321119Z", + "world_size": 8 + }, + "allocated_bytes_current": 75897316864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112219198976, + "allocated_bytes_peak_delta": 10371072, + "before": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 112208827904, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 112208827904, + "device": 7, + "inactive_split_bytes_current": 3890928128, + "inactive_split_bytes_peak": 14871617024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112583507968, + "reserved_bytes_peak": 112583507968, + "segment_count_current": 2863, + "segment_count_peak": 2863, + "timestamp_utc": "2026-08-26T19:16:22.322426Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3893025280, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112583507968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112583507968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 2863, + "segment_count_current_delta": 0, + "segment_count_peak": 2863, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T191623321537Z.pickle", + "timestamp_utc": "2026-08-26T19:16:23.448400Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191623321537Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191623321537Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a5be7d34e4f39469454326e73a65fec2e979d68 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191623321537Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:398db2eb88d04d4d2a29a1c414160c778ec35ed7941cb388bf3fac3f9da81103 +size 6711478 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191626684245Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191626684245Z.json new file mode 100644 index 0000000000000000000000000000000000000000..84aa43218eb697465c37f6c9a9cebe881ca0e278 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191626684245Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75914360320, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 112219198976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 7, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:26.683656Z", + "world_size": 8 + }, + "allocated_bytes_current": 75914360320, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 112219198976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75897316864, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75897316864, + "allocated_bytes_peak": 112219198976, + "device": 7, + "inactive_split_bytes_current": 3893025280, + "inactive_split_bytes_peak": 14871617024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112583507968, + "reserved_bytes_peak": 112583507968, + "segment_count_current": 2863, + "segment_count_peak": 2863, + "timestamp_utc": "2026-08-26T19:16:23.463850Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_current_delta": 73134080, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115800539136, + "reserved_bytes_current_delta": 3217031168, + "reserved_bytes_peak": 115800539136, + "reserved_bytes_peak_delta": 3217031168, + "segment_count_current": 3075, + "segment_count_current_delta": 212, + "segment_count_peak": 3075, + "segment_count_peak_delta": 212, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T191626684245Z.pickle", + "timestamp_utc": "2026-08-26T19:16:26.823039Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191626684245Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191626684245Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..260b6a96b23703e052dc9d9661519fc372bffebc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191626684245Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62390baa32e37c60bf4892531891ca559ff175b30ffcfc9ddddb9d74dc32229e +size 7494047 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191627204412Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191627204412Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a7c030c459d4493e1a84b9f447776334f4f7f64 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191627204412Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75914360320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112219198976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 7, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:27.203976Z", + "world_size": 8 + }, + "allocated_bytes_current": 75914360320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112219198976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 7, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:26.839884Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115800539136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115800539136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3075, + "segment_count_current_delta": 0, + "segment_count_peak": 3075, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T191627204412Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.224321Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191627204412Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191627204412Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a160ba03ca16bc6518163ed054d84e3eb0b970bf --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191627204412Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b90d01a8f84b707ac4567ffe61cf0a1919485ffac0c00c0a69eb51236266747b +size 879601 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191627580116Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191627580116Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0645b0c8bdf64d9f53b55c551b0fc7d46a760015 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191627580116Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75914360320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 112219198976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 7, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:27.579730Z", + "world_size": 8 + }, + "allocated_bytes_current": 75914360320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 112219198976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75914360320, + "active_bytes_peak": 112219198976, + "allocated_bytes_current": 75914360320, + "allocated_bytes_peak": 112219198976, + "device": 7, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_peak": 14871617024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 115800539136, + "reserved_bytes_peak": 115800539136, + "segment_count_current": 3075, + "segment_count_peak": 3075, + "timestamp_utc": "2026-08-26T19:16:27.225909Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3966159360, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14871617024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115800539136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115800539136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3075, + "segment_count_current_delta": 0, + "segment_count_peak": 3075, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T191627580116Z.pickle", + "timestamp_utc": "2026-08-26T19:16:27.598263Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191627580116Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191627580116Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7c2ad4f8f74d7be4359ac128389846f085c18e3a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191627580116Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b4c85f8d35d45c443586a98c058ac9df97b86278f580437e0d6a9059e285d77 +size 879601 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/run.env b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..ec0501e3aff3d8c2f44577eebdf80884b40ec28a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-better-memory-dump-support +commit=c3a8a8a7 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.00 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=256 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/train.log b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..239af967ee67831604755445c70f5bb745d91185 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/train.log @@ -0,0 +1,2875 @@ +[19:15:23] WARN: ultraep-better-memory-dump-support lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / ultraep (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.00 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ c3a8a8a7 +============================================================== +W0826 19:15:25.201000 639730 torch/distributed/run.py:766] +W0826 19:15:25.201000 639730 torch/distributed/run.py:766] ***************************************** +W0826 19:15:25.201000 639730 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:15:25.201000 639730 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 2345 is connected to is connected to 7 is connected to is connected to 77 is connected to 77 peer ranks. peer ranks. 7 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77Expected number of connected peer ranks is : 77 + +7 + + +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 43 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank1]:[W826 19:15:43.309652534 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:15:43.309704174 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:15:43.309709000 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:15:43.309725969 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:15:43.309750094 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:15:43.309843786 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:15:43.310930872 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:15:43.459477207 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.692 seconds +time to initialize megatron (seconds): 6.809 +[after megatron is initialized] datetime: 2026-08-26 19:15:49 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 8372566016 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.final_layernorm.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.47.self_attention.linear_qkv.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.self_attention.linear_proj.weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.self_attention.linear_proj.weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.linear_proj.weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.self_attention.k_layernorm.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_proj.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.self_attention.linear_proj.weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.linear_proj.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.self_attention.linear_proj.weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.linear_proj.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.self_attention.linear_proj.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.k_layernorm.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.k_layernorm.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:15:53 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:15:53 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (4419.27, 4428.69) + train/valid/test-data-iterators-setup ..........: (72.52, 89.12) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:15:53 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:16:20] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 27040.2 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095325E+01 | loss scale: 1.0 | grad norm: 54.382 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 32768 | min safe capacity: 1.0000 |Number of parameters in transformer block in billions: 29.90 + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 72366.080078125 | max allocated: 77631.43701171875 | reserved: 77952.0 | max reserved: 77952.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 19:16:22] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 1483.7 | throughput per GPU (TFLOP/s/GPU): 60.1 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095441E+01 | loss scale: 1.0 | grad norm: 56.502 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 32768 | min safe capacity: 1.0000 | + [2026-08-26 19:16:23] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 1141.8 | throughput per GPU (TFLOP/s/GPU): 78.0 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008717E+01 | loss scale: 1.0 | grad norm: 180.384 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 32768 | min safe capacity: 1.0000 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:16:26] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 3362.5 | throughput per GPU (TFLOP/s/GPU): 26.5 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604643E+00 | loss scale: 1.0 | grad norm: 8.747 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 32768 | min safe capacity: 1.0000 | + [2026-08-26 19:16:27] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 520.6 | throughput per GPU (TFLOP/s/GPU): 171.2 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.309206E+00 | loss scale: 1.0 | grad norm: 7.408 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 32768 | min safe capacity: 1.0000 | + [2026-08-26 19:16:27] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 375.7 | throughput per GPU (TFLOP/s/GPU): 237.2 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.137036E+00 | loss scale: 1.0 | grad norm: 2.907 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 32768 | min safe capacity: 1.0000 | +[after training is done] datetime: 2026-08-26 19:16:27 +[rank0]:[W826 19:16:28.791771021 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 19:16:30.123896644 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 19:16:30.125482212 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 19:16:30.365151880 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 19:16:30.460410104 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 19:16:30.468306974 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 19:16:30.468597169 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 19:16:30.469721232 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:16:33.887000 639730 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:16:33.888000 639730 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 639821 closing signal SIGTERM +W0826 19:16:33.888000 639730 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 639822 closing signal SIGTERM +W0826 19:16:33.889000 639730 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 639823 closing signal SIGTERM +W0826 19:16:33.889000 639730 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 639824 closing signal SIGTERM +W0826 19:16:33.890000 639730 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 639825 closing signal SIGTERM +W0826 19:16:33.891000 639730 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 639826 closing signal SIGTERM +W0826 19:16:33.892000 639730 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 639827 closing signal SIGTERM +W0826 19:16:33.893000 639730 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 639828 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 639730 got signal: 15 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..e4746a022a8ef40115b01d95ff63032b9bb2c9c0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 2223 +1, 2239 +2, 2243 +3, 2259 +4, 2233 +5, 2239 +6, 2273 +7, 2241 +0, 10131 +1, 9951 +2, 10191 +3, 10251 +4, 10351 +5, 9951 +6, 10171 +7, 10431 +0, 47001 +1, 47073 +2, 47057 +3, 46969 +4, 47033 +5, 47079 +6, 47001 +7, 47029 +0, 48183 +1, 48185 +2, 48195 +3, 48187 +4, 48171 +5, 48225 +6, 48195 +7, 48185 +0, 48647 +1, 48649 +2, 48659 +3, 48621 +4, 48645 +5, 48659 +6, 48649 +7, 48659 +0, 48791 +1, 48793 +2, 48803 +3, 48765 +4, 48789 +5, 48803 +6, 48793 +7, 48803 +0, 49447 +1, 49447 +2, 49451 +3, 49421 +4, 49419 +5, 49451 +6, 49405 +7, 49451 +0, 57047 +1, 57103 +2, 57241 +3, 57103 +4, 57263 +5, 57261 +6, 57109 +7, 57317 +0, 85063 +1, 85121 +2, 85065 +3, 85093 +4, 85071 +5, 85071 +6, 85049 +7, 85089 +0, 85509 +1, 85535 +2, 85511 +3, 85535 +4, 85515 +5, 85511 +6, 85527 +7, 85511 +0, 85509 +1, 85535 +2, 85511 +3, 85535 +4, 85515 +5, 85511 +6, 85527 +7, 85511 +0, 85751 +1, 85783 +2, 85763 +3, 85819 +4, 85737 +5, 85711 +6, 85753 +7, 85715 +0, 86839 +1, 86863 +2, 86851 +3, 86835 +4, 86823 +5, 86821 +6, 86829 +7, 86841 +0, 98271 +1, 98285 +2, 98273 +3, 98273 +4, 98045 +5, 98273 +6, 97985 +7, 98557 +0, 116773 +1, 116783 +2, 116753 +3, 116773 +4, 116803 +5, 116753 +6, 116745 +7, 116753 +0, 113053 +1, 113049 +2, 112729 +3, 113689 +4, 112737 +5, 112897 +6, 113077 +7, 113169 +0, 121771 +1, 121787 +2, 121759 +3, 121803 +4, 121767 +5, 121759 +6, 121795 +7, 121759 +0, 120949 +1, 120965 +2, 120937 +3, 120981 +4, 120945 +5, 120937 +6, 120973 +7, 120937 +0, 117733 +1, 117749 +2, 117721 +3, 117765 +4, 117939 +5, 117721 +6, 117757 +7, 117721 +0, 115665 +1, 116069 +2, 116041 +3, 116085 +4, 116049 +5, 116041 +6, 115689 +7, 116041 +0, 1 +1, 4 +2, 4 +3, 4 +4, 116049 +5, 4 +6, 4 +7, 4 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191735910851Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191735910851Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ee6b6ed93d2129af9f916e97d4a3bc0aad16c44 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191735910851Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75890141696, + "active_bytes_current_delta": 32337888256, + "active_bytes_peak": 82627984896, + "active_bytes_peak_delta": 39069440000, + "after": { + "active_bytes_current": 75890141696, + "active_bytes_peak": 82627984896, + "allocated_bytes_current": 75890141696, + "allocated_bytes_peak": 82627984896, + "device": 0, + "inactive_split_bytes_current": 5907174912, + "inactive_split_bytes_peak": 13888238592, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 82994790400, + "reserved_bytes_peak": 82994790400, + "segment_count_current": 2126, + "segment_count_peak": 2126, + "timestamp_utc": "2026-08-26T19:17:35.910418Z", + "world_size": 8 + }, + "allocated_bytes_current": 75890141696, + "allocated_bytes_current_delta": 32337888256, + "allocated_bytes_peak": 82627984896, + "allocated_bytes_peak_delta": 39069440000, + "before": { + "active_bytes_current": 43552253440, + "active_bytes_peak": 43558544896, + "allocated_bytes_current": 43552253440, + "allocated_bytes_peak": 43558544896, + "device": 0, + "inactive_split_bytes_current": 1255496192, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 44822429696, + "reserved_bytes_peak": 44822429696, + "segment_count_current": 885, + "segment_count_peak": 885, + "timestamp_utc": "2026-08-26T19:17:09.289257Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5907174912, + "inactive_split_bytes_current_delta": 4651678720, + "inactive_split_bytes_peak": 13888238592, + "inactive_split_bytes_peak_delta": 12275977216, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 82994790400, + "reserved_bytes_current_delta": 38172360704, + "reserved_bytes_peak": 82994790400, + "reserved_bytes_peak_delta": 38172360704, + "segment_count_current": 2126, + "segment_count_current_delta": 1241, + "segment_count_peak": 2126, + "segment_count_peak_delta": 1241, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T191735910851Z.pickle", + "timestamp_utc": "2026-08-26T19:17:36.047695Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191735910851Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191735910851Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..89499d4d4638fbbe56c49d3d6f8fb1fc2262c0d2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191735910851Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0853ba69f1fe4fa783a2186cb79ae95bd3592be036bcd01725938a06f8616ec8 +size 7015163 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191737349283Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191737349283Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a0e22955199acd995c9a04b006820f8692ef193b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191737349283Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75890141696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113450855424, + "active_bytes_peak_delta": 30822870528, + "after": { + "active_bytes_current": 75890141696, + "active_bytes_peak": 113450855424, + "allocated_bytes_current": 75890141696, + "allocated_bytes_peak": 113450855424, + "device": 0, + "inactive_split_bytes_current": 5907174912, + "inactive_split_bytes_peak": 13888238592, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113860673536, + "reserved_bytes_peak": 113860673536, + "segment_count_current": 2836, + "segment_count_peak": 2836, + "timestamp_utc": "2026-08-26T19:17:37.348845Z", + "world_size": 8 + }, + "allocated_bytes_current": 75890141696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113450855424, + "allocated_bytes_peak_delta": 30822870528, + "before": { + "active_bytes_current": 75890141696, + "active_bytes_peak": 82627984896, + "allocated_bytes_current": 75890141696, + "allocated_bytes_peak": 82627984896, + "device": 0, + "inactive_split_bytes_current": 5907174912, + "inactive_split_bytes_peak": 13888238592, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 82994790400, + "reserved_bytes_peak": 82994790400, + "segment_count_current": 2126, + "segment_count_peak": 2126, + "timestamp_utc": "2026-08-26T19:17:36.068836Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5907174912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13888238592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113860673536, + "reserved_bytes_current_delta": 30865883136, + "reserved_bytes_peak": 113860673536, + "reserved_bytes_peak_delta": 30865883136, + "segment_count_current": 2836, + "segment_count_current_delta": 710, + "segment_count_peak": 2836, + "segment_count_peak_delta": 710, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T191737349283Z.pickle", + "timestamp_utc": "2026-08-26T19:17:37.480100Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191737349283Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191737349283Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d47e38c3b276fdab62f6fe9f5953e0ecd1fc5ba --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191737349283Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1f141d6cfba586c2065eafbed6562531459da33aed2bf277281f10a059242c3 +size 7024598 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191738517838Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191738517838Z.json new file mode 100644 index 0000000000000000000000000000000000000000..05485a8faa181db92662cef52f1465d2da379742 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191738517838Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75890141696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113481584128, + "active_bytes_peak_delta": 30728704, + "after": { + "active_bytes_current": 75890141696, + "active_bytes_peak": 113481584128, + "allocated_bytes_current": 75890141696, + "allocated_bytes_peak": 113481584128, + "device": 0, + "inactive_split_bytes_current": 5907174912, + "inactive_split_bytes_peak": 13888238592, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113862770688, + "reserved_bytes_peak": 113862770688, + "segment_count_current": 2837, + "segment_count_peak": 2837, + "timestamp_utc": "2026-08-26T19:17:38.517405Z", + "world_size": 8 + }, + "allocated_bytes_current": 75890141696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113481584128, + "allocated_bytes_peak_delta": 30728704, + "before": { + "active_bytes_current": 75890141696, + "active_bytes_peak": 113450855424, + "allocated_bytes_current": 75890141696, + "allocated_bytes_peak": 113450855424, + "device": 0, + "inactive_split_bytes_current": 5907174912, + "inactive_split_bytes_peak": 13888238592, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113860673536, + "reserved_bytes_peak": 113860673536, + "segment_count_current": 2836, + "segment_count_peak": 2836, + "timestamp_utc": "2026-08-26T19:17:37.496349Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5907174912, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13888238592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113862770688, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 113862770688, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 2837, + "segment_count_current_delta": 1, + "segment_count_peak": 2837, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T191738517838Z.pickle", + "timestamp_utc": "2026-08-26T19:17:38.650275Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191738517838Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191738517838Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e89d09ceaa591cbcd3a31bb0b58dd5761fb1dcc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191738517838Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7632bcda6e885b21ca14247f464661b87b4b48296816b600a2986b4df70d2eb6 +size 7030784 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191741821925Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191741821925Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6fd45bdf89ad80018348881f684c78ccd5e2611a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191741821925Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75907185152, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 113481584128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75907185152, + "active_bytes_peak": 113481584128, + "allocated_bytes_current": 75907185152, + "allocated_bytes_peak": 113481584128, + "device": 0, + "inactive_split_bytes_current": 5944657408, + "inactive_split_bytes_peak": 15461838848, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 119025958912, + "reserved_bytes_peak": 119025958912, + "segment_count_current": 3132, + "segment_count_peak": 3132, + "timestamp_utc": "2026-08-26T19:17:41.821429Z", + "world_size": 8 + }, + "allocated_bytes_current": 75907185152, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113481584128, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75890141696, + "active_bytes_peak": 113481584128, + "allocated_bytes_current": 75890141696, + "allocated_bytes_peak": 113481584128, + "device": 0, + "inactive_split_bytes_current": 5907174912, + "inactive_split_bytes_peak": 13888238592, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113862770688, + "reserved_bytes_peak": 113862770688, + "segment_count_current": 2837, + "segment_count_peak": 2837, + "timestamp_utc": "2026-08-26T19:17:38.665684Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5944657408, + "inactive_split_bytes_current_delta": 37482496, + "inactive_split_bytes_peak": 15461838848, + "inactive_split_bytes_peak_delta": 1573600256, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 119025958912, + "reserved_bytes_current_delta": 5163188224, + "reserved_bytes_peak": 119025958912, + "reserved_bytes_peak_delta": 5163188224, + "segment_count_current": 3132, + "segment_count_current_delta": 295, + "segment_count_peak": 3132, + "segment_count_peak_delta": 295, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T191741821925Z.pickle", + "timestamp_utc": "2026-08-26T19:17:41.961256Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191741821925Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191741821925Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8d1015d8ac402ca7d7c0663df47daf4767cbef8f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191741821925Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7e12319bba5f830a0c558ba6462a8abd3cf4f4ec756d061ff3fae4c9b0ca3ab +size 7504044 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191742352042Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191742352042Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d61887011874ef7770fe0f53bdc7534fdc495180 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191742352042Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75907185152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113481584128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75907185152, + "active_bytes_peak": 113481584128, + "allocated_bytes_current": 75907185152, + "allocated_bytes_peak": 113481584128, + "device": 0, + "inactive_split_bytes_current": 5944657408, + "inactive_split_bytes_peak": 15461838848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 119025958912, + "reserved_bytes_peak": 119025958912, + "segment_count_current": 3132, + "segment_count_peak": 3132, + "timestamp_utc": "2026-08-26T19:17:42.351623Z", + "world_size": 8 + }, + "allocated_bytes_current": 75907185152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113481584128, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75907185152, + "active_bytes_peak": 113481584128, + "allocated_bytes_current": 75907185152, + "allocated_bytes_peak": 113481584128, + "device": 0, + "inactive_split_bytes_current": 5944657408, + "inactive_split_bytes_peak": 15461838848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 119025958912, + "reserved_bytes_peak": 119025958912, + "segment_count_current": 3132, + "segment_count_peak": 3132, + "timestamp_utc": "2026-08-26T19:17:41.978107Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5944657408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15461838848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 119025958912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119025958912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3132, + "segment_count_current_delta": 0, + "segment_count_peak": 3132, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T191742352042Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.372176Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191742352042Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191742352042Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e6ac35e83948bdf74cc8e82c8d8787171a6147a3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191742352042Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b5be25f514e22f0c2b92b2a2f51768dde51adbdda798c92bfa38b4f6d2eb7b7 +size 898346 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191742733016Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191742733016Z.json new file mode 100644 index 0000000000000000000000000000000000000000..326f4b20ecaa21088c81b7f7cb3e2e466212d00f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191742733016Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75907185152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113481584128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 75907185152, + "active_bytes_peak": 113481584128, + "allocated_bytes_current": 75907185152, + "allocated_bytes_peak": 113481584128, + "device": 0, + "inactive_split_bytes_current": 5944657408, + "inactive_split_bytes_peak": 15461838848, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 119025958912, + "reserved_bytes_peak": 119025958912, + "segment_count_current": 3132, + "segment_count_peak": 3132, + "timestamp_utc": "2026-08-26T19:17:42.732633Z", + "world_size": 8 + }, + "allocated_bytes_current": 75907185152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113481584128, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 75907185152, + "active_bytes_peak": 113481584128, + "allocated_bytes_current": 75907185152, + "allocated_bytes_peak": 113481584128, + "device": 0, + "inactive_split_bytes_current": 5944657408, + "inactive_split_bytes_peak": 15461838848, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 119025958912, + "reserved_bytes_peak": 119025958912, + "segment_count_current": 3132, + "segment_count_peak": 3132, + "timestamp_utc": "2026-08-26T19:17:42.373799Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5944657408, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15461838848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 119025958912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119025958912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 3132, + "segment_count_current_delta": 0, + "segment_count_peak": 3132, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T191742733016Z.pickle", + "timestamp_utc": "2026-08-26T19:17:42.751190Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191742733016Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191742733016Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe8558510cb46410ffe6f22454e3d4785ecb8f6c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191742733016Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2087331768ac8b51761bedd704f36e5ac0b3510d54905d19c3f8b87d0fb15fc6 +size 898346 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191735910676Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191735910676Z.json new file mode 100644 index 0000000000000000000000000000000000000000..78df4512c144025f495fb8df1e73258eae0b5ffd --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191735910676Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 75901254656, + "active_bytes_current_delta": 32347258368, + "active_bytes_peak": 82620516352, + "active_bytes_peak_delta": 39060228608, + "after": { + "active_bytes_current": 75901254656, + "active_bytes_peak": 82620516352, + "allocated_bytes_current": 75901254656, + "allocated_bytes_peak": 82620516352, + "device": 1, + "inactive_split_bytes_current": 5912839168, + "inactive_split_bytes_peak": 14139202560, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 83022053376, + "reserved_bytes_peak": 83022053376, + "segment_count_current": 2125, + "segment_count_peak": 2125, + "timestamp_utc": "2026-08-26T19:17:35.910195Z", + "world_size": 8 + }, + "allocated_bytes_current": 75901254656, + "allocated_bytes_current_delta": 32347258368, + "allocated_bytes_peak": 82620516352, + "allocated_bytes_peak_delta": 39060228608, + "before": { + "active_bytes_current": 43553996288, + "active_bytes_peak": 43560287744, + "allocated_bytes_current": 43553996288, + "allocated_bytes_peak": 43560287744, + "device": 1, + "inactive_split_bytes_current": 1255850496, + "inactive_split_bytes_peak": 1612261376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 44824526848, + "reserved_bytes_peak": 44824526848, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-26T19:17:09.289532Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5912839168, + "inactive_split_bytes_current_delta": 4656988672, + "inactive_split_bytes_peak": 14139202560, + "inactive_split_bytes_peak_delta": 12526941184, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 83022053376, + "reserved_bytes_current_delta": 38197526528, + "reserved_bytes_peak": 83022053376, + "reserved_bytes_peak_delta": 38197526528, + "segment_count_current": 2125, + "segment_count_current_delta": 1235, + "segment_count_peak": 2125, + "segment_count_peak_delta": 1235, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T191735910676Z.pickle", + "timestamp_utc": "2026-08-26T19:17:36.046459Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191735910676Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191735910676Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..91e2d7edb7fdad862e04dc2ca23aa1dd9bc89c76 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191735910676Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d7a30cd2793e37ae069334e8f663e21c3b09aea08556331b17eaf42b6134e3 +size 7018634