diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a968aaceb65776bafb91f992cf36de50f77e4e66 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38993883648, + "active_bytes_current_delta": 15945322496, + "active_bytes_peak": 109588680192, + "active_bytes_peak_delta": 86527536128, + "after": { + "active_bytes_current": 38993883648, + "active_bytes_peak": 109588680192, + "allocated_bytes_current": 38993883648, + "allocated_bytes_peak": 109588680192, + "device": 4, + "inactive_split_bytes_current": 14059867648, + "inactive_split_bytes_peak": 27846074880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112956801024, + "reserved_bytes_peak": 112956801024, + "segment_count_current": 847, + "segment_count_peak": 847, + "timestamp_utc": "2026-08-26T19:23:45.258808Z", + "world_size": 8 + }, + "allocated_bytes_current": 38993883648, + "allocated_bytes_current_delta": 15945322496, + "allocated_bytes_peak": 109588680192, + "allocated_bytes_peak_delta": 86527536128, + "before": { + "active_bytes_current": 23048561152, + "active_bytes_peak": 23061144064, + "allocated_bytes_current": 23048561152, + "allocated_bytes_peak": 23061144064, + "device": 4, + "inactive_split_bytes_current": 389209600, + "inactive_split_bytes_peak": 883057664, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:23:17.758985Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 14059867648, + "inactive_split_bytes_current_delta": 13670658048, + "inactive_split_bytes_peak": 27846074880, + "inactive_split_bytes_peak_delta": 26963017216, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112956801024, + "reserved_bytes_current_delta": 87333797888, + "reserved_bytes_peak": 112956801024, + "reserved_bytes_peak_delta": 87333797888, + "segment_count_current": 847, + "segment_count_current_delta": 296, + "segment_count_peak": 847, + "segment_count_peak_delta": 296, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.pickle", + "timestamp_utc": "2026-08-26T19:23:45.320281Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c63147ec3ebc66a87fb7cdc6ef14992bd43ecb0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83a7120506693b5adcf9d91a1bca38a7b521d79c9f2046a36149f5c375e2c804 +size 3054015 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dbef656b26ed25e3402606e79c3452f2ce30f4e9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38993883648, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123802562048, + "active_bytes_peak_delta": 14213881856, + "after": { + "active_bytes_current": 38993883648, + "active_bytes_peak": 123802562048, + "allocated_bytes_current": 38993883648, + "allocated_bytes_peak": 123802562048, + "device": 4, + "inactive_split_bytes_current": 14059867648, + "inactive_split_bytes_peak": 27846074880, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127500550144, + "reserved_bytes_peak": 127500550144, + "segment_count_current": 859, + "segment_count_peak": 859, + "timestamp_utc": "2026-08-26T19:23:45.929126Z", + "world_size": 8 + }, + "allocated_bytes_current": 38993883648, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123802562048, + "allocated_bytes_peak_delta": 14213881856, + "before": { + "active_bytes_current": 38993883648, + "active_bytes_peak": 109588680192, + "allocated_bytes_current": 38993883648, + "allocated_bytes_peak": 109588680192, + "device": 4, + "inactive_split_bytes_current": 14059867648, + "inactive_split_bytes_peak": 27846074880, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112956801024, + "reserved_bytes_peak": 112956801024, + "segment_count_current": 847, + "segment_count_peak": 847, + "timestamp_utc": "2026-08-26T19:23:45.329151Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 14059867648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 27846074880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127500550144, + "reserved_bytes_current_delta": 14543749120, + "reserved_bytes_peak": 127500550144, + "reserved_bytes_peak_delta": 14543749120, + "segment_count_current": 859, + "segment_count_current_delta": 12, + "segment_count_peak": 859, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.pickle", + "timestamp_utc": "2026-08-26T19:23:45.980787Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3eff28689f3a92a3978437d0e51b4e9bb7bbb3d9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc4f989a013e267c2cfaf9ce568bff5374f09480dedd764567cb3e2664e09bdb +size 3049002 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a94d16731eabb741759a4f2c30c52487ee78fd1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38993883648, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123803930112, + "active_bytes_peak_delta": 1368064, + "after": { + "active_bytes_current": 38993883648, + "active_bytes_peak": 123803930112, + "allocated_bytes_current": 38993883648, + "allocated_bytes_peak": 123803930112, + "device": 4, + "inactive_split_bytes_current": 14059867648, + "inactive_split_bytes_peak": 27846074880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 127500550144, + "reserved_bytes_peak": 127500550144, + "segment_count_current": 859, + "segment_count_peak": 859, + "timestamp_utc": "2026-08-26T19:23:46.587185Z", + "world_size": 8 + }, + "allocated_bytes_current": 38993883648, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123803930112, + "allocated_bytes_peak_delta": 1368064, + "before": { + "active_bytes_current": 38993883648, + "active_bytes_peak": 123802562048, + "allocated_bytes_current": 38993883648, + "allocated_bytes_peak": 123802562048, + "device": 4, + "inactive_split_bytes_current": 14059867648, + "inactive_split_bytes_peak": 27846074880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 127500550144, + "reserved_bytes_peak": 127500550144, + "segment_count_current": 859, + "segment_count_peak": 859, + "timestamp_utc": "2026-08-26T19:23:45.986674Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 14059867648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 27846074880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 127500550144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127500550144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 859, + "segment_count_current_delta": 0, + "segment_count_peak": 859, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.pickle", + "timestamp_utc": "2026-08-26T19:23:46.638228Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..51783b7504cabef6ebbbf582eb3b188c8cdfe254 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a31cb9f06e39cd018076e2932e645017551c8ecdf23505ffa7b8d84aae3810c2 +size 3065016 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b36dff4a2fd9ea2b73a1e06edf7488c6dac10a2d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39011844608, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 123803930112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39011844608, + "active_bytes_peak": 123803930112, + "allocated_bytes_current": 39011844608, + "allocated_bytes_peak": 123803930112, + "device": 4, + "inactive_split_bytes_current": 16214556160, + "inactive_split_bytes_peak": 39152205824, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 140314148864, + "reserved_bytes_peak": 140314148864, + "segment_count_current": 970, + "segment_count_peak": 970, + "timestamp_utc": "2026-08-26T19:23:48.442889Z", + "world_size": 8 + }, + "allocated_bytes_current": 39011844608, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 123803930112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 38993883648, + "active_bytes_peak": 123803930112, + "allocated_bytes_current": 38993883648, + "allocated_bytes_peak": 123803930112, + "device": 4, + "inactive_split_bytes_current": 14059867648, + "inactive_split_bytes_peak": 27846074880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 127500550144, + "reserved_bytes_peak": 127500550144, + "segment_count_current": 859, + "segment_count_peak": 859, + "timestamp_utc": "2026-08-26T19:23:46.644129Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16214556160, + "inactive_split_bytes_current_delta": 2154688512, + "inactive_split_bytes_peak": 39152205824, + "inactive_split_bytes_peak_delta": 11306130944, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 140314148864, + "reserved_bytes_current_delta": 12813598720, + "reserved_bytes_peak": 140314148864, + "reserved_bytes_peak_delta": 12813598720, + "segment_count_current": 970, + "segment_count_current_delta": 111, + "segment_count_peak": 970, + "segment_count_peak_delta": 111, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.pickle", + "timestamp_utc": "2026-08-26T19:23:48.501132Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29623740fa82745c26bc8696b03f5f7aa1c8c8f0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17efcabdc174de09d2a136bf51ce42521716d10ed82c4020b784e72e341b1763 +size 3226224 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d7feedd13b0c784393c3537b1a36038efd95129f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39011844608, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123803930112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39011844608, + "active_bytes_peak": 123803930112, + "allocated_bytes_current": 39011844608, + "allocated_bytes_peak": 123803930112, + "device": 4, + "inactive_split_bytes_current": 16214556160, + "inactive_split_bytes_peak": 39152205824, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 140314148864, + "reserved_bytes_peak": 140314148864, + "segment_count_current": 970, + "segment_count_peak": 970, + "timestamp_utc": "2026-08-26T19:23:49.089210Z", + "world_size": 8 + }, + "allocated_bytes_current": 39011844608, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123803930112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39011844608, + "active_bytes_peak": 123803930112, + "allocated_bytes_current": 39011844608, + "allocated_bytes_peak": 123803930112, + "device": 4, + "inactive_split_bytes_current": 16214556160, + "inactive_split_bytes_peak": 39152205824, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 140314148864, + "reserved_bytes_peak": 140314148864, + "segment_count_current": 970, + "segment_count_peak": 970, + "timestamp_utc": "2026-08-26T19:23:48.508069Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16214556160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 39152205824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 140314148864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 140314148864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 970, + "segment_count_current_delta": 0, + "segment_count_peak": 970, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.pickle", + "timestamp_utc": "2026-08-26T19:23:49.097417Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c7bbc586f5538c469d8902d2af3aca88d7286778 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cec6e5732ef24b557f71d1208d1d83a838a2c444e47941d558eb4b8c6d85b91a +size 370864 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42bf3d4ea20fb4a933ed590bda74d5c669946562 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39011844608, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123803930112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39011844608, + "active_bytes_peak": 123803930112, + "allocated_bytes_current": 39011844608, + "allocated_bytes_peak": 123803930112, + "device": 4, + "inactive_split_bytes_current": 16214556160, + "inactive_split_bytes_peak": 39152205824, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 140314148864, + "reserved_bytes_peak": 140314148864, + "segment_count_current": 970, + "segment_count_peak": 970, + "timestamp_utc": "2026-08-26T19:23:49.684045Z", + "world_size": 8 + }, + "allocated_bytes_current": 39011844608, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123803930112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39011844608, + "active_bytes_peak": 123803930112, + "allocated_bytes_current": 39011844608, + "allocated_bytes_peak": 123803930112, + "device": 4, + "inactive_split_bytes_current": 16214556160, + "inactive_split_bytes_peak": 39152205824, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 140314148864, + "reserved_bytes_peak": 140314148864, + "segment_count_current": 970, + "segment_count_peak": 970, + "timestamp_utc": "2026-08-26T19:23:49.098450Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16214556160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 39152205824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 140314148864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 140314148864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 970, + "segment_count_current_delta": 0, + "segment_count_peak": 970, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.pickle", + "timestamp_utc": "2026-08-26T19:23:49.692107Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c099878c9913cffde0b5931e583b6517deb6f290 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50d5a1c42eda960f09f3c2d51a8efd01aafd7a1c7a2f33260dd3fb2f9ea3d71a +size 370864 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ea3cb4390a1dd2ec6891ed805cb9922783085e7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38984414720, + "active_bytes_current_delta": 15931054080, + "active_bytes_peak": 109604034048, + "active_bytes_peak_delta": 86538090496, + "after": { + "active_bytes_current": 38984414720, + "active_bytes_peak": 109604034048, + "allocated_bytes_current": 38984414720, + "allocated_bytes_peak": 109604034048, + "device": 5, + "inactive_split_bytes_current": 13970770432, + "inactive_split_bytes_peak": 27816583680, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112885497856, + "reserved_bytes_peak": 112885497856, + "segment_count_current": 844, + "segment_count_peak": 844, + "timestamp_utc": "2026-08-26T19:23:45.258776Z", + "world_size": 8 + }, + "allocated_bytes_current": 38984414720, + "allocated_bytes_current_delta": 15931054080, + "allocated_bytes_peak": 109604034048, + "allocated_bytes_peak_delta": 86538090496, + "before": { + "active_bytes_current": 23053360640, + "active_bytes_peak": 23065943552, + "allocated_bytes_current": 23053360640, + "allocated_bytes_peak": 23065943552, + "device": 5, + "inactive_split_bytes_current": 474587648, + "inactive_split_bytes_peak": 883057664, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:23:17.759118Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 13970770432, + "inactive_split_bytes_current_delta": 13496182784, + "inactive_split_bytes_peak": 27816583680, + "inactive_split_bytes_peak_delta": 26933526016, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112885497856, + "reserved_bytes_current_delta": 87262494720, + "reserved_bytes_peak": 112885497856, + "reserved_bytes_peak_delta": 87262494720, + "segment_count_current": 844, + "segment_count_current_delta": 293, + "segment_count_peak": 844, + "segment_count_peak_delta": 293, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.pickle", + "timestamp_utc": "2026-08-26T19:23:45.319604Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f6901f6a7fc6c1f0b004bfd9de44c348c1f3a1d1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9341d9d5fa4db59064b5f59b8a55b4c55fde01c00f22771fd9bf802548f669d5 +size 3011418 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2262de12f274b54a49b0bb72cdaf7374bb366ebf --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38984414720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123775942144, + "active_bytes_peak_delta": 14171908096, + "after": { + "active_bytes_current": 38984414720, + "active_bytes_peak": 123775942144, + "allocated_bytes_current": 38984414720, + "allocated_bytes_peak": 123775942144, + "device": 5, + "inactive_split_bytes_current": 13970770432, + "inactive_split_bytes_peak": 27816583680, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 127425052672, + "reserved_bytes_peak": 127425052672, + "segment_count_current": 854, + "segment_count_peak": 854, + "timestamp_utc": "2026-08-26T19:23:45.929130Z", + "world_size": 8 + }, + "allocated_bytes_current": 38984414720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123775942144, + "allocated_bytes_peak_delta": 14171908096, + "before": { + "active_bytes_current": 38984414720, + "active_bytes_peak": 109604034048, + "allocated_bytes_current": 38984414720, + "allocated_bytes_peak": 109604034048, + "device": 5, + "inactive_split_bytes_current": 13970770432, + "inactive_split_bytes_peak": 27816583680, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112885497856, + "reserved_bytes_peak": 112885497856, + "segment_count_current": 844, + "segment_count_peak": 844, + "timestamp_utc": "2026-08-26T19:23:45.328437Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 13970770432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 27816583680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 127425052672, + "reserved_bytes_current_delta": 14539554816, + "reserved_bytes_peak": 127425052672, + "reserved_bytes_peak_delta": 14539554816, + "segment_count_current": 854, + "segment_count_current_delta": 10, + "segment_count_peak": 854, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.pickle", + "timestamp_utc": "2026-08-26T19:23:45.980410Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a81e63439e79365375b1171fb37ad2b9beddf548 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ed801a69e72e5224d35aa1d873046563c8e396d2748ad13c4ea89e09b92a6a6 +size 2977330 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c128c798a6bfcf7210493d73e6d25ed4745016aa --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38984414720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123783904768, + "active_bytes_peak_delta": 7962624, + "after": { + "active_bytes_current": 38984414720, + "active_bytes_peak": 123783904768, + "allocated_bytes_current": 38984414720, + "allocated_bytes_peak": 123783904768, + "device": 5, + "inactive_split_bytes_current": 13970770432, + "inactive_split_bytes_peak": 27816583680, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 127425052672, + "reserved_bytes_peak": 127425052672, + "segment_count_current": 854, + "segment_count_peak": 854, + "timestamp_utc": "2026-08-26T19:23:46.587182Z", + "world_size": 8 + }, + "allocated_bytes_current": 38984414720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123783904768, + "allocated_bytes_peak_delta": 7962624, + "before": { + "active_bytes_current": 38984414720, + "active_bytes_peak": 123775942144, + "allocated_bytes_current": 38984414720, + "allocated_bytes_peak": 123775942144, + "device": 5, + "inactive_split_bytes_current": 13970770432, + "inactive_split_bytes_peak": 27816583680, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 127425052672, + "reserved_bytes_peak": 127425052672, + "segment_count_current": 854, + "segment_count_peak": 854, + "timestamp_utc": "2026-08-26T19:23:45.986292Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 13970770432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 27816583680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 127425052672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127425052672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 854, + "segment_count_current_delta": 0, + "segment_count_peak": 854, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.pickle", + "timestamp_utc": "2026-08-26T19:23:46.638582Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b103e4a247a6fad62d44897530aae5eca4c17c7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddcccc20560180b7aa3d3f3772e82fdc11a13e38838401f8f6f5aec5b52d1e2f +size 2954732 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96d9e3cbdac853afe0da55bd78462905401fae28 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39002375680, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 123783904768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39002375680, + "active_bytes_peak": 123783904768, + "allocated_bytes_current": 39002375680, + "allocated_bytes_peak": 123783904768, + "device": 5, + "inactive_split_bytes_current": 16125458944, + "inactive_split_bytes_peak": 39063108608, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 140215582720, + "reserved_bytes_peak": 140215582720, + "segment_count_current": 954, + "segment_count_peak": 954, + "timestamp_utc": "2026-08-26T19:23:48.442887Z", + "world_size": 8 + }, + "allocated_bytes_current": 39002375680, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 123783904768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 38984414720, + "active_bytes_peak": 123783904768, + "allocated_bytes_current": 38984414720, + "allocated_bytes_peak": 123783904768, + "device": 5, + "inactive_split_bytes_current": 13970770432, + "inactive_split_bytes_peak": 27816583680, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 127425052672, + "reserved_bytes_peak": 127425052672, + "segment_count_current": 854, + "segment_count_peak": 854, + "timestamp_utc": "2026-08-26T19:23:46.644412Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16125458944, + "inactive_split_bytes_current_delta": 2154688512, + "inactive_split_bytes_peak": 39063108608, + "inactive_split_bytes_peak_delta": 11246524928, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 140215582720, + "reserved_bytes_current_delta": 12790530048, + "reserved_bytes_peak": 140215582720, + "reserved_bytes_peak_delta": 12790530048, + "segment_count_current": 954, + "segment_count_current_delta": 100, + "segment_count_peak": 954, + "segment_count_peak_delta": 100, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.pickle", + "timestamp_utc": "2026-08-26T19:23:48.500293Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2c87d7d4dee1de6bca2ca1f18d2aa3204330a458 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4fb213e3645e5468f55929eff49a7c322a3e786f17fee99dfde3863d7ee5289 +size 3107068 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.json new file mode 100644 index 0000000000000000000000000000000000000000..994cc2668b0114bfeacda2e15871aac7fd0469cc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39002375680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123783904768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39002375680, + "active_bytes_peak": 123783904768, + "allocated_bytes_current": 39002375680, + "allocated_bytes_peak": 123783904768, + "device": 5, + "inactive_split_bytes_current": 16125458944, + "inactive_split_bytes_peak": 39063108608, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 140215582720, + "reserved_bytes_peak": 140215582720, + "segment_count_current": 954, + "segment_count_peak": 954, + "timestamp_utc": "2026-08-26T19:23:49.089156Z", + "world_size": 8 + }, + "allocated_bytes_current": 39002375680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123783904768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39002375680, + "active_bytes_peak": 123783904768, + "allocated_bytes_current": 39002375680, + "allocated_bytes_peak": 123783904768, + "device": 5, + "inactive_split_bytes_current": 16125458944, + "inactive_split_bytes_peak": 39063108608, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 140215582720, + "reserved_bytes_peak": 140215582720, + "segment_count_current": 954, + "segment_count_peak": 954, + "timestamp_utc": "2026-08-26T19:23:48.507139Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16125458944, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 39063108608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 140215582720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 140215582720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 954, + "segment_count_current_delta": 0, + "segment_count_peak": 954, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.pickle", + "timestamp_utc": "2026-08-26T19:23:49.097290Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6c837c911b1a06abe90e8218b28bc178f3ceb9e6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86ee3b1e2b4ba11de5e21e6070dd5cc3455d6c71933b42b8b11cb08a5089ce7a +size 342390 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f3dcee81a3b2f8e60054c07e5940a471768b5ba7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39002375680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123783904768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39002375680, + "active_bytes_peak": 123783904768, + "allocated_bytes_current": 39002375680, + "allocated_bytes_peak": 123783904768, + "device": 5, + "inactive_split_bytes_current": 16125458944, + "inactive_split_bytes_peak": 39063108608, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 140215582720, + "reserved_bytes_peak": 140215582720, + "segment_count_current": 954, + "segment_count_peak": 954, + "timestamp_utc": "2026-08-26T19:23:49.684055Z", + "world_size": 8 + }, + "allocated_bytes_current": 39002375680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123783904768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39002375680, + "active_bytes_peak": 123783904768, + "allocated_bytes_current": 39002375680, + "allocated_bytes_peak": 123783904768, + "device": 5, + "inactive_split_bytes_current": 16125458944, + "inactive_split_bytes_peak": 39063108608, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 140215582720, + "reserved_bytes_peak": 140215582720, + "segment_count_current": 954, + "segment_count_peak": 954, + "timestamp_utc": "2026-08-26T19:23:49.098305Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16125458944, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 39063108608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 140215582720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 140215582720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 954, + "segment_count_current_delta": 0, + "segment_count_peak": 954, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.pickle", + "timestamp_utc": "2026-08-26T19:23:49.691896Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ccddaa517fbd5f8a875b639197d8ac4e3de9cb10 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e269c98f92de8a97b417be2464a4e702baebd152c70326d4eee25ebc3edef8f +size 342390 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6212180055e0f1e431a368a74c786de69a712840 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38969498112, + "active_bytes_current_delta": 15923573248, + "active_bytes_peak": 109581087232, + "active_bytes_peak_delta": 86522579456, + "after": { + "active_bytes_current": 38969498112, + "active_bytes_peak": 109581087232, + "allocated_bytes_current": 38969498112, + "allocated_bytes_peak": 109581087232, + "device": 6, + "inactive_split_bytes_current": 14163944960, + "inactive_split_bytes_peak": 27855711232, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 110880620544, + "reserved_bytes_peak": 110880620544, + "segment_count_current": 851, + "segment_count_peak": 851, + "timestamp_utc": "2026-08-26T19:23:45.258693Z", + "world_size": 8 + }, + "allocated_bytes_current": 38969498112, + "allocated_bytes_current_delta": 15923573248, + "allocated_bytes_peak": 109581087232, + "allocated_bytes_peak_delta": 86522579456, + "before": { + "active_bytes_current": 23045924864, + "active_bytes_peak": 23058507776, + "allocated_bytes_current": 23045924864, + "allocated_bytes_peak": 23058507776, + "device": 6, + "inactive_split_bytes_current": 370874368, + "inactive_split_bytes_peak": 883057664, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 25761415168, + "reserved_bytes_peak": 25761415168, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-26T19:23:17.759100Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 14163944960, + "inactive_split_bytes_current_delta": 13793070592, + "inactive_split_bytes_peak": 27855711232, + "inactive_split_bytes_peak_delta": 26972653568, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 110880620544, + "reserved_bytes_current_delta": 85119205376, + "reserved_bytes_peak": 110880620544, + "reserved_bytes_peak_delta": 85119205376, + "segment_count_current": 851, + "segment_count_current_delta": 294, + "segment_count_peak": 851, + "segment_count_peak_delta": 294, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.pickle", + "timestamp_utc": "2026-08-26T19:23:45.318867Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b17146f8b8d682878b32e1fe3c0155a64a150c79 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab9d452f6c8ec9d84dfc01c9f6afa1d0424918b0ff568ccb7d0cf88afb520f03 +size 3059007 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70754b767d298a9a654a9c8444d6f04759752d20 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38969498112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123804231168, + "active_bytes_peak_delta": 14223143936, + "after": { + "active_bytes_current": 38969498112, + "active_bytes_peak": 123804231168, + "allocated_bytes_current": 38969498112, + "allocated_bytes_peak": 123804231168, + "device": 6, + "inactive_split_bytes_current": 14163944960, + "inactive_split_bytes_peak": 27855711232, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 127641059328, + "reserved_bytes_peak": 127641059328, + "segment_count_current": 866, + "segment_count_peak": 866, + "timestamp_utc": "2026-08-26T19:23:45.929006Z", + "world_size": 8 + }, + "allocated_bytes_current": 38969498112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123804231168, + "allocated_bytes_peak_delta": 14223143936, + "before": { + "active_bytes_current": 38969498112, + "active_bytes_peak": 109581087232, + "allocated_bytes_current": 38969498112, + "allocated_bytes_peak": 109581087232, + "device": 6, + "inactive_split_bytes_current": 14163944960, + "inactive_split_bytes_peak": 27855711232, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 110880620544, + "reserved_bytes_peak": 110880620544, + "segment_count_current": 851, + "segment_count_peak": 851, + "timestamp_utc": "2026-08-26T19:23:45.327625Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 14163944960, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 27855711232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 127641059328, + "reserved_bytes_current_delta": 16760438784, + "reserved_bytes_peak": 127641059328, + "reserved_bytes_peak_delta": 16760438784, + "segment_count_current": 866, + "segment_count_current_delta": 15, + "segment_count_peak": 866, + "segment_count_peak_delta": 15, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.pickle", + "timestamp_utc": "2026-08-26T19:23:45.980292Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..69895299b5639185bac1052e0bcdc250bfca6c1e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1e36a3bd906118e6570f5b6d01261ce332954e8ca22ee517fec097c83996e02 +size 2973374 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.json new file mode 100644 index 0000000000000000000000000000000000000000..671269308a28fab8ad28643968541c7a8da6a567 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38969498112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123805369856, + "active_bytes_peak_delta": 1138688, + "after": { + "active_bytes_current": 38969498112, + "active_bytes_peak": 123805369856, + "allocated_bytes_current": 38969498112, + "allocated_bytes_peak": 123805369856, + "device": 6, + "inactive_split_bytes_current": 14163944960, + "inactive_split_bytes_peak": 27855711232, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 127641059328, + "reserved_bytes_peak": 127641059328, + "segment_count_current": 866, + "segment_count_peak": 866, + "timestamp_utc": "2026-08-26T19:23:46.587092Z", + "world_size": 8 + }, + "allocated_bytes_current": 38969498112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123805369856, + "allocated_bytes_peak_delta": 1138688, + "before": { + "active_bytes_current": 38969498112, + "active_bytes_peak": 123804231168, + "allocated_bytes_current": 38969498112, + "allocated_bytes_peak": 123804231168, + "device": 6, + "inactive_split_bytes_current": 14163944960, + "inactive_split_bytes_peak": 27855711232, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 127641059328, + "reserved_bytes_peak": 127641059328, + "segment_count_current": 866, + "segment_count_peak": 866, + "timestamp_utc": "2026-08-26T19:23:45.986039Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 14163944960, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 27855711232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 127641059328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127641059328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 866, + "segment_count_current_delta": 0, + "segment_count_peak": 866, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.pickle", + "timestamp_utc": "2026-08-26T19:23:46.637391Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a409932b31eab10c371f6c6657f371ef4384b13a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:842e23b86e4332c4ad01fd6a72563c2fb7ea3e22c09e87c938975ad7e18151a6 +size 2972521 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ea167d9b59696e3b5e95c683c1b6f1918844d4f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38987459072, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 123805369856, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 38987459072, + "active_bytes_peak": 123805369856, + "allocated_bytes_current": 38987459072, + "allocated_bytes_peak": 123805369856, + "device": 6, + "inactive_split_bytes_current": 16318633472, + "inactive_split_bytes_peak": 39258380288, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 140393840640, + "reserved_bytes_peak": 140393840640, + "segment_count_current": 979, + "segment_count_peak": 979, + "timestamp_utc": "2026-08-26T19:23:48.442740Z", + "world_size": 8 + }, + "allocated_bytes_current": 38987459072, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 123805369856, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 38969498112, + "active_bytes_peak": 123805369856, + "allocated_bytes_current": 38969498112, + "allocated_bytes_peak": 123805369856, + "device": 6, + "inactive_split_bytes_current": 14163944960, + "inactive_split_bytes_peak": 27855711232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 127641059328, + "reserved_bytes_peak": 127641059328, + "segment_count_current": 866, + "segment_count_peak": 866, + "timestamp_utc": "2026-08-26T19:23:46.643091Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16318633472, + "inactive_split_bytes_current_delta": 2154688512, + "inactive_split_bytes_peak": 39258380288, + "inactive_split_bytes_peak_delta": 11402669056, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 140393840640, + "reserved_bytes_current_delta": 12752781312, + "reserved_bytes_peak": 140393840640, + "reserved_bytes_peak_delta": 12752781312, + "segment_count_current": 979, + "segment_count_current_delta": 113, + "segment_count_peak": 979, + "segment_count_peak_delta": 113, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.pickle", + "timestamp_utc": "2026-08-26T19:23:48.500266Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d8536e3dc821f0480bad5c2754beec4b409630fb --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88135e920b0d584c0c8bd4e63216f4476de7dd75c98632e43ac7c3e4dc526a16 +size 3125976 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e4791cf4c0153bd35fba0373e1ac198d5fd6af68 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38987459072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123805369856, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 38987459072, + "active_bytes_peak": 123805369856, + "allocated_bytes_current": 38987459072, + "allocated_bytes_peak": 123805369856, + "device": 6, + "inactive_split_bytes_current": 16318633472, + "inactive_split_bytes_peak": 39258380288, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 140393840640, + "reserved_bytes_peak": 140393840640, + "segment_count_current": 979, + "segment_count_peak": 979, + "timestamp_utc": "2026-08-26T19:23:49.089066Z", + "world_size": 8 + }, + "allocated_bytes_current": 38987459072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123805369856, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 38987459072, + "active_bytes_peak": 123805369856, + "allocated_bytes_current": 38987459072, + "allocated_bytes_peak": 123805369856, + "device": 6, + "inactive_split_bytes_current": 16318633472, + "inactive_split_bytes_peak": 39258380288, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 140393840640, + "reserved_bytes_peak": 140393840640, + "segment_count_current": 979, + "segment_count_peak": 979, + "timestamp_utc": "2026-08-26T19:23:48.507012Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16318633472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 39258380288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 140393840640, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 140393840640, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 979, + "segment_count_current_delta": 0, + "segment_count_peak": 979, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.pickle", + "timestamp_utc": "2026-08-26T19:23:49.097344Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4758a972d280dd6b60852190b105834dcc3bd472 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6335d55966a4a3dc0c04469505d09d68dcc1b64c470488b99ed688dbf506b084 +size 356377 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2320afe1e60e38dedb5091c0910105f7d069a3d0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38987459072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123805369856, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 38987459072, + "active_bytes_peak": 123805369856, + "allocated_bytes_current": 38987459072, + "allocated_bytes_peak": 123805369856, + "device": 6, + "inactive_split_bytes_current": 16318633472, + "inactive_split_bytes_peak": 39258380288, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 140393840640, + "reserved_bytes_peak": 140393840640, + "segment_count_current": 979, + "segment_count_peak": 979, + "timestamp_utc": "2026-08-26T19:23:49.683944Z", + "world_size": 8 + }, + "allocated_bytes_current": 38987459072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123805369856, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 38987459072, + "active_bytes_peak": 123805369856, + "allocated_bytes_current": 38987459072, + "allocated_bytes_peak": 123805369856, + "device": 6, + "inactive_split_bytes_current": 16318633472, + "inactive_split_bytes_peak": 39258380288, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 140393840640, + "reserved_bytes_peak": 140393840640, + "segment_count_current": 979, + "segment_count_peak": 979, + "timestamp_utc": "2026-08-26T19:23:49.098370Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16318633472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 39258380288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 140393840640, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 140393840640, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 979, + "segment_count_current_delta": 0, + "segment_count_peak": 979, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.pickle", + "timestamp_utc": "2026-08-26T19:23:49.691999Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c420841b721d02b0ce85a1d4cc94741ba2ef736d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78de216bceed7727e49d5802db3c33c9402b572d8949a983c77023509c976944 +size 356377 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd43586546b02809e23b678eb1e9110354ac78e1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38973895680, + "active_bytes_current_delta": 15925412352, + "active_bytes_peak": 109599972864, + "active_bytes_peak_delta": 86538906624, + "after": { + "active_bytes_current": 38973895680, + "active_bytes_peak": 109599972864, + "allocated_bytes_current": 38973895680, + "allocated_bytes_peak": 109599972864, + "device": 7, + "inactive_split_bytes_current": 16290253824, + "inactive_split_bytes_peak": 28493658112, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 110899494912, + "reserved_bytes_peak": 110899494912, + "segment_count_current": 850, + "segment_count_peak": 850, + "timestamp_utc": "2026-08-26T19:23:45.258776Z", + "world_size": 8 + }, + "allocated_bytes_current": 38973895680, + "allocated_bytes_current_delta": 15925412352, + "allocated_bytes_peak": 109599972864, + "allocated_bytes_peak_delta": 86538906624, + "before": { + "active_bytes_current": 23048483328, + "active_bytes_peak": 23061066240, + "allocated_bytes_current": 23048483328, + "allocated_bytes_peak": 23061066240, + "device": 7, + "inactive_split_bytes_current": 515116544, + "inactive_split_bytes_peak": 883057664, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 25784483840, + "reserved_bytes_peak": 25784483840, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-26T19:23:17.758943Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16290253824, + "inactive_split_bytes_current_delta": 15775137280, + "inactive_split_bytes_peak": 28493658112, + "inactive_split_bytes_peak_delta": 27610600448, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 110899494912, + "reserved_bytes_current_delta": 85115011072, + "reserved_bytes_peak": 110899494912, + "reserved_bytes_peak_delta": 85115011072, + "segment_count_current": 850, + "segment_count_current_delta": 292, + "segment_count_peak": 850, + "segment_count_peak_delta": 292, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.pickle", + "timestamp_utc": "2026-08-26T19:23:45.318988Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fae5a8de323cfdaaec3e99910450c278b71fb67f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7091e3a842cdfd425213707c3f5267d552da2f9e66e04529c51e6eaf79d149de +size 3038267 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.json new file mode 100644 index 0000000000000000000000000000000000000000..58e14f9cd336d314171cf19371fe8e06ee846014 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38973895680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123793538560, + "active_bytes_peak_delta": 14193565696, + "after": { + "active_bytes_current": 38973895680, + "active_bytes_peak": 123793538560, + "allocated_bytes_current": 38973895680, + "allocated_bytes_peak": 123793538560, + "device": 7, + "inactive_split_bytes_current": 16290253824, + "inactive_split_bytes_peak": 28493658112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 127586533376, + "reserved_bytes_peak": 127586533376, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-26T19:23:45.929094Z", + "world_size": 8 + }, + "allocated_bytes_current": 38973895680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123793538560, + "allocated_bytes_peak_delta": 14193565696, + "before": { + "active_bytes_current": 38973895680, + "active_bytes_peak": 109599972864, + "allocated_bytes_current": 38973895680, + "allocated_bytes_peak": 109599972864, + "device": 7, + "inactive_split_bytes_current": 16290253824, + "inactive_split_bytes_peak": 28493658112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 110899494912, + "reserved_bytes_peak": 110899494912, + "segment_count_current": 850, + "segment_count_peak": 850, + "timestamp_utc": "2026-08-26T19:23:45.327786Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16290253824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28493658112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 127586533376, + "reserved_bytes_current_delta": 16687038464, + "reserved_bytes_peak": 127586533376, + "reserved_bytes_peak_delta": 16687038464, + "segment_count_current": 861, + "segment_count_current_delta": 11, + "segment_count_peak": 861, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.pickle", + "timestamp_utc": "2026-08-26T19:23:45.981568Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d8bad57ff7cba0589cab6f970e9484fc4c7959f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:921ccb5f50ad1a49b8d0912c1cc8e07ff0bbfad73b6f251c7525b20aa860cf18 +size 2983046 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e650c8637b39ff97a8ec8ae56c9989faf370f525 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38973895680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123795095040, + "active_bytes_peak_delta": 1556480, + "after": { + "active_bytes_current": 38973895680, + "active_bytes_peak": 123795095040, + "allocated_bytes_current": 38973895680, + "allocated_bytes_peak": 123795095040, + "device": 7, + "inactive_split_bytes_current": 16290253824, + "inactive_split_bytes_peak": 28493658112, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 127586533376, + "reserved_bytes_peak": 127586533376, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-26T19:23:46.587246Z", + "world_size": 8 + }, + "allocated_bytes_current": 38973895680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123795095040, + "allocated_bytes_peak_delta": 1556480, + "before": { + "active_bytes_current": 38973895680, + "active_bytes_peak": 123793538560, + "allocated_bytes_current": 38973895680, + "allocated_bytes_peak": 123793538560, + "device": 7, + "inactive_split_bytes_current": 16290253824, + "inactive_split_bytes_peak": 28493658112, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 127586533376, + "reserved_bytes_peak": 127586533376, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-26T19:23:45.987587Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16290253824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28493658112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 127586533376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 127586533376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 861, + "segment_count_current_delta": 0, + "segment_count_peak": 861, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.pickle", + "timestamp_utc": "2026-08-26T19:23:46.639856Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ade598a34bc25336127260492a116395292953bd --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9531abba1c87d4d6b1014745dfad6fdcfdb5457308bed0c61f600d1cc55ec5c4 +size 2959464 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04c9890f36e452fa439dcdb5a7bc1a3ebea83f98 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38991856640, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 123795095040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 38991856640, + "active_bytes_peak": 123795095040, + "allocated_bytes_current": 38991856640, + "allocated_bytes_peak": 123795095040, + "device": 7, + "inactive_split_bytes_current": 18444942336, + "inactive_split_bytes_peak": 41384689152, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 962, + "segment_count_peak": 962, + "timestamp_utc": "2026-08-26T19:23:48.442905Z", + "world_size": 8 + }, + "allocated_bytes_current": 38991856640, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 123795095040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 38973895680, + "active_bytes_peak": 123795095040, + "allocated_bytes_current": 38973895680, + "allocated_bytes_peak": 123795095040, + "device": 7, + "inactive_split_bytes_current": 16290253824, + "inactive_split_bytes_peak": 28493658112, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 127586533376, + "reserved_bytes_peak": 127586533376, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-26T19:23:46.645936Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 18444942336, + "inactive_split_bytes_current_delta": 2154688512, + "inactive_split_bytes_peak": 41384689152, + "inactive_split_bytes_peak_delta": 12891031040, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142524547072, + "reserved_bytes_current_delta": 14938013696, + "reserved_bytes_peak": 142524547072, + "reserved_bytes_peak_delta": 14938013696, + "segment_count_current": 962, + "segment_count_current_delta": 101, + "segment_count_peak": 962, + "segment_count_peak_delta": 101, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.pickle", + "timestamp_utc": "2026-08-26T19:23:48.502282Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d0207108433ecbbd77f10ecef8623dd673fc8b8a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc766681d2a1724bc84a2bead9bfdc7b53d38c247c32707168acff7efc391325 +size 3106732 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c6841340d652c156b26debbb3caafaf8beedf091 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38991856640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123795095040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 38991856640, + "active_bytes_peak": 123795095040, + "allocated_bytes_current": 38991856640, + "allocated_bytes_peak": 123795095040, + "device": 7, + "inactive_split_bytes_current": 18444942336, + "inactive_split_bytes_peak": 41384689152, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 962, + "segment_count_peak": 962, + "timestamp_utc": "2026-08-26T19:23:49.089185Z", + "world_size": 8 + }, + "allocated_bytes_current": 38991856640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123795095040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 38991856640, + "active_bytes_peak": 123795095040, + "allocated_bytes_current": 38991856640, + "allocated_bytes_peak": 123795095040, + "device": 7, + "inactive_split_bytes_current": 18444942336, + "inactive_split_bytes_peak": 41384689152, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 962, + "segment_count_peak": 962, + "timestamp_utc": "2026-08-26T19:23:48.509296Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 18444942336, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 41384689152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142524547072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142524547072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 962, + "segment_count_current_delta": 0, + "segment_count_peak": 962, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.pickle", + "timestamp_utc": "2026-08-26T19:23:49.097168Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0eee333e396df148435e5b0f2bb7e4e8507b64d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12265aafa532e83b466dd405387b287217c43b5f574287368cbd7661d56b9682 +size 342163 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed02b1c5184ae5a27a723095ee3b4f3bd64c2807 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 38991856640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123795095040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 38991856640, + "active_bytes_peak": 123795095040, + "allocated_bytes_current": 38991856640, + "allocated_bytes_peak": 123795095040, + "device": 7, + "inactive_split_bytes_current": 18444942336, + "inactive_split_bytes_peak": 41384689152, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 962, + "segment_count_peak": 962, + "timestamp_utc": "2026-08-26T19:23:49.684043Z", + "world_size": 8 + }, + "allocated_bytes_current": 38991856640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 123795095040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 38991856640, + "active_bytes_peak": 123795095040, + "allocated_bytes_current": 38991856640, + "allocated_bytes_peak": 123795095040, + "device": 7, + "inactive_split_bytes_current": 18444942336, + "inactive_split_bytes_peak": 41384689152, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 962, + "segment_count_peak": 962, + "timestamp_utc": "2026-08-26T19:23:49.098209Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 18444942336, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 41384689152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142524547072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142524547072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 962, + "segment_count_current_delta": 0, + "segment_count_peak": 962, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.pickle", + "timestamp_utc": "2026-08-26T19:23:49.692003Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..31722394563428fc4a03c72412d67c17b77180cc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0ecac16ad4203410bfbb2d55f36c425f98f921cc60078918955a8745939a1f7 +size 342163 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..b28672f335108aac30863e761b2fc2b410573883 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-better-memory-dump-support +commit=c3a8a8a7 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.25 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=768 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..82c09bb5dfe404d04b35384d93fff36763b1ace2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log @@ -0,0 +1,1601 @@ +============================================================== + DSV2_LITE_L24 / ultraep (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=1.25 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ c3a8a8a7 +============================================================== +W0826 19:22:49.570000 659990 torch/distributed/run.py:766] +W0826 19:22:49.570000 659990 torch/distributed/run.py:766] ***************************************** +W0826 19:22:49.570000 659990 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:22:49.570000 659990 torch/distributed/run.py:766] ***************************************** +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 01 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 00 + is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank1]:[W826 19:23:07.988157027 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:23:07.988249546 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:23:07.988253359 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:23:07.988323927 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:23:07.988345164 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:23:07.989630018 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:23:07.990035084 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:23:08.141657457 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.922 seconds +time to initialize megatron (seconds): 7.062 +[after megatron is initialized] datetime: 2026-08-26 19:23:13 +building GPT model ... +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4188125184 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.input_layernorm.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:23:17 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:23:17 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3599.16, 3603.98) + train/valid/test-data-iterators-setup ..........: (72.90, 90.63) +[before the start of training step] datetime: 2026-08-26 19:23:17 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +Number of parameters in transformer block in billions: 13.53 [2026-08-26 19:23:45] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 27500.7 | throughput per GPU (TFLOP/s/GPU): 4.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092127E+01 | loss scale: 1.0 | grad norm: 30.623 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 37174.25927734375 | max allocated: 104505.01904296875 | reserved: 107726.0 | max reserved: 107726.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:23:45] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 670.8 | throughput per GPU (TFLOP/s/GPU): 190.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091995E+01 | loss scale: 1.0 | grad norm: 30.567 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | + [2026-08-26 19:23:46] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 658.1 | throughput per GPU (TFLOP/s/GPU): 193.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952354E+00 | loss scale: 1.0 | grad norm: 42.232 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:23:48] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 1855.6 | throughput per GPU (TFLOP/s/GPU): 68.8 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.477323E+00 | loss scale: 1.0 | grad norm: 7.143 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | + [2026-08-26 19:23:49] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 646.3 | throughput per GPU (TFLOP/s/GPU): 197.4 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.119604E+01 | loss scale: 1.0 | grad norm: 396.977 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | + [2026-08-26 19:23:49] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 594.8 | throughput per GPU (TFLOP/s/GPU): 214.5 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.367741E+00 | loss scale: 1.0 | grad norm: 5.090 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +[after training is done] datetime: 2026-08-26 19:23:49 +[rank0]:[W826 19:23:50.923387958 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 19:23:52.568605960 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 19:23:52.570547276 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 19:23:52.578239062 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 19:23:52.579700832 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 19:23:52.581090975 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 19:23:52.604535695 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 19:23:53.121980852 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:23:58.261000 659990 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:23:58.262000 659990 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 660080 closing signal SIGTERM +W0826 19:23:58.262000 659990 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 660083 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 659990 got signal: 15 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..198657a49e0b48af8855ffbba464d9759bd36e3c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 27 +5, 4 +6, 35 +7, 47 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 2353 +1, 2359 +2, 2363 +3, 2387 +4, 2341 +5, 2345 +6, 2357 +7, 2347 +0, 25239 +1, 25239 +2, 25239 +3, 25239 +4, 25239 +5, 25239 +6, 25239 +7, 25239 +0, 28963 +1, 29051 +2, 28865 +3, 28897 +4, 28897 +5, 28865 +6, 29029 +7, 29051 +0, 39051 +1, 39159 +2, 39015 +3, 39007 +4, 39001 +5, 38999 +6, 39147 +7, 39161 +0, 39771 +1, 39855 +2, 39705 +3, 39703 +4, 39703 +5, 39701 +6, 39837 +7, 39855 +0, 39771 +1, 39855 +2, 39705 +3, 39703 +4, 39703 +5, 39701 +6, 39837 +7, 39855 +0, 39781 +1, 39865 +2, 39715 +3, 39715 +4, 39715 +5, 39711 +6, 39849 +7, 39865 +0, 39783 +1, 39869 +2, 39719 +3, 39717 +4, 39717 +5, 39715 +6, 39849 +7, 39869 +0, 97581 +1, 97601 +2, 97451 +3, 97449 +4, 97451 +5, 97447 +6, 97583 +7, 97601 +0, 110169 +1, 110189 +2, 110039 +3, 110037 +4, 110037 +5, 110035 +6, 110169 +7, 110189 +0, 110169 +1, 110189 +2, 110039 +3, 110037 +4, 110037 +5, 110035 +6, 110169 +7, 110189 +0, 110171 +1, 110193 +2, 110043 +3, 110041 +4, 110041 +5, 110039 +6, 110173 +7, 110193 +0, 112221 +1, 112243 +2, 112093 +3, 112091 +4, 112091 +5, 112089 +6, 112223 +7, 112243 +0, 112743 +1, 112775 +2, 112637 +3, 112641 +4, 112621 +5, 112629 +6, 112767 +7, 112769 +0, 113885 +1, 113967 +2, 113735 +3, 113797 +4, 113811 +5, 113801 +6, 113871 +7, 113929 +0, 125859 +1, 125875 +2, 127779 +3, 127775 +4, 127775 +5, 127769 +6, 127909 +7, 127923 +0, 125859 +1, 125875 +2, 127779 +3, 127775 +4, 127775 +5, 127769 +6, 127909 +7, 127923 +0, 122731 +1, 122747 +2, 124651 +3, 124647 +4, 124647 +5, 124641 +6, 124781 +7, 124795 +0, 121047 +1, 121325 +2, 123229 +3, 123225 +4, 123225 +5, 123607 +6, 123097 +7, 123373 +0, 4 +1, 4 +2, 4 +3, 4 +4, 121545 +5, 121927 +6, 4 +7, 121693 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192501130664Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192501130664Z.json new file mode 100644 index 0000000000000000000000000000000000000000..43a1b2b33fc14a29d2916646cee8ce45331f7ed3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192501130664Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39032005120, + "active_bytes_current_delta": 15986273280, + "active_bytes_peak": 107251034624, + "active_bytes_peak_delta": 84192719872, + "after": { + "active_bytes_current": 39032005120, + "active_bytes_peak": 107251034624, + "allocated_bytes_current": 39032005120, + "allocated_bytes_peak": 107251034624, + "device": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_peak": 27910280192, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 110876426240, + "reserved_bytes_peak": 110876426240, + "segment_count_current": 848, + "segment_count_peak": 848, + "timestamp_utc": "2026-08-26T19:25:01.130140Z", + "world_size": 8 + }, + "allocated_bytes_current": 39032005120, + "allocated_bytes_current_delta": 15986273280, + "allocated_bytes_peak": 107251034624, + "allocated_bytes_peak_delta": 84192719872, + "before": { + "active_bytes_current": 23045731840, + "active_bytes_peak": 23058314752, + "allocated_bytes_current": 23045731840, + "allocated_bytes_peak": 23058314752, + "device": 0, + "inactive_split_bytes_current": 364775936, + "inactive_split_bytes_peak": 883069440, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 25692209152, + "reserved_bytes_peak": 25692209152, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-26T19:24:30.831234Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_current_delta": 15368246272, + "inactive_split_bytes_peak": 27910280192, + "inactive_split_bytes_peak_delta": 27027210752, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 110876426240, + "reserved_bytes_current_delta": 85184217088, + "reserved_bytes_peak": 110876426240, + "reserved_bytes_peak_delta": 85184217088, + "segment_count_current": 848, + "segment_count_current_delta": 294, + "segment_count_peak": 848, + "segment_count_peak_delta": 294, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T192501130664Z.pickle", + "timestamp_utc": "2026-08-26T19:25:01.189421Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192501130664Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192501130664Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b05b02664c8f9ed55bab008f71f12c5fa501b040 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192501130664Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3c8a0f50e38373922dfd589caaf769f7c5bfd2619919756543f457c16b2b1ef +size 2949604 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192502041160Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192502041160Z.json new file mode 100644 index 0000000000000000000000000000000000000000..420dcbe28f2a24fb45ae40649859f050ea5569a8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192502041160Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39032005120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121442382848, + "active_bytes_peak_delta": 14191348224, + "after": { + "active_bytes_current": 39032005120, + "active_bytes_peak": 121442382848, + "allocated_bytes_current": 39032005120, + "allocated_bytes_peak": 121442382848, + "device": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_peak": 27910280192, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 123274788864, + "reserved_bytes_peak": 123274788864, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:02.040753Z", + "world_size": 8 + }, + "allocated_bytes_current": 39032005120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121442382848, + "allocated_bytes_peak_delta": 14191348224, + "before": { + "active_bytes_current": 39032005120, + "active_bytes_peak": 107251034624, + "allocated_bytes_current": 39032005120, + "allocated_bytes_peak": 107251034624, + "device": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_peak": 27910280192, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 110876426240, + "reserved_bytes_peak": 110876426240, + "segment_count_current": 848, + "segment_count_peak": 848, + "timestamp_utc": "2026-08-26T19:25:01.197860Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 27910280192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 123274788864, + "reserved_bytes_current_delta": 12398362624, + "reserved_bytes_peak": 123274788864, + "reserved_bytes_peak_delta": 12398362624, + "segment_count_current": 860, + "segment_count_current_delta": 12, + "segment_count_peak": 860, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T192502041160Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.094526Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192502041160Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192502041160Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea2b82a4090d1b02fb45dad94fc35489b47e315d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192502041160Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c79172b496a2a345dd0767aae05595ad7752a37cad86a9fcdc7cc83f6c16cb11 +size 2986690 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192502868541Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192502868541Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a3c72fa4115903927208a1526fbba417c01894a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192502868541Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39032005120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121485712384, + "active_bytes_peak_delta": 43329536, + "after": { + "active_bytes_current": 39032005120, + "active_bytes_peak": 121485712384, + "allocated_bytes_current": 39032005120, + "allocated_bytes_peak": 121485712384, + "device": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_peak": 27910280192, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 123274788864, + "reserved_bytes_peak": 123274788864, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:02.868142Z", + "world_size": 8 + }, + "allocated_bytes_current": 39032005120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121485712384, + "allocated_bytes_peak_delta": 43329536, + "before": { + "active_bytes_current": 39032005120, + "active_bytes_peak": 121442382848, + "allocated_bytes_current": 39032005120, + "allocated_bytes_peak": 121442382848, + "device": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_peak": 27910280192, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 123274788864, + "reserved_bytes_peak": 123274788864, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:02.100555Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 27910280192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 123274788864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123274788864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 860, + "segment_count_current_delta": 0, + "segment_count_peak": 860, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T192502868541Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.917135Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192502868541Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192502868541Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..050368be4a306e9a0d95848dbc374980b902669a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192502868541Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bfd9ba53dc0a076060017a16545c9ebb56872b2f11cf898f2357f03ae7d8f43 +size 2985933 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192503679432Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192503679432Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db628451f016d39d34deade20aa03ed7a88098db --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192503679432Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39032005120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121485712384, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39032005120, + "active_bytes_peak": 121485712384, + "allocated_bytes_current": 39032005120, + "allocated_bytes_peak": 121485712384, + "device": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_peak": 27910280192, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 123274788864, + "reserved_bytes_peak": 123274788864, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:03.679028Z", + "world_size": 8 + }, + "allocated_bytes_current": 39032005120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121485712384, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39032005120, + "active_bytes_peak": 121485712384, + "allocated_bytes_current": 39032005120, + "allocated_bytes_peak": 121485712384, + "device": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_peak": 27910280192, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 123274788864, + "reserved_bytes_peak": 123274788864, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:02.922821Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 27910280192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 123274788864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123274788864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 860, + "segment_count_current_delta": 0, + "segment_count_peak": 860, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T192503679432Z.pickle", + "timestamp_utc": "2026-08-26T19:25:03.727647Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192503679432Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192503679432Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..585c0c54b8a14723af445afb72ede8a882575835 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192503679432Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bff6f32b23834ea3bb234c392cae617a5c5a5c59b8b6d5ee71440411680c6a45 +size 2985930 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192504498015Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192504498015Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3c1253173920b4a91bf2ef0c81326fef06dc8263 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192504498015Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39032005120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121506621440, + "active_bytes_peak_delta": 20909056, + "after": { + "active_bytes_current": 39032005120, + "active_bytes_peak": 121506621440, + "allocated_bytes_current": 39032005120, + "allocated_bytes_peak": 121506621440, + "device": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_peak": 27910280192, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 123274788864, + "reserved_bytes_peak": 123274788864, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:04.497598Z", + "world_size": 8 + }, + "allocated_bytes_current": 39032005120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121506621440, + "allocated_bytes_peak_delta": 20909056, + "before": { + "active_bytes_current": 39032005120, + "active_bytes_peak": 121485712384, + "allocated_bytes_current": 39032005120, + "allocated_bytes_peak": 121485712384, + "device": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_peak": 27910280192, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 123274788864, + "reserved_bytes_peak": 123274788864, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:03.733224Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 15733022208, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 27910280192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 123274788864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123274788864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 860, + "segment_count_current_delta": 0, + "segment_count_peak": 860, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T192504498015Z.pickle", + "timestamp_utc": "2026-08-26T19:25:04.547673Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192504498015Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192504498015Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d80e82d7ee42e20e35361d9c093d0c47dd0f3b65 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192504498015Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f09108283191f89da4e30a2b5e9938a7bf31b9dc0e5e31fb5f62897e97fcca01 +size 2985932 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192501130966Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192501130966Z.json new file mode 100644 index 0000000000000000000000000000000000000000..700bcd71f971495cce4343861eb84b79644ccb37 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192501130966Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39017585664, + "active_bytes_current_delta": 15969114112, + "active_bytes_peak": 107145516032, + "active_bytes_peak_delta": 84084461568, + "after": { + "active_bytes_current": 39017585664, + "active_bytes_peak": 107145516032, + "allocated_bytes_current": 39017585664, + "allocated_bytes_peak": 107145516032, + "device": 1, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_peak": 28396835840, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 110899494912, + "reserved_bytes_peak": 110899494912, + "segment_count_current": 850, + "segment_count_peak": 850, + "timestamp_utc": "2026-08-26T19:25:01.130458Z", + "world_size": 8 + }, + "allocated_bytes_current": 39017585664, + "allocated_bytes_current_delta": 15969114112, + "allocated_bytes_peak": 107145516032, + "allocated_bytes_peak_delta": 84084461568, + "before": { + "active_bytes_current": 23048471552, + "active_bytes_peak": 23061054464, + "allocated_bytes_current": 23048471552, + "allocated_bytes_peak": 23061054464, + "device": 1, + "inactive_split_bytes_current": 515128320, + "inactive_split_bytes_peak": 883069440, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 25784483840, + "reserved_bytes_peak": 25784483840, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-26T19:24:30.831191Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_current_delta": 15750309888, + "inactive_split_bytes_peak": 28396835840, + "inactive_split_bytes_peak_delta": 27513766400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 110899494912, + "reserved_bytes_current_delta": 85115011072, + "reserved_bytes_peak": 110899494912, + "reserved_bytes_peak_delta": 85115011072, + "segment_count_current": 850, + "segment_count_current_delta": 292, + "segment_count_peak": 850, + "segment_count_peak_delta": 292, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T192501130966Z.pickle", + "timestamp_utc": "2026-08-26T19:25:01.190353Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192501130966Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192501130966Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d8a28b373c62c49491ebf92ed8176b671725e762 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192501130966Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d745f5d0e58a36289ab5e3c7a1a03e2b26d0f4f14586ca57b1534ade32d5f196 +size 3002084 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192502041390Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192502041390Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fe172f45f85d4858a2a6e7e44e867f0d311de84c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192502041390Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39017585664, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121471769088, + "active_bytes_peak_delta": 14326253056, + "after": { + "active_bytes_current": 39017585664, + "active_bytes_peak": 121471769088, + "allocated_bytes_current": 39017585664, + "allocated_bytes_peak": 121471769088, + "device": 1, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_peak": 28396835840, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 123291566080, + "reserved_bytes_peak": 123291566080, + "segment_count_current": 859, + "segment_count_peak": 859, + "timestamp_utc": "2026-08-26T19:25:02.040919Z", + "world_size": 8 + }, + "allocated_bytes_current": 39017585664, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121471769088, + "allocated_bytes_peak_delta": 14326253056, + "before": { + "active_bytes_current": 39017585664, + "active_bytes_peak": 107145516032, + "allocated_bytes_current": 39017585664, + "allocated_bytes_peak": 107145516032, + "device": 1, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_peak": 28396835840, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 110899494912, + "reserved_bytes_peak": 110899494912, + "segment_count_current": 850, + "segment_count_peak": 850, + "timestamp_utc": "2026-08-26T19:25:01.198769Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28396835840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 123291566080, + "reserved_bytes_current_delta": 12392071168, + "reserved_bytes_peak": 123291566080, + "reserved_bytes_peak_delta": 12392071168, + "segment_count_current": 859, + "segment_count_current_delta": 9, + "segment_count_peak": 859, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T192502041390Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.096099Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192502041390Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192502041390Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6bc90efe036917061232bdd057bd49e4e7a5f2da --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192502041390Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:010394a92ba3403b91e4f0e1ba5a919d6f6dd4d2788c2fc37790b87fcbbdba60 +size 2961418 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192502868577Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192502868577Z.json new file mode 100644 index 0000000000000000000000000000000000000000..51c9c07036274621cd1684cb3ef692294c0851f5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192502868577Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39017585664, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121471769088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39017585664, + "active_bytes_peak": 121471769088, + "allocated_bytes_current": 39017585664, + "allocated_bytes_peak": 121471769088, + "device": 1, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_peak": 28396835840, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 123291566080, + "reserved_bytes_peak": 123291566080, + "segment_count_current": 859, + "segment_count_peak": 859, + "timestamp_utc": "2026-08-26T19:25:02.868168Z", + "world_size": 8 + }, + "allocated_bytes_current": 39017585664, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121471769088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39017585664, + "active_bytes_peak": 121471769088, + "allocated_bytes_current": 39017585664, + "allocated_bytes_peak": 121471769088, + "device": 1, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_peak": 28396835840, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 123291566080, + "reserved_bytes_peak": 123291566080, + "segment_count_current": 859, + "segment_count_peak": 859, + "timestamp_utc": "2026-08-26T19:25:02.102372Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28396835840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 123291566080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123291566080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 859, + "segment_count_current_delta": 0, + "segment_count_peak": 859, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T192502868577Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.917513Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192502868577Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192502868577Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba909456dee7b5221100c7747592c2c30c720c35 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192502868577Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b5273b03e2df30f687c2dce44711893fbbc41ccb4d38a86dc22e120804eef2d +size 2961007 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192503679528Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192503679528Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b2b4d8893bd3f74f660aae69b1889aa6b7b6b7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192503679528Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39017585664, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121471769088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39017585664, + "active_bytes_peak": 121471769088, + "allocated_bytes_current": 39017585664, + "allocated_bytes_peak": 121471769088, + "device": 1, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_peak": 28396835840, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 123291566080, + "reserved_bytes_peak": 123291566080, + "segment_count_current": 859, + "segment_count_peak": 859, + "timestamp_utc": "2026-08-26T19:25:03.679124Z", + "world_size": 8 + }, + "allocated_bytes_current": 39017585664, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121471769088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39017585664, + "active_bytes_peak": 121471769088, + "allocated_bytes_current": 39017585664, + "allocated_bytes_peak": 121471769088, + "device": 1, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_peak": 28396835840, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 123291566080, + "reserved_bytes_peak": 123291566080, + "segment_count_current": 859, + "segment_count_peak": 859, + "timestamp_utc": "2026-08-26T19:25:02.923192Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28396835840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 123291566080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123291566080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 859, + "segment_count_current_delta": 0, + "segment_count_peak": 859, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T192503679528Z.pickle", + "timestamp_utc": "2026-08-26T19:25:03.729007Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192503679528Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192503679528Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5038686765c068823b15acb64c4961477f8ba815 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192503679528Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1a0d5f16c8a49e6fea098027ff2cd988b58886d5ae231dc59d91eda331d497d +size 2961004 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192504498152Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192504498152Z.json new file mode 100644 index 0000000000000000000000000000000000000000..59efe8a2728a93be72827a832c5d206ca593db24 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192504498152Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39017585664, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121471769088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39017585664, + "active_bytes_peak": 121471769088, + "allocated_bytes_current": 39017585664, + "allocated_bytes_peak": 121471769088, + "device": 1, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_peak": 28396835840, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 123291566080, + "reserved_bytes_peak": 123291566080, + "segment_count_current": 859, + "segment_count_peak": 859, + "timestamp_utc": "2026-08-26T19:25:04.497721Z", + "world_size": 8 + }, + "allocated_bytes_current": 39017585664, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121471769088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39017585664, + "active_bytes_peak": 121471769088, + "allocated_bytes_current": 39017585664, + "allocated_bytes_peak": 121471769088, + "device": 1, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_peak": 28396835840, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 123291566080, + "reserved_bytes_peak": 123291566080, + "segment_count_current": 859, + "segment_count_peak": 859, + "timestamp_utc": "2026-08-26T19:25:03.734782Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16265438208, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28396835840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 123291566080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 123291566080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 859, + "segment_count_current_delta": 0, + "segment_count_peak": 859, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T192504498152Z.pickle", + "timestamp_utc": "2026-08-26T19:25:04.547735Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192504498152Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192504498152Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bc21ffc71e600ff54e1f7782bd476a30a230251b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192504498152Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:097176d8980a32c06b669fa181e0c9282e6f5474fabcb966e82995ccffbea76b +size 2961004 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192501130664Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192501130664Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94f10a8f97c1afef999d5eae19bed01f5d8c1c0b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192501130664Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39021886464, + "active_bytes_current_delta": 15970850816, + "active_bytes_peak": 107227355648, + "active_bytes_peak_delta": 84163737088, + "after": { + "active_bytes_current": 39021886464, + "active_bytes_peak": 107227355648, + "allocated_bytes_current": 39021886464, + "allocated_bytes_peak": 107227355648, + "device": 2, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_peak": 28254047744, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 110742208512, + "reserved_bytes_peak": 110742208512, + "segment_count_current": 845, + "segment_count_peak": 845, + "timestamp_utc": "2026-08-26T19:25:01.130122Z", + "world_size": 8 + }, + "allocated_bytes_current": 39021886464, + "allocated_bytes_current_delta": 15970850816, + "allocated_bytes_peak": 107227355648, + "allocated_bytes_peak_delta": 84163737088, + "before": { + "active_bytes_current": 23051035648, + "active_bytes_peak": 23063618560, + "allocated_bytes_current": 23051035648, + "allocated_bytes_peak": 23063618560, + "device": 2, + "inactive_split_bytes_current": 376249344, + "inactive_split_bytes_peak": 883069440, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:24:30.831359Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_current_delta": 15731795968, + "inactive_split_bytes_peak": 28254047744, + "inactive_split_bytes_peak_delta": 27370978304, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 110742208512, + "reserved_bytes_current_delta": 85119205376, + "reserved_bytes_peak": 110742208512, + "reserved_bytes_peak_delta": 85119205376, + "segment_count_current": 845, + "segment_count_current_delta": 294, + "segment_count_peak": 845, + "segment_count_peak_delta": 294, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T192501130664Z.pickle", + "timestamp_utc": "2026-08-26T19:25:01.190760Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192501130664Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192501130664Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a17c91d801ee566c0d80a6a7f0cecb9d64c7a7ab --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192501130664Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83a0469a75d369c4b98a4524d8f10f9aa0c070e6c8695d23e6534b7718b7c2b2 +size 3018568 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192502041157Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192502041157Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9f7e1720cc330373bc480d54be752e420462489b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192502041157Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39021886464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121400772096, + "active_bytes_peak_delta": 14173416448, + "after": { + "active_bytes_current": 39021886464, + "active_bytes_peak": 121400772096, + "allocated_bytes_current": 39021886464, + "allocated_bytes_peak": 121400772096, + "device": 2, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_peak": 28254047744, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 125288054784, + "reserved_bytes_peak": 125288054784, + "segment_count_current": 858, + "segment_count_peak": 858, + "timestamp_utc": "2026-08-26T19:25:02.040741Z", + "world_size": 8 + }, + "allocated_bytes_current": 39021886464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121400772096, + "allocated_bytes_peak_delta": 14173416448, + "before": { + "active_bytes_current": 39021886464, + "active_bytes_peak": 107227355648, + "allocated_bytes_current": 39021886464, + "allocated_bytes_peak": 107227355648, + "device": 2, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_peak": 28254047744, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 110742208512, + "reserved_bytes_peak": 110742208512, + "segment_count_current": 845, + "segment_count_peak": 845, + "timestamp_utc": "2026-08-26T19:25:01.199377Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28254047744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 125288054784, + "reserved_bytes_current_delta": 14545846272, + "reserved_bytes_peak": 125288054784, + "reserved_bytes_peak_delta": 14545846272, + "segment_count_current": 858, + "segment_count_current_delta": 13, + "segment_count_peak": 858, + "segment_count_peak_delta": 13, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T192502041157Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.094751Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192502041157Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192502041157Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..796e0e815c533a897a4c060f0fa990155405fd7d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192502041157Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:069447d29b0183f904cfe8cad9381245df0e06dad8aeeb823e08afdb683cb903 +size 2974491 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192502868500Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192502868500Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bdd003f303281faa79499038ef6a45595cc88d09 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192502868500Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39021886464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121439844864, + "active_bytes_peak_delta": 39072768, + "after": { + "active_bytes_current": 39021886464, + "active_bytes_peak": 121439844864, + "allocated_bytes_current": 39021886464, + "allocated_bytes_peak": 121439844864, + "device": 2, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_peak": 28254047744, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 125288054784, + "reserved_bytes_peak": 125288054784, + "segment_count_current": 858, + "segment_count_peak": 858, + "timestamp_utc": "2026-08-26T19:25:02.868102Z", + "world_size": 8 + }, + "allocated_bytes_current": 39021886464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121439844864, + "allocated_bytes_peak_delta": 39072768, + "before": { + "active_bytes_current": 39021886464, + "active_bytes_peak": 121400772096, + "allocated_bytes_current": 39021886464, + "allocated_bytes_peak": 121400772096, + "device": 2, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_peak": 28254047744, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 125288054784, + "reserved_bytes_peak": 125288054784, + "segment_count_current": 858, + "segment_count_peak": 858, + "timestamp_utc": "2026-08-26T19:25:02.100818Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28254047744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 125288054784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125288054784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 858, + "segment_count_current_delta": 0, + "segment_count_peak": 858, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T192502868500Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.917030Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192502868500Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192502868500Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b50035b2519d4196e3e415763fadc54a43b566b4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192502868500Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86e94c2fdaf298fd1a52618047578d740ce3c93f387579100d0f449ad5d0a5c0 +size 2973774 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192503679416Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192503679416Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9fcd7eb39891c8b01adffdd02346cbf9ced0c865 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192503679416Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39021886464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121468293120, + "active_bytes_peak_delta": 28448256, + "after": { + "active_bytes_current": 39021886464, + "active_bytes_peak": 121468293120, + "allocated_bytes_current": 39021886464, + "allocated_bytes_peak": 121468293120, + "device": 2, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_peak": 28254047744, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 125288054784, + "reserved_bytes_peak": 125288054784, + "segment_count_current": 858, + "segment_count_peak": 858, + "timestamp_utc": "2026-08-26T19:25:03.679020Z", + "world_size": 8 + }, + "allocated_bytes_current": 39021886464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121468293120, + "allocated_bytes_peak_delta": 28448256, + "before": { + "active_bytes_current": 39021886464, + "active_bytes_peak": 121439844864, + "allocated_bytes_current": 39021886464, + "allocated_bytes_peak": 121439844864, + "device": 2, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_peak": 28254047744, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 125288054784, + "reserved_bytes_peak": 125288054784, + "segment_count_current": 858, + "segment_count_peak": 858, + "timestamp_utc": "2026-08-26T19:25:02.922653Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28254047744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 125288054784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125288054784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 858, + "segment_count_current_delta": 0, + "segment_count_peak": 858, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T192503679416Z.pickle", + "timestamp_utc": "2026-08-26T19:25:03.728765Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192503679416Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192503679416Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c6cfc9c7457f3b8f0c1f83bdd3b4b5a3f76345d4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192503679416Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8052c048b711dffc0665bfec6aa573b6f09932bd8908f3ea5e436a382d1626b9 +size 2973779 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192504498015Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192504498015Z.json new file mode 100644 index 0000000000000000000000000000000000000000..660bd1e3f8b1515539c9137f1ab5ab2f6a7998d0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192504498015Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39021886464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121468293120, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39021886464, + "active_bytes_peak": 121468293120, + "allocated_bytes_current": 39021886464, + "allocated_bytes_peak": 121468293120, + "device": 2, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_peak": 28254047744, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 125288054784, + "reserved_bytes_peak": 125288054784, + "segment_count_current": 858, + "segment_count_peak": 858, + "timestamp_utc": "2026-08-26T19:25:04.497621Z", + "world_size": 8 + }, + "allocated_bytes_current": 39021886464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121468293120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39021886464, + "active_bytes_peak": 121468293120, + "allocated_bytes_current": 39021886464, + "allocated_bytes_peak": 121468293120, + "device": 2, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_peak": 28254047744, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 125288054784, + "reserved_bytes_peak": 125288054784, + "segment_count_current": 858, + "segment_count_peak": 858, + "timestamp_utc": "2026-08-26T19:25:03.734304Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16108045312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28254047744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 125288054784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125288054784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 858, + "segment_count_current_delta": 0, + "segment_count_peak": 858, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T192504498015Z.pickle", + "timestamp_utc": "2026-08-26T19:25:04.546461Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192504498015Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192504498015Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7f5f10e7b0258dbd2f4b9b6146503afdbe4d91d6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192504498015Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58b02ddde0a6df6034e4cbb311127565f782c99208a481b9f63afbe739107200 +size 2973777 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192501130934Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192501130934Z.json new file mode 100644 index 0000000000000000000000000000000000000000..238c64354a7438909f432521e396e39906d27f67 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192501130934Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39020836352, + "active_bytes_current_delta": 15971727360, + "active_bytes_peak": 107242569728, + "active_bytes_peak_delta": 84180877824, + "after": { + "active_bytes_current": 39020836352, + "active_bytes_peak": 107242569728, + "allocated_bytes_current": 39020836352, + "allocated_bytes_peak": 107242569728, + "device": 3, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_peak": 26249835520, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110740111360, + "reserved_bytes_peak": 110740111360, + "segment_count_current": 844, + "segment_count_peak": 844, + "timestamp_utc": "2026-08-26T19:25:01.130462Z", + "world_size": 8 + }, + "allocated_bytes_current": 39020836352, + "allocated_bytes_current_delta": 15971727360, + "allocated_bytes_peak": 107242569728, + "allocated_bytes_peak_delta": 84180877824, + "before": { + "active_bytes_current": 23049108992, + "active_bytes_peak": 23061691904, + "allocated_bytes_current": 23049108992, + "allocated_bytes_peak": 23061691904, + "device": 3, + "inactive_split_bytes_current": 480936448, + "inactive_split_bytes_peak": 883069440, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:24:30.831313Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_current_delta": 13474383872, + "inactive_split_bytes_peak": 26249835520, + "inactive_split_bytes_peak_delta": 25366766080, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110740111360, + "reserved_bytes_current_delta": 85117108224, + "reserved_bytes_peak": 110740111360, + "reserved_bytes_peak_delta": 85117108224, + "segment_count_current": 844, + "segment_count_current_delta": 293, + "segment_count_peak": 844, + "segment_count_peak_delta": 293, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T192501130934Z.pickle", + "timestamp_utc": "2026-08-26T19:25:01.189899Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192501130934Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192501130934Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..297f21e96bcea1defaf19bdcbba0194ad5ee99c3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192501130934Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61f3287644ebd2e6f068d0b90e23b4504f066107d7b248c77028ff0d9630aa79 +size 2963249 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192502041306Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192502041306Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5264b375ce841b57c494815c81aacdce00954829 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192502041306Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39020836352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121395646464, + "active_bytes_peak_delta": 14153076736, + "after": { + "active_bytes_current": 39020836352, + "active_bytes_peak": 121395646464, + "allocated_bytes_current": 39020836352, + "allocated_bytes_peak": 121395646464, + "device": 3, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_peak": 26249835520, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:02.040865Z", + "world_size": 8 + }, + "allocated_bytes_current": 39020836352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121395646464, + "allocated_bytes_peak_delta": 14153076736, + "before": { + "active_bytes_current": 39020836352, + "active_bytes_peak": 107242569728, + "allocated_bytes_current": 39020836352, + "allocated_bytes_peak": 107242569728, + "device": 3, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_peak": 26249835520, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110740111360, + "reserved_bytes_peak": 110740111360, + "segment_count_current": 844, + "segment_count_peak": 844, + "timestamp_utc": "2026-08-26T19:25:01.198345Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 26249835520, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 125283860480, + "reserved_bytes_current_delta": 14543749120, + "reserved_bytes_peak": 125283860480, + "reserved_bytes_peak_delta": 14543749120, + "segment_count_current": 856, + "segment_count_current_delta": 12, + "segment_count_peak": 856, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T192502041306Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.095741Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192502041306Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192502041306Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8966218e7f3472f25f776cc675a7f73556dcfdea --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192502041306Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae092f4bd5474c894a32b78724aa7427b266e3daf0e8882b14062c5526ffe365 +size 2950629 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192502868582Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192502868582Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e24c94ce3667ab89ab98d84e7d65ca8075551678 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192502868582Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39020836352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121409800704, + "active_bytes_peak_delta": 14154240, + "after": { + "active_bytes_current": 39020836352, + "active_bytes_peak": 121409800704, + "allocated_bytes_current": 39020836352, + "allocated_bytes_peak": 121409800704, + "device": 3, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_peak": 26249835520, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:02.868179Z", + "world_size": 8 + }, + "allocated_bytes_current": 39020836352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121409800704, + "allocated_bytes_peak_delta": 14154240, + "before": { + "active_bytes_current": 39020836352, + "active_bytes_peak": 121395646464, + "allocated_bytes_current": 39020836352, + "allocated_bytes_peak": 121395646464, + "device": 3, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_peak": 26249835520, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:02.101941Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 26249835520, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 125283860480, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125283860480, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 856, + "segment_count_current_delta": 0, + "segment_count_peak": 856, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T192502868582Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.918070Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192502868582Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192502868582Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ad284335c28d72fd3ca90ae5a31c28562ec74713 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192502868582Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5b3f6c23e2efc43887d818c742d785874bb497c1d8437a600621229b32fedde +size 2949803 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192503679518Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192503679518Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e031c5a756dcee7869688c71e34479aa364578f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192503679518Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39020836352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121409800704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39020836352, + "active_bytes_peak": 121409800704, + "allocated_bytes_current": 39020836352, + "allocated_bytes_peak": 121409800704, + "device": 3, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_peak": 26249835520, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:03.679104Z", + "world_size": 8 + }, + "allocated_bytes_current": 39020836352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121409800704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39020836352, + "active_bytes_peak": 121409800704, + "allocated_bytes_current": 39020836352, + "allocated_bytes_peak": 121409800704, + "device": 3, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_peak": 26249835520, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:02.923951Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 26249835520, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 125283860480, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125283860480, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 856, + "segment_count_current_delta": 0, + "segment_count_peak": 856, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T192503679518Z.pickle", + "timestamp_utc": "2026-08-26T19:25:03.728381Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192503679518Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192503679518Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..158e5090e5b9c44f4b80e2905d200d2d0e8f0650 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192503679518Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0be0e365de3b072532be467efc6c6ba359fd5c9bd5fdfdd560079ccd6f5334dc +size 2949803 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192504498161Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192504498161Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae6f0d170e3dbc320fd9c8e57d4d53fdb5690b0c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192504498161Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39020836352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121501803008, + "active_bytes_peak_delta": 92002304, + "after": { + "active_bytes_current": 39020836352, + "active_bytes_peak": 121501803008, + "allocated_bytes_current": 39020836352, + "allocated_bytes_peak": 121501803008, + "device": 3, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_peak": 26249835520, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:04.497756Z", + "world_size": 8 + }, + "allocated_bytes_current": 39020836352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121501803008, + "allocated_bytes_peak_delta": 92002304, + "before": { + "active_bytes_current": 39020836352, + "active_bytes_peak": 121409800704, + "allocated_bytes_current": 39020836352, + "allocated_bytes_peak": 121409800704, + "device": 3, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_peak": 26249835520, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:03.734158Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 13955320320, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 26249835520, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 125283860480, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125283860480, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 856, + "segment_count_current_delta": 0, + "segment_count_peak": 856, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T192504498161Z.pickle", + "timestamp_utc": "2026-08-26T19:25:04.548119Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192504498161Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192504498161Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c9310c89506ae9727b5d3bd9b068eb78ad066de1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192504498161Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5f24189f4f118406d5dedd815d6fdd6f6469f5a9eba8b89056d6f02f1f2e0de +size 2949803 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192501130922Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192501130922Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3d97f112e55217cd0410177affbecee279fb25e0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192501130922Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39031600128, + "active_bytes_current_delta": 15983050752, + "active_bytes_peak": 107234716160, + "active_bytes_peak_delta": 84173583872, + "after": { + "active_bytes_current": 39031600128, + "active_bytes_peak": 107234716160, + "allocated_bytes_current": 39031600128, + "allocated_bytes_peak": 107234716160, + "device": 4, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_peak": 28245021184, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 110740111360, + "reserved_bytes_peak": 110740111360, + "segment_count_current": 844, + "segment_count_peak": 844, + "timestamp_utc": "2026-08-26T19:25:01.130394Z", + "world_size": 8 + }, + "allocated_bytes_current": 39031600128, + "allocated_bytes_current_delta": 15983050752, + "allocated_bytes_peak": 107234716160, + "allocated_bytes_peak_delta": 84173583872, + "before": { + "active_bytes_current": 23048549376, + "active_bytes_peak": 23061132288, + "allocated_bytes_current": 23048549376, + "allocated_bytes_peak": 23061132288, + "device": 4, + "inactive_split_bytes_current": 389221376, + "inactive_split_bytes_peak": 883069440, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:24:30.831391Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_current_delta": 15700721664, + "inactive_split_bytes_peak": 28245021184, + "inactive_split_bytes_peak_delta": 27361951744, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 110740111360, + "reserved_bytes_current_delta": 85117108224, + "reserved_bytes_peak": 110740111360, + "reserved_bytes_peak_delta": 85117108224, + "segment_count_current": 844, + "segment_count_current_delta": 293, + "segment_count_peak": 844, + "segment_count_peak_delta": 293, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T192501130922Z.pickle", + "timestamp_utc": "2026-08-26T19:25:01.191179Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192501130922Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192501130922Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..428192ea329d85bc97a2a9b7d2e4d8472302b3b4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192501130922Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e91502910ee5df67ce6bffc90c0cf0551de0321d0af99345f48a574deaa7be5 +size 2964247 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192502041371Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192502041371Z.json new file mode 100644 index 0000000000000000000000000000000000000000..450754eefd2dcef2b3a3445b5fe5cb1d7631be0b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192502041371Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39031600128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121417127424, + "active_bytes_peak_delta": 14182411264, + "after": { + "active_bytes_current": 39031600128, + "active_bytes_peak": 121417127424, + "allocated_bytes_current": 39031600128, + "allocated_bytes_peak": 121417127424, + "device": 4, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_peak": 28245021184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:02.040895Z", + "world_size": 8 + }, + "allocated_bytes_current": 39031600128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121417127424, + "allocated_bytes_peak_delta": 14182411264, + "before": { + "active_bytes_current": 39031600128, + "active_bytes_peak": 107234716160, + "allocated_bytes_current": 39031600128, + "allocated_bytes_peak": 107234716160, + "device": 4, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_peak": 28245021184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 110740111360, + "reserved_bytes_peak": 110740111360, + "segment_count_current": 844, + "segment_count_peak": 844, + "timestamp_utc": "2026-08-26T19:25:01.199855Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28245021184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 125283860480, + "reserved_bytes_current_delta": 14543749120, + "reserved_bytes_peak": 125283860480, + "reserved_bytes_peak_delta": 14543749120, + "segment_count_current": 856, + "segment_count_current_delta": 12, + "segment_count_peak": 856, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T192502041371Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.095862Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192502041371Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192502041371Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a199a220e7b0e46560c28cea0a53772e6aea4a29 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192502041371Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4baed68fae1bc79c192f5bdb0435cfb3a567d8b5e715ae1991206636d976f58 +size 3015798 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192502868581Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192502868581Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f16f60ba73e9e16ae8d5e402696f94cc8350bac --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192502868581Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39031600128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121462509056, + "active_bytes_peak_delta": 45381632, + "after": { + "active_bytes_current": 39031600128, + "active_bytes_peak": 121462509056, + "allocated_bytes_current": 39031600128, + "allocated_bytes_peak": 121462509056, + "device": 4, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_peak": 28245021184, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:02.868165Z", + "world_size": 8 + }, + "allocated_bytes_current": 39031600128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121462509056, + "allocated_bytes_peak_delta": 45381632, + "before": { + "active_bytes_current": 39031600128, + "active_bytes_peak": 121417127424, + "allocated_bytes_current": 39031600128, + "allocated_bytes_peak": 121417127424, + "device": 4, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_peak": 28245021184, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:02.102095Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28245021184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 125283860480, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125283860480, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 856, + "segment_count_current_delta": 0, + "segment_count_peak": 856, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T192502868581Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.918008Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192502868581Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192502868581Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..92e7050ff57c1fe677b2c2d5ca99b67f483abb4b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192502868581Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46c01d688f29ee38a94c65117a0b1873a49fb94cc150eee41b76bda5cbd92ee8 +size 3015139 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192503679515Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192503679515Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3c5377657a123f94a4ea73dd9fd43c4f85b3404d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192503679515Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39031600128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121462509056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39031600128, + "active_bytes_peak": 121462509056, + "allocated_bytes_current": 39031600128, + "allocated_bytes_peak": 121462509056, + "device": 4, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_peak": 28245021184, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:03.679103Z", + "world_size": 8 + }, + "allocated_bytes_current": 39031600128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121462509056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39031600128, + "active_bytes_peak": 121462509056, + "allocated_bytes_current": 39031600128, + "allocated_bytes_peak": 121462509056, + "device": 4, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_peak": 28245021184, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:02.923657Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28245021184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 125283860480, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125283860480, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 856, + "segment_count_current_delta": 0, + "segment_count_peak": 856, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T192503679515Z.pickle", + "timestamp_utc": "2026-08-26T19:25:03.728823Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192503679515Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192503679515Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c15d8336ea20de26b4aaa8428c94efbcaab83509 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192503679515Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:798b961fdffdfb9f53fdb51439b0cb2bf80ea50d4096061f5c9d96328982a34c +size 3015133 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192504498185Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192504498185Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e324c06f87b383349e26c56634e2441aac4c2809 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192504498185Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39031600128, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121462509056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39031600128, + "active_bytes_peak": 121462509056, + "allocated_bytes_current": 39031600128, + "allocated_bytes_peak": 121462509056, + "device": 4, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_peak": 28245021184, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:04.497733Z", + "world_size": 8 + }, + "allocated_bytes_current": 39031600128, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121462509056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39031600128, + "active_bytes_peak": 121462509056, + "allocated_bytes_current": 39031600128, + "allocated_bytes_peak": 121462509056, + "device": 4, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_peak": 28245021184, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 125283860480, + "reserved_bytes_peak": 125283860480, + "segment_count_current": 856, + "segment_count_peak": 856, + "timestamp_utc": "2026-08-26T19:25:03.734486Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 16089943040, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28245021184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 125283860480, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125283860480, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 856, + "segment_count_current_delta": 0, + "segment_count_peak": 856, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T192504498185Z.pickle", + "timestamp_utc": "2026-08-26T19:25:04.548281Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192504498185Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192504498185Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b385809ec9a300c4d26bcf90c7247dadc388ad76 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192504498185Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99a8acf2339d435e442ad82af646dc507547ed42e084e5cc607e8042666198c2 +size 3015139 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192501130666Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192501130666Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ec42897f62d43bb05ae26dac5f7309d39286f4c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192501130666Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39028323328, + "active_bytes_current_delta": 15974974464, + "active_bytes_peak": 107234675712, + "active_bytes_peak_delta": 84168743936, + "after": { + "active_bytes_current": 39028323328, + "active_bytes_peak": 107234675712, + "allocated_bytes_current": 39028323328, + "allocated_bytes_peak": 107234675712, + "device": 5, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_peak": 28353631232, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 110738014208, + "reserved_bytes_peak": 110738014208, + "segment_count_current": 843, + "segment_count_peak": 843, + "timestamp_utc": "2026-08-26T19:25:01.130118Z", + "world_size": 8 + }, + "allocated_bytes_current": 39028323328, + "allocated_bytes_current_delta": 15974974464, + "allocated_bytes_peak": 107234675712, + "allocated_bytes_peak_delta": 84168743936, + "before": { + "active_bytes_current": 23053348864, + "active_bytes_peak": 23065931776, + "allocated_bytes_current": 23053348864, + "allocated_bytes_peak": 23065931776, + "device": 5, + "inactive_split_bytes_current": 474599424, + "inactive_split_bytes_peak": 883069440, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 25623003136, + "reserved_bytes_peak": 25623003136, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-26T19:24:30.831280Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_current_delta": 15624911872, + "inactive_split_bytes_peak": 28353631232, + "inactive_split_bytes_peak_delta": 27470561792, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 110738014208, + "reserved_bytes_current_delta": 85115011072, + "reserved_bytes_peak": 110738014208, + "reserved_bytes_peak_delta": 85115011072, + "segment_count_current": 843, + "segment_count_current_delta": 292, + "segment_count_peak": 843, + "segment_count_peak_delta": 292, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T192501130666Z.pickle", + "timestamp_utc": "2026-08-26T19:25:01.191952Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192501130666Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192501130666Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..45dfcbe5a28c7b1675c15aeb8048e48286ad5e5c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192501130666Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ad31f4601606c0478a3dd9cef445144a8fa9b9eb52ed882268e623ef7d415ff +size 3000506 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192502041162Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192502041162Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4f2284b4a42a485e5df08dd737a8bd01607cc660 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192502041162Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39028323328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121446202368, + "active_bytes_peak_delta": 14211526656, + "after": { + "active_bytes_current": 39028323328, + "active_bytes_peak": 121446202368, + "allocated_bytes_current": 39028323328, + "allocated_bytes_peak": 121446202368, + "device": 5, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_peak": 28353631232, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 125277569024, + "reserved_bytes_peak": 125277569024, + "segment_count_current": 853, + "segment_count_peak": 853, + "timestamp_utc": "2026-08-26T19:25:02.040754Z", + "world_size": 8 + }, + "allocated_bytes_current": 39028323328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121446202368, + "allocated_bytes_peak_delta": 14211526656, + "before": { + "active_bytes_current": 39028323328, + "active_bytes_peak": 107234675712, + "allocated_bytes_current": 39028323328, + "allocated_bytes_peak": 107234675712, + "device": 5, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_peak": 28353631232, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 110738014208, + "reserved_bytes_peak": 110738014208, + "segment_count_current": 843, + "segment_count_peak": 843, + "timestamp_utc": "2026-08-26T19:25:01.200471Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28353631232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 125277569024, + "reserved_bytes_current_delta": 14539554816, + "reserved_bytes_peak": 125277569024, + "reserved_bytes_peak_delta": 14539554816, + "segment_count_current": 853, + "segment_count_current_delta": 10, + "segment_count_peak": 853, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T192502041162Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.097187Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192502041162Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192502041162Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..859999787e80a0e14211789c0aa7b387832e0b69 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192502041162Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:313d136cb0764987ef5a8bad1e18ea606351ef8abd2cefd6d849e0041af2dd7a +size 2957787 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192502868502Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192502868502Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e65036ad0519fa1534b3d49a978dbd5e2d5a5f51 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192502868502Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39028323328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121446202368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39028323328, + "active_bytes_peak": 121446202368, + "allocated_bytes_current": 39028323328, + "allocated_bytes_peak": 121446202368, + "device": 5, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_peak": 28353631232, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 125277569024, + "reserved_bytes_peak": 125277569024, + "segment_count_current": 853, + "segment_count_peak": 853, + "timestamp_utc": "2026-08-26T19:25:02.868097Z", + "world_size": 8 + }, + "allocated_bytes_current": 39028323328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121446202368, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39028323328, + "active_bytes_peak": 121446202368, + "allocated_bytes_current": 39028323328, + "allocated_bytes_peak": 121446202368, + "device": 5, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_peak": 28353631232, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 125277569024, + "reserved_bytes_peak": 125277569024, + "segment_count_current": 853, + "segment_count_peak": 853, + "timestamp_utc": "2026-08-26T19:25:02.103154Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28353631232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 125277569024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125277569024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 853, + "segment_count_current_delta": 0, + "segment_count_peak": 853, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T192502868502Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.919117Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192502868502Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192502868502Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a0213dddd631d1a9ad48a86a12c6c3dd5613d805 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192502868502Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52fe84158f0755340cfa3abcfab2486fbae6890e267e5505c5ac2e62d2d5c929 +size 2957123 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192503679395Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192503679395Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2572b4d6c5bb3ca5bbc6eb698becb113b08f6f60 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192503679395Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39028323328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121446202368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39028323328, + "active_bytes_peak": 121446202368, + "allocated_bytes_current": 39028323328, + "allocated_bytes_peak": 121446202368, + "device": 5, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_peak": 28353631232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 125277569024, + "reserved_bytes_peak": 125277569024, + "segment_count_current": 853, + "segment_count_peak": 853, + "timestamp_utc": "2026-08-26T19:25:03.678991Z", + "world_size": 8 + }, + "allocated_bytes_current": 39028323328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121446202368, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39028323328, + "active_bytes_peak": 121446202368, + "allocated_bytes_current": 39028323328, + "allocated_bytes_peak": 121446202368, + "device": 5, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_peak": 28353631232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 125277569024, + "reserved_bytes_peak": 125277569024, + "segment_count_current": 853, + "segment_count_peak": 853, + "timestamp_utc": "2026-08-26T19:25:02.924641Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28353631232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 125277569024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125277569024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 853, + "segment_count_current_delta": 0, + "segment_count_peak": 853, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T192503679395Z.pickle", + "timestamp_utc": "2026-08-26T19:25:03.731002Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192503679395Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192503679395Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cd5441ad3308a969a49d3419710b206c334ca316 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192503679395Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f66bb7f444addb6028e729b4323256b3e0333212819e3acc67e3c9bcc592a05c +size 2957157 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192504498054Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192504498054Z.json new file mode 100644 index 0000000000000000000000000000000000000000..941afc4a90bb77dd313f38c42455adfa00237c0c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192504498054Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39028323328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121446202368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39028323328, + "active_bytes_peak": 121446202368, + "allocated_bytes_current": 39028323328, + "allocated_bytes_peak": 121446202368, + "device": 5, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_peak": 28353631232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 125277569024, + "reserved_bytes_peak": 125277569024, + "segment_count_current": 853, + "segment_count_peak": 853, + "timestamp_utc": "2026-08-26T19:25:04.497653Z", + "world_size": 8 + }, + "allocated_bytes_current": 39028323328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121446202368, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39028323328, + "active_bytes_peak": 121446202368, + "allocated_bytes_current": 39028323328, + "allocated_bytes_peak": 121446202368, + "device": 5, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_peak": 28353631232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 125277569024, + "reserved_bytes_peak": 125277569024, + "segment_count_current": 853, + "segment_count_peak": 853, + "timestamp_utc": "2026-08-26T19:25:03.736643Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16099511296, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28353631232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 125277569024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125277569024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 853, + "segment_count_current_delta": 0, + "segment_count_peak": 853, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T192504498054Z.pickle", + "timestamp_utc": "2026-08-26T19:25:04.551931Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192504498054Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192504498054Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f7d4c4cc89e0c1ecd959c8e6f92aff47c78fce6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192504498054Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6d75e10a7bb17b5046c612ccdd377b044d597aab15f6762e641b5c8876a7012 +size 2957126 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192501130651Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192501130651Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ba3c5ef3327aea2a53e1a7f69dd998a7fb726d0e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192501130651Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39031955456, + "active_bytes_current_delta": 15986042368, + "active_bytes_peak": 107189430784, + "active_bytes_peak_delta": 84130934784, + "after": { + "active_bytes_current": 39031955456, + "active_bytes_peak": 107189430784, + "allocated_bytes_current": 39031955456, + "allocated_bytes_peak": 107189430784, + "device": 6, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_peak": 28298123776, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 110878523392, + "reserved_bytes_peak": 110878523392, + "segment_count_current": 850, + "segment_count_peak": 850, + "timestamp_utc": "2026-08-26T19:25:01.130100Z", + "world_size": 8 + }, + "allocated_bytes_current": 39031955456, + "allocated_bytes_current_delta": 15986042368, + "allocated_bytes_peak": 107189430784, + "allocated_bytes_peak_delta": 84130934784, + "before": { + "active_bytes_current": 23045913088, + "active_bytes_peak": 23058496000, + "allocated_bytes_current": 23045913088, + "allocated_bytes_peak": 23058496000, + "device": 6, + "inactive_split_bytes_current": 370886144, + "inactive_split_bytes_peak": 883069440, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 25761415168, + "reserved_bytes_peak": 25761415168, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-26T19:24:30.831218Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_current_delta": 15861307904, + "inactive_split_bytes_peak": 28298123776, + "inactive_split_bytes_peak_delta": 27415054336, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 110878523392, + "reserved_bytes_current_delta": 85117108224, + "reserved_bytes_peak": 110878523392, + "reserved_bytes_peak_delta": 85117108224, + "segment_count_current": 850, + "segment_count_current_delta": 293, + "segment_count_peak": 850, + "segment_count_peak_delta": 293, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T192501130651Z.pickle", + "timestamp_utc": "2026-08-26T19:25:01.189947Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192501130651Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192501130651Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d6652a3b6199a0620166c7a73fedbdff13bd21b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192501130651Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05828fda6c5b671d83e17443788dc67b496ead5b88a0e53878b98013a9d71206 +size 2943602 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192502041177Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192502041177Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d3a17ce406d632a25481a608ab78d79d15188b7f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192502041177Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39031955456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121483363840, + "active_bytes_peak_delta": 14293933056, + "after": { + "active_bytes_current": 39031955456, + "active_bytes_peak": 121483363840, + "allocated_bytes_current": 39031955456, + "allocated_bytes_peak": 121483363840, + "device": 6, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_peak": 28298123776, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 125424369664, + "reserved_bytes_peak": 125424369664, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-26T19:25:02.040761Z", + "world_size": 8 + }, + "allocated_bytes_current": 39031955456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121483363840, + "allocated_bytes_peak_delta": 14293933056, + "before": { + "active_bytes_current": 39031955456, + "active_bytes_peak": 107189430784, + "allocated_bytes_current": 39031955456, + "allocated_bytes_peak": 107189430784, + "device": 6, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_peak": 28298123776, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 110878523392, + "reserved_bytes_peak": 110878523392, + "segment_count_current": 850, + "segment_count_peak": 850, + "timestamp_utc": "2026-08-26T19:25:01.198581Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28298123776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 125424369664, + "reserved_bytes_current_delta": 14545846272, + "reserved_bytes_peak": 125424369664, + "reserved_bytes_peak_delta": 14545846272, + "segment_count_current": 863, + "segment_count_current_delta": 13, + "segment_count_peak": 863, + "segment_count_peak_delta": 13, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T192502041177Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.095021Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192502041177Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192502041177Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b0e3b26867092c1f9399d1d948898136e85421f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192502041177Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75975f9e268be755bffa835cc5e0f8bf0c107145016a22d0331b951275aac845 +size 2956581 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192502868509Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192502868509Z.json new file mode 100644 index 0000000000000000000000000000000000000000..141aa1d9c7a9a96d7e98db1109976c1dbe224911 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192502868509Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39031955456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121506166784, + "active_bytes_peak_delta": 22802944, + "after": { + "active_bytes_current": 39031955456, + "active_bytes_peak": 121506166784, + "allocated_bytes_current": 39031955456, + "allocated_bytes_peak": 121506166784, + "device": 6, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_peak": 28298123776, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 125424369664, + "reserved_bytes_peak": 125424369664, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-26T19:25:02.868103Z", + "world_size": 8 + }, + "allocated_bytes_current": 39031955456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121506166784, + "allocated_bytes_peak_delta": 22802944, + "before": { + "active_bytes_current": 39031955456, + "active_bytes_peak": 121483363840, + "allocated_bytes_current": 39031955456, + "allocated_bytes_peak": 121483363840, + "device": 6, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_peak": 28298123776, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 125424369664, + "reserved_bytes_peak": 125424369664, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-26T19:25:02.101152Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28298123776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 125424369664, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125424369664, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 863, + "segment_count_current_delta": 0, + "segment_count_peak": 863, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T192502868509Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.917593Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192502868509Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192502868509Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4cc001c4d77b4a314c5ce499e875b118b1992b1a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192502868509Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffc7973d6b0834fc60757b422527c68f19f09475218425411ca420995bddbcab +size 2955843 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192503679421Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192503679421Z.json new file mode 100644 index 0000000000000000000000000000000000000000..07152a6ab5b76aaecfcb8864be1011f316edaece --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192503679421Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39031955456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121506166784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39031955456, + "active_bytes_peak": 121506166784, + "allocated_bytes_current": 39031955456, + "allocated_bytes_peak": 121506166784, + "device": 6, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_peak": 28298123776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 125424369664, + "reserved_bytes_peak": 125424369664, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-26T19:25:03.679007Z", + "world_size": 8 + }, + "allocated_bytes_current": 39031955456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121506166784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39031955456, + "active_bytes_peak": 121506166784, + "allocated_bytes_current": 39031955456, + "allocated_bytes_peak": 121506166784, + "device": 6, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_peak": 28298123776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 125424369664, + "reserved_bytes_peak": 125424369664, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-26T19:25:02.923342Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28298123776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 125424369664, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125424369664, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 863, + "segment_count_current_delta": 0, + "segment_count_peak": 863, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T192503679421Z.pickle", + "timestamp_utc": "2026-08-26T19:25:03.728552Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192503679421Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192503679421Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..242d65295e451a4c29a3ce1f037a41c36021c4fc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192503679421Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e623c0f24917bec2a3b563e4981d154454807f94862d7927e41f2576d13f433 +size 2955846 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192504498009Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192504498009Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4364e921ed4bcb4736eee1e8a50078b59d4a2896 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192504498009Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39031955456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121506166784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 39031955456, + "active_bytes_peak": 121506166784, + "allocated_bytes_current": 39031955456, + "allocated_bytes_peak": 121506166784, + "device": 6, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_peak": 28298123776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 125424369664, + "reserved_bytes_peak": 125424369664, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-26T19:25:04.497613Z", + "world_size": 8 + }, + "allocated_bytes_current": 39031955456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121506166784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 39031955456, + "active_bytes_peak": 121506166784, + "allocated_bytes_current": 39031955456, + "allocated_bytes_peak": 121506166784, + "device": 6, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_peak": 28298123776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 125424369664, + "reserved_bytes_peak": 125424369664, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-26T19:25:03.734255Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16232194048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28298123776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 125424369664, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125424369664, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 863, + "segment_count_current_delta": 0, + "segment_count_peak": 863, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T192504498009Z.pickle", + "timestamp_utc": "2026-08-26T19:25:04.547710Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192504498009Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192504498009Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..93678ede452f4c967a275e605306d62ebcb4a64c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192504498009Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adb1c71ba86f2ae921d5696ffc860889ccdfb66307dccd6a1dcce4ce2b3d5b96 +size 2955842 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192501130718Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192501130718Z.json new file mode 100644 index 0000000000000000000000000000000000000000..10f6bff86ea1063d61e54c506691bfacbf6ae5d1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192501130718Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39024664576, + "active_bytes_current_delta": 15976193024, + "active_bytes_peak": 107179177472, + "active_bytes_peak_delta": 84118123008, + "after": { + "active_bytes_current": 39024664576, + "active_bytes_peak": 107179177472, + "allocated_bytes_current": 39024664576, + "allocated_bytes_peak": 107179177472, + "device": 7, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_peak": 28335117824, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 110899494912, + "reserved_bytes_peak": 110899494912, + "segment_count_current": 850, + "segment_count_peak": 850, + "timestamp_utc": "2026-08-26T19:25:01.130226Z", + "world_size": 8 + }, + "allocated_bytes_current": 39024664576, + "allocated_bytes_current_delta": 15976193024, + "allocated_bytes_peak": 107179177472, + "allocated_bytes_peak_delta": 84118123008, + "before": { + "active_bytes_current": 23048471552, + "active_bytes_peak": 23061054464, + "allocated_bytes_current": 23048471552, + "allocated_bytes_peak": 23061054464, + "device": 7, + "inactive_split_bytes_current": 515128320, + "inactive_split_bytes_peak": 883069440, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 25784483840, + "reserved_bytes_peak": 25784483840, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-26T19:24:30.831279Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_current_delta": 15745328128, + "inactive_split_bytes_peak": 28335117824, + "inactive_split_bytes_peak_delta": 27452048384, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 110899494912, + "reserved_bytes_current_delta": 85115011072, + "reserved_bytes_peak": 110899494912, + "reserved_bytes_peak_delta": 85115011072, + "segment_count_current": 850, + "segment_count_current_delta": 292, + "segment_count_peak": 850, + "segment_count_peak_delta": 292, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T192501130718Z.pickle", + "timestamp_utc": "2026-08-26T19:25:01.190962Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192501130718Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192501130718Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..59880ecfa6a7f9525ae7e9be289527b184a3a55e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192501130718Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db758e81f09cfe034be11564971714469e126cfa7bd38d289c6de54e48560b8a +size 3001013 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192502041262Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192502041262Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7317d292c57aac8f8e50f2597d95e204482836ff --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192502041262Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39024664576, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121357208064, + "active_bytes_peak_delta": 14178030592, + "after": { + "active_bytes_current": 39024664576, + "active_bytes_peak": 121357208064, + "allocated_bytes_current": 39024664576, + "allocated_bytes_peak": 121357208064, + "device": 7, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_peak": 28335117824, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 125439049728, + "reserved_bytes_peak": 125439049728, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:02.040844Z", + "world_size": 8 + }, + "allocated_bytes_current": 39024664576, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121357208064, + "allocated_bytes_peak_delta": 14178030592, + "before": { + "active_bytes_current": 39024664576, + "active_bytes_peak": 107179177472, + "allocated_bytes_current": 39024664576, + "allocated_bytes_peak": 107179177472, + "device": 7, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_peak": 28335117824, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 110899494912, + "reserved_bytes_peak": 110899494912, + "segment_count_current": 850, + "segment_count_peak": 850, + "timestamp_utc": "2026-08-26T19:25:01.199715Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28335117824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 125439049728, + "reserved_bytes_current_delta": 14539554816, + "reserved_bytes_peak": 125439049728, + "reserved_bytes_peak_delta": 14539554816, + "segment_count_current": 860, + "segment_count_current_delta": 10, + "segment_count_peak": 860, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T192502041262Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.095523Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192502041262Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192502041262Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dc1f95dffb56d7e5e19662682d3892605addf3cd --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192502041262Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65746174e637e9e48d6afaa2a334226a26cac8a4b8d6cb26cd18e27dfa304ce4 +size 2942655 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192502868578Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192502868578Z.json new file mode 100644 index 0000000000000000000000000000000000000000..101190ef7915cd132da0c4ca4501143e899dbfe5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192502868578Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39024664576, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121366956032, + "active_bytes_peak_delta": 9747968, + "after": { + "active_bytes_current": 39024664576, + "active_bytes_peak": 121366956032, + "allocated_bytes_current": 39024664576, + "allocated_bytes_peak": 121366956032, + "device": 7, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_peak": 28335117824, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 125439049728, + "reserved_bytes_peak": 125439049728, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:02.868156Z", + "world_size": 8 + }, + "allocated_bytes_current": 39024664576, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121366956032, + "allocated_bytes_peak_delta": 9747968, + "before": { + "active_bytes_current": 39024664576, + "active_bytes_peak": 121357208064, + "allocated_bytes_current": 39024664576, + "allocated_bytes_peak": 121357208064, + "device": 7, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_peak": 28335117824, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 125439049728, + "reserved_bytes_peak": 125439049728, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:02.101556Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28335117824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 125439049728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125439049728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 860, + "segment_count_current_delta": 0, + "segment_count_peak": 860, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T192502868578Z.pickle", + "timestamp_utc": "2026-08-26T19:25:02.918885Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192502868578Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192502868578Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5abc737ffa229ec3bebab0cc0e78e866004048df --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192502868578Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be3d8025dd061ab7e99b05700218662a53fcfa6e46f7e51132937d2d26c7aee1 +size 2942094 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192503679459Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192503679459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e1e34a6f35c454346c354def6edafc18472b84b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192503679459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39024664576, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121400211968, + "active_bytes_peak_delta": 33255936, + "after": { + "active_bytes_current": 39024664576, + "active_bytes_peak": 121400211968, + "allocated_bytes_current": 39024664576, + "allocated_bytes_peak": 121400211968, + "device": 7, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_peak": 28335117824, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 125439049728, + "reserved_bytes_peak": 125439049728, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:03.679055Z", + "world_size": 8 + }, + "allocated_bytes_current": 39024664576, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121400211968, + "allocated_bytes_peak_delta": 33255936, + "before": { + "active_bytes_current": 39024664576, + "active_bytes_peak": 121366956032, + "allocated_bytes_current": 39024664576, + "allocated_bytes_peak": 121366956032, + "device": 7, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_peak": 28335117824, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 125439049728, + "reserved_bytes_peak": 125439049728, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:02.924644Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28335117824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 125439049728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125439049728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 860, + "segment_count_current_delta": 0, + "segment_count_peak": 860, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T192503679459Z.pickle", + "timestamp_utc": "2026-08-26T19:25:03.729774Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192503679459Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192503679459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..667a5743e01bd394425bcaf0d65596522a9b2677 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192503679459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a917bcffc64634e69b780d9c168a943ffea767f515147431d600988223c3478f +size 2942100 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192504498097Z.json b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192504498097Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f011ac08e8eec333ad4bb0ae21891b019655082b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192504498097Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 39024664576, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121497929728, + "active_bytes_peak_delta": 97717760, + "after": { + "active_bytes_current": 39024664576, + "active_bytes_peak": 121497929728, + "allocated_bytes_current": 39024664576, + "allocated_bytes_peak": 121497929728, + "device": 7, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_peak": 28335117824, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 125439049728, + "reserved_bytes_peak": 125439049728, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:04.497661Z", + "world_size": 8 + }, + "allocated_bytes_current": 39024664576, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 121497929728, + "allocated_bytes_peak_delta": 97717760, + "before": { + "active_bytes_current": 39024664576, + "active_bytes_peak": 121400211968, + "allocated_bytes_current": 39024664576, + "allocated_bytes_peak": 121400211968, + "device": 7, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_peak": 28335117824, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 125439049728, + "reserved_bytes_peak": 125439049728, + "segment_count_current": 860, + "segment_count_peak": 860, + "timestamp_utc": "2026-08-26T19:25:03.735515Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16260456448, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 28335117824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 125439049728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 125439049728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 860, + "segment_count_current_delta": 0, + "segment_count_peak": 860, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T192504498097Z.pickle", + "timestamp_utc": "2026-08-26T19:25:04.549934Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192504498097Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192504498097Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8331aac36ec67e46201645cb37023b1595e15b9c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192504498097Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce49e68b5e9930ac370a3db3252a7fc19dfacb942111ca32278067b5da614462 +size 2942100 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..8835358fa518f45765c64ef1c87b4fd73e5b4cc7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-better-memory-dump-support +commit=c3a8a8a7 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=0 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=dynamic +capacity=dynamic +cuda_graph=0 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=5 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=768 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..68b60d58c316cd647b511cbc01cba9c0175324b0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log @@ -0,0 +1,1620 @@ +============================================================== + DSV2_LITE_L24 / ultraep (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=dynamic cap=dynamic cg=0 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ c3a8a8a7 +============================================================== +W0826 19:24:02.793000 663160 torch/distributed/run.py:766] +W0826 19:24:02.793000 663160 torch/distributed/run.py:766] ***************************************** +W0826 19:24:02.793000 663160 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:24:02.793000 663160 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ False + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... False + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to [Gloo] Rank 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 000 is connected to is connected to is connected to 000 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 000 + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W826 19:24:21.194155494 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:24:21.197325118 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:24:21.197423715 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:24:21.197448573 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:24:21.197560072 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:24:21.198764416 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:24:21.198784290 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:24:21.338677733 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.710 seconds +time to initialize megatron (seconds): 7.057 +[after megatron is initialized] datetime: 2026-08-26 19:24:26 +building GPT model ... +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1526: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4188125184 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_q_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc2.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:24:30 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 40 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(40, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from ddd6755186c6b13339ab7bd4e0fae553-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from ddd6755186c6b13339ab7bd4e0fae553-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from ddd6755186c6b13339ab7bd4e0fae553-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:24:30 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3695.78, 3709.61) + train/valid/test-data-iterators-setup ..........: (70.28, 91.59) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:24:30 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:25:01] iteration 1/ 5 | consumed samples: 8 | elapsed time per iteration (ms): 30298.9 | throughput per GPU (TFLOP/s/GPU): 4.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092127E+01 | loss scale: 1.0 | grad norm: 30.623 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 13.53 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 37223.82080078125 | max allocated: 102282.55712890625 | reserved: 105740.0 | max reserved: 105740.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:25:02] iteration 2/ 5 | consumed samples: 16 | elapsed time per iteration (ms): 912.3 | throughput per GPU (TFLOP/s/GPU): 139.8 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091995E+01 | loss scale: 1.0 | grad norm: 30.567 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:25:02] iteration 3/ 5 | consumed samples: 24 | elapsed time per iteration (ms): 827.2 | throughput per GPU (TFLOP/s/GPU): 154.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952354E+00 | loss scale: 1.0 | grad norm: 42.232 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:25:03] iteration 4/ 5 | consumed samples: 32 | elapsed time per iteration (ms): 810.9 | throughput per GPU (TFLOP/s/GPU): 157.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.477323E+00 | loss scale: 1.0 | grad norm: 7.143 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:25:04] iteration 5/ 5 | consumed samples: 40 | elapsed time per iteration (ms): 818.5 | throughput per GPU (TFLOP/s/GPU): 155.9 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.119604E+01 | loss scale: 1.0 | grad norm: 396.977 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 19:25:04 +[rank0]:[W826 19:25:05.826343909 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 19:25:07.380537636 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 19:25:07.382971592 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 19:25:07.384953796 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 19:25:07.392223328 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 19:25:07.412200530 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 19:25:07.936666352 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 19:25:07.941532250 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:25:11.486000 663160 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:25:11.487000 663160 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 663249 closing signal SIGTERM +W0826 19:25:11.487000 663160 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 663250 closing signal SIGTERM +W0826 19:25:11.489000 663160 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 663251 closing signal SIGTERM +W0826 19:25:11.489000 663160 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 663252 closing signal SIGTERM +W0826 19:25:11.489000 663160 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 663253 closing signal SIGTERM +W0826 19:25:11.491000 663160 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 663254 closing signal SIGTERM +W0826 19:25:11.492000 663160 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 663255 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 663160 got signal: 15 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..2cd63aca8cb9d6cede5ff234f08df8a064e92526 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,192 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 47 +3, 47 +4, 47 +5, 629 +6, 629 +7, 629 +0, 933 +1, 939 +2, 947 +3, 963 +4, 935 +5, 931 +6, 937 +7, 937 +0, 1803 +1, 1803 +2, 1803 +3, 1803 +4, 1803 +5, 1803 +6, 1803 +7, 1803 +0, 2347 +1, 2345 +2, 2313 +3, 2339 +4, 2333 +5, 2325 +6, 2341 +7, 2355 +0, 46703 +1, 46137 +2, 46137 +3, 46837 +4, 46837 +5, 46837 +6, 46837 +7, 46837 +0, 47125 +1, 46555 +2, 46555 +3, 47255 +4, 47255 +5, 47255 +6, 47255 +7, 47255 +0, 48413 +1, 47843 +2, 47843 +3, 48543 +4, 48543 +5, 48543 +6, 48543 +7, 48543 +0, 48637 +1, 48067 +2, 48067 +3, 48767 +4, 48767 +5, 48767 +6, 48767 +7, 48767 +0, 48637 +1, 48067 +2, 48067 +3, 48767 +4, 48767 +5, 48767 +6, 48767 +7, 48767 +0, 50457 +1, 49887 +2, 49887 +3, 50587 +4, 50587 +5, 50587 +6, 50587 +7, 50587 +0, 72977 +1, 72983 +2, 72983 +3, 72979 +4, 72979 +5, 72979 +6, 72979 +7, 72979 +0, 73623 +1, 73629 +2, 73629 +3, 73625 +4, 73625 +5, 73625 +6, 73625 +7, 73625 +0, 73623 +1, 73631 +2, 73631 +3, 73627 +4, 73627 +5, 73627 +6, 73627 +7, 73627 +0, 73987 +1, 73891 +2, 73915 +3, 73901 +4, 73863 +5, 73883 +6, 73879 +7, 73893 +0, 82337 +1, 82021 +2, 82097 +3, 82291 +4, 82299 +5, 82251 +6, 82287 +7, 82251 +0, 96261 +1, 96297 +2, 96293 +3, 96229 +4, 96229 +5, 96229 +6, 96229 +7, 96229 +0, 97969 +1, 97491 +2, 97527 +3, 98047 +4, 98047 +5, 98047 +6, 98047 +7, 98047 +0, 97075 +1, 96597 +2, 96633 +3, 97153 +4, 97153 +5, 97153 +6, 97153 +7, 97153 +0, 1 +1, 4 +2, 4 +3, 4 +4, 260 +5, 94201 +6, 1284 +7, 4 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192610591182Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192610591182Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bfb457fd61649e555157261ba71a9aaedde0c499 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192610591182Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67407492608, + "active_bytes_current_delta": 26722485760, + "active_bytes_peak": 68191325696, + "active_bytes_peak_delta": 27468570112, + "after": { + "active_bytes_current": 67407492608, + "active_bytes_peak": 68191325696, + "allocated_bytes_current": 67407492608, + "allocated_bytes_peak": 68191325696, + "device": 0, + "inactive_split_bytes_current": 2110999040, + "inactive_split_bytes_peak": 4587795968, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 77112279040, + "reserved_bytes_peak": 77112279040, + "segment_count_current": 995, + "segment_count_peak": 995, + "timestamp_utc": "2026-08-26T19:26:10.590560Z", + "world_size": 8 + }, + "allocated_bytes_current": 67407492608, + "allocated_bytes_current_delta": 26722485760, + "allocated_bytes_peak": 68191325696, + "allocated_bytes_peak_delta": 27468570112, + "before": { + "active_bytes_current": 40685006848, + "active_bytes_peak": 40722755584, + "allocated_bytes_current": 40685006848, + "allocated_bytes_peak": 40722755584, + "device": 0, + "inactive_split_bytes_current": 373035008, + "inactive_split_bytes_peak": 593235968, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 43723522048, + "reserved_bytes_peak": 43723522048, + "segment_count_current": 262, + "segment_count_peak": 262, + "timestamp_utc": "2026-08-26T19:25:44.719482Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2110999040, + "inactive_split_bytes_current_delta": 1737964032, + "inactive_split_bytes_peak": 4587795968, + "inactive_split_bytes_peak_delta": 3994560000, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 77112279040, + "reserved_bytes_current_delta": 33388756992, + "reserved_bytes_peak": 77112279040, + "reserved_bytes_peak_delta": 33388756992, + "segment_count_current": 995, + "segment_count_current_delta": 733, + "segment_count_peak": 995, + "segment_count_peak_delta": 733, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T192610591182Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.617407Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192610591182Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192610591182Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c1f76c9f52524a22199417d056a9726b72034a5b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192610591182Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34a48bce16936fc12ab3afae390c968fc8e95dbad8477349699ca0fd282fde04 +size 1459677 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192610914842Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192610914842Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e49657f2c437db4d7ef53839d1cec053a255c6e6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192610914842Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67407492608, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89682062336, + "active_bytes_peak_delta": 21490736640, + "after": { + "active_bytes_current": 67407492608, + "active_bytes_peak": 89682062336, + "allocated_bytes_current": 67407492608, + "allocated_bytes_peak": 89682062336, + "device": 0, + "inactive_split_bytes_current": 2113096192, + "inactive_split_bytes_peak": 4587795968, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 91194654720, + "reserved_bytes_peak": 91194654720, + "segment_count_current": 1112, + "segment_count_peak": 1112, + "timestamp_utc": "2026-08-26T19:26:10.914440Z", + "world_size": 8 + }, + "allocated_bytes_current": 67407492608, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89682062336, + "allocated_bytes_peak_delta": 21490736640, + "before": { + "active_bytes_current": 67407492608, + "active_bytes_peak": 68191325696, + "allocated_bytes_current": 67407492608, + "allocated_bytes_peak": 68191325696, + "device": 0, + "inactive_split_bytes_current": 2110999040, + "inactive_split_bytes_peak": 4587795968, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 77112279040, + "reserved_bytes_peak": 77112279040, + "segment_count_current": 995, + "segment_count_peak": 995, + "timestamp_utc": "2026-08-26T19:26:10.620287Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2113096192, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 4587795968, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 91194654720, + "reserved_bytes_current_delta": 14082375680, + "reserved_bytes_peak": 91194654720, + "reserved_bytes_peak_delta": 14082375680, + "segment_count_current": 1112, + "segment_count_current_delta": 117, + "segment_count_peak": 1112, + "segment_count_peak_delta": 117, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T192610914842Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.935061Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192610914842Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192610914842Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f321d7ae81efe35616128557576236e3b66680ef --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192610914842Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1d599b87106ed462f0e904908aff098098507bdffd5a9041fd197c5ccbfb642 +size 1183910 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192611159055Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192611159055Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9c74cc7534609da14a224f2448f0c1772841e08 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192611159055Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67407492608, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89682062336, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67407492608, + "active_bytes_peak": 89682062336, + "allocated_bytes_current": 67407492608, + "allocated_bytes_peak": 89682062336, + "device": 0, + "inactive_split_bytes_current": 2113096192, + "inactive_split_bytes_peak": 4587795968, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 91196751872, + "reserved_bytes_peak": 91196751872, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:26:11.158663Z", + "world_size": 8 + }, + "allocated_bytes_current": 67407492608, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89682062336, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67407492608, + "active_bytes_peak": 89682062336, + "allocated_bytes_current": 67407492608, + "allocated_bytes_peak": 89682062336, + "device": 0, + "inactive_split_bytes_current": 2113096192, + "inactive_split_bytes_peak": 4587795968, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 91194654720, + "reserved_bytes_peak": 91194654720, + "segment_count_current": 1112, + "segment_count_peak": 1112, + "timestamp_utc": "2026-08-26T19:26:10.937599Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2113096192, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4587795968, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 91196751872, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 91196751872, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 1113, + "segment_count_current_delta": 1, + "segment_count_peak": 1113, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T192611159055Z.pickle", + "timestamp_utc": "2026-08-26T19:26:11.177561Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192611159055Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192611159055Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c7b9e1ffa921efe37f79a9d0fa4731f7b13eb575 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192611159055Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c8f8dee02d23f6f289a800b61ec69c14ed0bbb9d3cf3c7e01d180d6ed7902b7 +size 1174228 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192612968846Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192612968846Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad778c680cca4ac51f565d5f94a94512a230bb8e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192612968846Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67424536064, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 89682062336, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67424536064, + "active_bytes_peak": 89682062336, + "allocated_bytes_current": 67424536064, + "allocated_bytes_peak": 89682062336, + "device": 0, + "inactive_split_bytes_current": 2354002432, + "inactive_split_bytes_peak": 5787103232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 92968845312, + "reserved_bytes_peak": 92968845312, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:26:12.968418Z", + "world_size": 8 + }, + "allocated_bytes_current": 67424536064, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 89682062336, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67407492608, + "active_bytes_peak": 89682062336, + "allocated_bytes_current": 67407492608, + "allocated_bytes_peak": 89682062336, + "device": 0, + "inactive_split_bytes_current": 2113096192, + "inactive_split_bytes_peak": 4587795968, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 91196751872, + "reserved_bytes_peak": 91196751872, + "segment_count_current": 1113, + "segment_count_peak": 1113, + "timestamp_utc": "2026-08-26T19:26:11.179848Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2354002432, + "inactive_split_bytes_current_delta": 240906240, + "inactive_split_bytes_peak": 5787103232, + "inactive_split_bytes_peak_delta": 1199307264, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 92968845312, + "reserved_bytes_current_delta": 1772093440, + "reserved_bytes_peak": 92968845312, + "reserved_bytes_peak_delta": 1772093440, + "segment_count_current": 1127, + "segment_count_current_delta": 14, + "segment_count_peak": 1127, + "segment_count_peak_delta": 14, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T192612968846Z.pickle", + "timestamp_utc": "2026-08-26T19:26:12.992551Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192612968846Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192612968846Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..935158df47b20e9d9108915ed5befe406ef6a6ee --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192612968846Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53dd60d80dc9b35db8fe4e3df14ce8e3b4e9ecd07eac6913134d1f12dc8045f1 +size 1266288 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192613193594Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192613193594Z.json new file mode 100644 index 0000000000000000000000000000000000000000..56db22b8841436791688a19b97b109b6f391800e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192613193594Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67424536064, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89682062336, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67424536064, + "active_bytes_peak": 89682062336, + "allocated_bytes_current": 67424536064, + "allocated_bytes_peak": 89682062336, + "device": 0, + "inactive_split_bytes_current": 2354002432, + "inactive_split_bytes_peak": 5787103232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 92968845312, + "reserved_bytes_peak": 92968845312, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:26:13.193181Z", + "world_size": 8 + }, + "allocated_bytes_current": 67424536064, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89682062336, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67424536064, + "active_bytes_peak": 89682062336, + "allocated_bytes_current": 67424536064, + "allocated_bytes_peak": 89682062336, + "device": 0, + "inactive_split_bytes_current": 2354002432, + "inactive_split_bytes_peak": 5787103232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 92968845312, + "reserved_bytes_peak": 92968845312, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:26:12.995242Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2354002432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5787103232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 92968845312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 92968845312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1127, + "segment_count_current_delta": 0, + "segment_count_peak": 1127, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T192613193594Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.199980Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192613193594Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192613193594Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c332e170e69db9582e13b730d965bbb23b08c4cb --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192613193594Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f38cb7f7df7d7bdc92432595bd288045752ac5c99023fe6e11d86123fb34a3ce +size 276029 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T192613392424Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T192613392424Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3bfc451de6e9c0234612b5e8f737eab0bdcd8576 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T192613392424Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67424536064, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89682062336, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67424536064, + "active_bytes_peak": 89682062336, + "allocated_bytes_current": 67424536064, + "allocated_bytes_peak": 89682062336, + "device": 0, + "inactive_split_bytes_current": 2354002432, + "inactive_split_bytes_peak": 5787103232, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 92968845312, + "reserved_bytes_peak": 92968845312, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:26:13.392037Z", + "world_size": 8 + }, + "allocated_bytes_current": 67424536064, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89682062336, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67424536064, + "active_bytes_peak": 89682062336, + "allocated_bytes_current": 67424536064, + "allocated_bytes_peak": 89682062336, + "device": 0, + "inactive_split_bytes_current": 2354002432, + "inactive_split_bytes_peak": 5787103232, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 92968845312, + "reserved_bytes_peak": 92968845312, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:26:13.200942Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2354002432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5787103232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 92968845312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 92968845312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1127, + "segment_count_current_delta": 0, + "segment_count_peak": 1127, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T192613392424Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.398743Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T192613392424Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T192613392424Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..77ddded50fbe5f01e00535ea5154cf3d8ab420c2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T192613392424Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:083b01de6f3d1fd51a4b9ea08a5f7321536e83e8c2438ac82f1681befca2f2d4 +size 276029 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192610591217Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192610591217Z.json new file mode 100644 index 0000000000000000000000000000000000000000..def8a319c1ccf005a5329fb15be96237e89c0670 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192610591217Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67391359488, + "active_bytes_current_delta": 26705306112, + "active_bytes_peak": 68185932288, + "active_bytes_peak_delta": 27462130176, + "after": { + "active_bytes_current": 67391359488, + "active_bytes_peak": 68185932288, + "allocated_bytes_current": 67391359488, + "allocated_bytes_peak": 68185932288, + "device": 1, + "inactive_split_bytes_current": 1628009984, + "inactive_split_bytes_peak": 4420232192, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 76789317632, + "reserved_bytes_peak": 76789317632, + "segment_count_current": 998, + "segment_count_peak": 998, + "timestamp_utc": "2026-08-26T19:26:10.590585Z", + "world_size": 8 + }, + "allocated_bytes_current": 67391359488, + "allocated_bytes_current_delta": 26705306112, + "allocated_bytes_peak": 68185932288, + "allocated_bytes_peak_delta": 27462130176, + "before": { + "active_bytes_current": 40686053376, + "active_bytes_peak": 40723802112, + "allocated_bytes_current": 40686053376, + "allocated_bytes_peak": 40723802112, + "device": 1, + "inactive_split_bytes_current": 302782464, + "inactive_split_bytes_peak": 522983424, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 43130028032, + "reserved_bytes_peak": 43130028032, + "segment_count_current": 251, + "segment_count_peak": 251, + "timestamp_utc": "2026-08-26T19:25:44.719587Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1628009984, + "inactive_split_bytes_current_delta": 1325227520, + "inactive_split_bytes_peak": 4420232192, + "inactive_split_bytes_peak_delta": 3897248768, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 76789317632, + "reserved_bytes_current_delta": 33659289600, + "reserved_bytes_peak": 76789317632, + "reserved_bytes_peak_delta": 33659289600, + "segment_count_current": 998, + "segment_count_current_delta": 747, + "segment_count_peak": 998, + "segment_count_peak_delta": 747, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T192610591217Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.617619Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192610591217Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192610591217Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f9691c58d9c67d0c95270ea0ea4aee23465bacf6 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192610591217Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce4396fec790bc1fd10c757575adee1e209866b159729a1bb9f26e21606b992c +size 1458032 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192610914903Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192610914903Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67ba1404415390ff044e10ec836beb302e87f8fc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192610914903Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67391359488, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89685132800, + "active_bytes_peak_delta": 21499200512, + "after": { + "active_bytes_current": 67391359488, + "active_bytes_peak": 89685132800, + "allocated_bytes_current": 67391359488, + "allocated_bytes_peak": 89685132800, + "device": 1, + "inactive_split_bytes_current": 1630107136, + "inactive_split_bytes_peak": 4420232192, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 91234500608, + "reserved_bytes_peak": 91234500608, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:26:10.914501Z", + "world_size": 8 + }, + "allocated_bytes_current": 67391359488, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89685132800, + "allocated_bytes_peak_delta": 21499200512, + "before": { + "active_bytes_current": 67391359488, + "active_bytes_peak": 68185932288, + "allocated_bytes_current": 67391359488, + "allocated_bytes_peak": 68185932288, + "device": 1, + "inactive_split_bytes_current": 1628009984, + "inactive_split_bytes_peak": 4420232192, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 76789317632, + "reserved_bytes_peak": 76789317632, + "segment_count_current": 998, + "segment_count_peak": 998, + "timestamp_utc": "2026-08-26T19:26:10.620464Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1630107136, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 4420232192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 91234500608, + "reserved_bytes_current_delta": 14445182976, + "reserved_bytes_peak": 91234500608, + "reserved_bytes_peak_delta": 14445182976, + "segment_count_current": 1121, + "segment_count_current_delta": 123, + "segment_count_peak": 1121, + "segment_count_peak_delta": 123, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T192610914903Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.935527Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192610914903Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192610914903Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ad7ab887880affc9ba0fa8530deec53612ba2be8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192610914903Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:727bd2ff945e83c547621c76d46b3fd701a71d26358c4c91d771674fc2231b09 +size 1165623 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192611159138Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192611159138Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0aacf000aca2156c4faac3efed6ac02ff0785d96 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192611159138Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67391359488, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89685132800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67391359488, + "active_bytes_peak": 89685132800, + "allocated_bytes_current": 67391359488, + "allocated_bytes_peak": 89685132800, + "device": 1, + "inactive_split_bytes_current": 1628009984, + "inactive_split_bytes_peak": 4420232192, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 91234500608, + "reserved_bytes_peak": 91234500608, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:26:11.158755Z", + "world_size": 8 + }, + "allocated_bytes_current": 67391359488, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89685132800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67391359488, + "active_bytes_peak": 89685132800, + "allocated_bytes_current": 67391359488, + "allocated_bytes_peak": 89685132800, + "device": 1, + "inactive_split_bytes_current": 1630107136, + "inactive_split_bytes_peak": 4420232192, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 91234500608, + "reserved_bytes_peak": 91234500608, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:26:10.938092Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1628009984, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 4420232192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 91234500608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 91234500608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1121, + "segment_count_current_delta": 0, + "segment_count_peak": 1121, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T192611159138Z.pickle", + "timestamp_utc": "2026-08-26T19:26:11.179179Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192611159138Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192611159138Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3c13c886caf3217d070ee60d7093ce36fcd44fc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192611159138Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37810fc98821b61403c9d2df81550314caf9874f934d897b24f54b9621d669f0 +size 1181481 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192612968923Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192612968923Z.json new file mode 100644 index 0000000000000000000000000000000000000000..feabe83eccb9c2aa9503eb20728d280b7b6dc373 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192612968923Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408402944, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 89685132800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67408402944, + "active_bytes_peak": 89685132800, + "allocated_bytes_current": 67408402944, + "allocated_bytes_peak": 89685132800, + "device": 1, + "inactive_split_bytes_current": 1871013376, + "inactive_split_bytes_peak": 5302017024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 92467625984, + "reserved_bytes_peak": 92467625984, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:26:12.968505Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408402944, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 89685132800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67391359488, + "active_bytes_peak": 89685132800, + "allocated_bytes_current": 67391359488, + "allocated_bytes_peak": 89685132800, + "device": 1, + "inactive_split_bytes_current": 1628009984, + "inactive_split_bytes_peak": 4420232192, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 91234500608, + "reserved_bytes_peak": 91234500608, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:26:11.181485Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1871013376, + "inactive_split_bytes_current_delta": 243003392, + "inactive_split_bytes_peak": 5302017024, + "inactive_split_bytes_peak_delta": 881784832, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 92467625984, + "reserved_bytes_current_delta": 1233125376, + "reserved_bytes_peak": 92467625984, + "reserved_bytes_peak_delta": 1233125376, + "segment_count_current": 1122, + "segment_count_current_delta": 1, + "segment_count_peak": 1122, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T192612968923Z.pickle", + "timestamp_utc": "2026-08-26T19:26:12.992771Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192612968923Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192612968923Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..917b8d357c569c8a3516f4daf4c8eea19aad80c9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T192612968923Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdadffd802bf9829ddb230404e8345f4596f25b3d18b50b992a6c2adf881ed33 +size 1291930 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192613193651Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192613193651Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e0fe764d33a16024b26610cf6af77891b08d867 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192613193651Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408402944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89685132800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67408402944, + "active_bytes_peak": 89685132800, + "allocated_bytes_current": 67408402944, + "allocated_bytes_peak": 89685132800, + "device": 1, + "inactive_split_bytes_current": 1871013376, + "inactive_split_bytes_peak": 5302017024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 92467625984, + "reserved_bytes_peak": 92467625984, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:26:13.193250Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408402944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89685132800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408402944, + "active_bytes_peak": 89685132800, + "allocated_bytes_current": 67408402944, + "allocated_bytes_peak": 89685132800, + "device": 1, + "inactive_split_bytes_current": 1871013376, + "inactive_split_bytes_peak": 5302017024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 92467625984, + "reserved_bytes_peak": 92467625984, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:26:12.995487Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1871013376, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5302017024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 92467625984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 92467625984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1122, + "segment_count_current_delta": 0, + "segment_count_peak": 1122, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T192613193651Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.199894Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192613193651Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192613193651Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da3ada4f98bfc87e9a76057ad2335dcab5e9d360 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T192613193651Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3cae6ee3ef3f1f4c238d3437ad87cc78f5e9324b6a37503ba7d92006a7ad929 +size 274873 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T192613392524Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T192613392524Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7447783121adc385f80b3226294d941ad6443a0a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T192613392524Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408402944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89685132800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67408402944, + "active_bytes_peak": 89685132800, + "allocated_bytes_current": 67408402944, + "allocated_bytes_peak": 89685132800, + "device": 1, + "inactive_split_bytes_current": 1871013376, + "inactive_split_bytes_peak": 5302017024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 92467625984, + "reserved_bytes_peak": 92467625984, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:26:13.392137Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408402944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89685132800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408402944, + "active_bytes_peak": 89685132800, + "allocated_bytes_current": 67408402944, + "allocated_bytes_peak": 89685132800, + "device": 1, + "inactive_split_bytes_current": 1871013376, + "inactive_split_bytes_peak": 5302017024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 92467625984, + "reserved_bytes_peak": 92467625984, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:26:13.200867Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 1871013376, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5302017024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 92467625984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 92467625984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1122, + "segment_count_current_delta": 0, + "segment_count_peak": 1122, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T192613392524Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.398749Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T192613392524Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T192613392524Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3ebec198ae090c635f40c140ddabf7d56f7a7d95 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T192613392524Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2c460562d2841d674f1d863ccbd4e0235f20139d3b0ad3a778623edfbae5d89 +size 274873 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192610591261Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192610591261Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a4f6f40e2696a9a2a323e87e296ee7758ae57e01 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192610591261Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67391940608, + "active_bytes_current_delta": 26705887232, + "active_bytes_peak": 68186226176, + "active_bytes_peak_delta": 27462424064, + "after": { + "active_bytes_current": 67391940608, + "active_bytes_peak": 68186226176, + "allocated_bytes_current": 67391940608, + "allocated_bytes_peak": 68186226176, + "device": 2, + "inactive_split_bytes_current": 1665177600, + "inactive_split_bytes_peak": 4421015040, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 76827066368, + "reserved_bytes_peak": 76827066368, + "segment_count_current": 999, + "segment_count_peak": 999, + "timestamp_utc": "2026-08-26T19:26:10.590580Z", + "world_size": 8 + }, + "allocated_bytes_current": 67391940608, + "allocated_bytes_current_delta": 26705887232, + "allocated_bytes_peak": 68186226176, + "allocated_bytes_peak_delta": 27462424064, + "before": { + "active_bytes_current": 40686053376, + "active_bytes_peak": 40723802112, + "allocated_bytes_current": 40686053376, + "allocated_bytes_peak": 40723802112, + "device": 2, + "inactive_split_bytes_current": 283908096, + "inactive_split_bytes_peak": 504109056, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 43130028032, + "reserved_bytes_peak": 43130028032, + "segment_count_current": 251, + "segment_count_peak": 251, + "timestamp_utc": "2026-08-26T19:25:44.719723Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1665177600, + "inactive_split_bytes_current_delta": 1381269504, + "inactive_split_bytes_peak": 4421015040, + "inactive_split_bytes_peak_delta": 3916905984, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 76827066368, + "reserved_bytes_current_delta": 33697038336, + "reserved_bytes_peak": 76827066368, + "reserved_bytes_peak_delta": 33697038336, + "segment_count_current": 999, + "segment_count_current_delta": 748, + "segment_count_peak": 999, + "segment_count_peak_delta": 748, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T192610591261Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.617837Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192610591261Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192610591261Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f758ef6ba5d3eb1676e1a10aa7927a2cfa6bb161 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T192610591261Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8136e747bfd996efc33799bfc42175757b4442baf029f1a05704334a33a5433 +size 1455414 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192610914953Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192610914953Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9af08a5fccfec6383faa6e043cbbcaf73cc20155 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192610914953Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67391940608, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89684086272, + "active_bytes_peak_delta": 21497860096, + "after": { + "active_bytes_current": 67391940608, + "active_bytes_peak": 89684086272, + "allocated_bytes_current": 67391940608, + "allocated_bytes_peak": 89684086272, + "device": 2, + "inactive_split_bytes_current": 1665177600, + "inactive_split_bytes_peak": 4421015040, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 91230306304, + "reserved_bytes_peak": 91230306304, + "segment_count_current": 1119, + "segment_count_peak": 1119, + "timestamp_utc": "2026-08-26T19:26:10.914528Z", + "world_size": 8 + }, + "allocated_bytes_current": 67391940608, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89684086272, + "allocated_bytes_peak_delta": 21497860096, + "before": { + "active_bytes_current": 67391940608, + "active_bytes_peak": 68186226176, + "allocated_bytes_current": 67391940608, + "allocated_bytes_peak": 68186226176, + "device": 2, + "inactive_split_bytes_current": 1665177600, + "inactive_split_bytes_peak": 4421015040, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 76827066368, + "reserved_bytes_peak": 76827066368, + "segment_count_current": 999, + "segment_count_peak": 999, + "timestamp_utc": "2026-08-26T19:26:10.620714Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1665177600, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4421015040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 91230306304, + "reserved_bytes_current_delta": 14403239936, + "reserved_bytes_peak": 91230306304, + "reserved_bytes_peak_delta": 14403239936, + "segment_count_current": 1119, + "segment_count_current_delta": 120, + "segment_count_peak": 1119, + "segment_count_peak_delta": 120, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T192610914953Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.936406Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192610914953Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192610914953Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..26f583073c2407daca0ea9f141de5b0811374135 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T192610914953Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d2ef14fd9f8306e39e1925cb407f57c78a957f755cedb23781d414a76334f65 +size 1160927 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192611159152Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192611159152Z.json new file mode 100644 index 0000000000000000000000000000000000000000..98200b45c47c318c0878013c711baa169a948e0f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192611159152Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67391940608, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89684086272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67391940608, + "active_bytes_peak": 89684086272, + "allocated_bytes_current": 67391940608, + "allocated_bytes_peak": 89684086272, + "device": 2, + "inactive_split_bytes_current": 1665177600, + "inactive_split_bytes_peak": 4421015040, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 91230306304, + "reserved_bytes_peak": 91230306304, + "segment_count_current": 1119, + "segment_count_peak": 1119, + "timestamp_utc": "2026-08-26T19:26:11.158748Z", + "world_size": 8 + }, + "allocated_bytes_current": 67391940608, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89684086272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67391940608, + "active_bytes_peak": 89684086272, + "allocated_bytes_current": 67391940608, + "allocated_bytes_peak": 89684086272, + "device": 2, + "inactive_split_bytes_current": 1665177600, + "inactive_split_bytes_peak": 4421015040, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 91230306304, + "reserved_bytes_peak": 91230306304, + "segment_count_current": 1119, + "segment_count_peak": 1119, + "timestamp_utc": "2026-08-26T19:26:10.938982Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1665177600, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4421015040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 91230306304, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 91230306304, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1119, + "segment_count_current_delta": 0, + "segment_count_peak": 1119, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T192611159152Z.pickle", + "timestamp_utc": "2026-08-26T19:26:11.178666Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192611159152Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192611159152Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..588bbcc9f2f212dd0cbaf20b077c3ec0bc5b8b73 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T192611159152Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd9904a317cb8ffeda5f7ca953f5b2229cc32233528f1f83b6dfa9a41a5b4bfc +size 1155983 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192612969062Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192612969062Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a3d8c32fa6da2ae23bf6412d16e1481b1023ccd --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192612969062Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408984064, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 89684086272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67408984064, + "active_bytes_peak": 89684086272, + "allocated_bytes_current": 67408984064, + "allocated_bytes_peak": 89684086272, + "device": 2, + "inactive_split_bytes_current": 1908180992, + "inactive_split_bytes_peak": 5339184640, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 92505374720, + "reserved_bytes_peak": 92505374720, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:26:12.968584Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408984064, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 89684086272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67391940608, + "active_bytes_peak": 89684086272, + "allocated_bytes_current": 67391940608, + "allocated_bytes_peak": 89684086272, + "device": 2, + "inactive_split_bytes_current": 1665177600, + "inactive_split_bytes_peak": 4421015040, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 91230306304, + "reserved_bytes_peak": 91230306304, + "segment_count_current": 1119, + "segment_count_peak": 1119, + "timestamp_utc": "2026-08-26T19:26:11.180985Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1908180992, + "inactive_split_bytes_current_delta": 243003392, + "inactive_split_bytes_peak": 5339184640, + "inactive_split_bytes_peak_delta": 918169600, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 92505374720, + "reserved_bytes_current_delta": 1275068416, + "reserved_bytes_peak": 92505374720, + "reserved_bytes_peak_delta": 1275068416, + "segment_count_current": 1122, + "segment_count_current_delta": 3, + "segment_count_peak": 1122, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T192612969062Z.pickle", + "timestamp_utc": "2026-08-26T19:26:12.992995Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192612969062Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192612969062Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..41c98805f69fd0c48f10c3473c3aa9563355b1b8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T192612969062Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:273aa02328b4bb3e39787556646369589d907aacaf522d82ed7cd3ab1d525edd +size 1287301 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192613193673Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192613193673Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca5f7f908c43223d696439d7893be4607e12fef1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192613193673Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408984064, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89684086272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67408984064, + "active_bytes_peak": 89684086272, + "allocated_bytes_current": 67408984064, + "allocated_bytes_peak": 89684086272, + "device": 2, + "inactive_split_bytes_current": 1908180992, + "inactive_split_bytes_peak": 5339184640, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 92505374720, + "reserved_bytes_peak": 92505374720, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:26:13.193258Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408984064, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89684086272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408984064, + "active_bytes_peak": 89684086272, + "allocated_bytes_current": 67408984064, + "allocated_bytes_peak": 89684086272, + "device": 2, + "inactive_split_bytes_current": 1908180992, + "inactive_split_bytes_peak": 5339184640, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 92505374720, + "reserved_bytes_peak": 92505374720, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:26:12.995762Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1908180992, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5339184640, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 92505374720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 92505374720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1122, + "segment_count_current_delta": 0, + "segment_count_peak": 1122, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T192613193673Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.199991Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192613193673Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192613193673Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8d595b52fd9f49d0925b4207116f0c96cde1db8b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T192613193673Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c45119519cab095185b20630e9513de2cf69e43ce19ea07c6075131f0e01790 +size 272179 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T192613392522Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T192613392522Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bca4abcf5c8b535c322775027ffbd0864a01b447 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T192613392522Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408984064, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89684086272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67408984064, + "active_bytes_peak": 89684086272, + "allocated_bytes_current": 67408984064, + "allocated_bytes_peak": 89684086272, + "device": 2, + "inactive_split_bytes_current": 1908180992, + "inactive_split_bytes_peak": 5339184640, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 92505374720, + "reserved_bytes_peak": 92505374720, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:26:13.392135Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408984064, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89684086272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408984064, + "active_bytes_peak": 89684086272, + "allocated_bytes_current": 67408984064, + "allocated_bytes_peak": 89684086272, + "device": 2, + "inactive_split_bytes_current": 1908180992, + "inactive_split_bytes_peak": 5339184640, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 92505374720, + "reserved_bytes_peak": 92505374720, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:26:13.200973Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 1908180992, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5339184640, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 92505374720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 92505374720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1122, + "segment_count_current_delta": 0, + "segment_count_peak": 1122, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T192613392522Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.398789Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T192613392522Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T192613392522Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..87fe1956f47724c317ef1c46790dee3a72270927 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T192613392522Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:817ec6a9c4fb7a9a0227fa0eaad02d41d2a5cd40da5bd9f53c3db4afbdab24d5 +size 272179 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192610591173Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192610591173Z.json new file mode 100644 index 0000000000000000000000000000000000000000..85bb852c86a692c539f2bebdae2123d16fc4fdcf --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192610591173Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 26722478592, + "active_bytes_peak": 68186698240, + "active_bytes_peak_delta": 27462896128, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 68186698240, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 68186698240, + "device": 3, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 77043073024, + "reserved_bytes_peak": 77043073024, + "segment_count_current": 985, + "segment_count_peak": 985, + "timestamp_utc": "2026-08-26T19:26:10.590447Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 26722478592, + "allocated_bytes_peak": 68186698240, + "allocated_bytes_peak_delta": 27462896128, + "before": { + "active_bytes_current": 40686053376, + "active_bytes_peak": 40723802112, + "allocated_bytes_current": 40686053376, + "allocated_bytes_peak": 40723802112, + "device": 3, + "inactive_split_bytes_current": 508303360, + "inactive_split_bytes_peak": 728504320, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:25:44.719344Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 1685542400, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 3887474176, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 77043073024, + "reserved_bytes_current_delta": 33179041792, + "reserved_bytes_peak": 77043073024, + "reserved_bytes_peak_delta": 33179041792, + "segment_count_current": 985, + "segment_count_current_delta": 724, + "segment_count_peak": 985, + "segment_count_peak_delta": 724, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T192610591173Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.617921Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192610591173Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192610591173Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d481dad454cab8ee84fa9a57956f7e5938dd59b2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T192610591173Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5775e8d2330c298f2ca59d58304ac6e3bfe4737032b5cf62fc4388dad63b69f1 +size 1449009 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192610914851Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192610914851Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cc944862a254ac90bcabc3a9d0686939fdf7a012 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192610914851Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 21505789440, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 3, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:10.914453Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 21505789440, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 68186698240, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 68186698240, + "device": 3, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 77043073024, + "reserved_bytes_peak": 77043073024, + "segment_count_current": 985, + "segment_count_peak": 985, + "timestamp_utc": "2026-08-26T19:26:10.620786Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 91163197440, + "reserved_bytes_current_delta": 14120124416, + "reserved_bytes_peak": 91163197440, + "reserved_bytes_peak_delta": 14120124416, + "segment_count_current": 1096, + "segment_count_current_delta": 111, + "segment_count_peak": 1096, + "segment_count_peak_delta": 111, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T192610914851Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.935799Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192610914851Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192610914851Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6aef9ca7cc857c2fc0bb8d2a16a324a7e279e742 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T192610914851Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a5257b79d558b47a33509e329ee85048a27caaae9c51374347d37ac831017c0 +size 1174906 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192611159026Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192611159026Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76c3ee189d8820be1a99860c6b0a408a563e225c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192611159026Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 3, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:11.158637Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 3, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:10.938364Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 91163197440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 91163197440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1096, + "segment_count_current_delta": 0, + "segment_count_peak": 1096, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T192611159026Z.pickle", + "timestamp_utc": "2026-08-26T19:26:11.179131Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192611159026Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192611159026Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e4906ef9c834d506518a86a87430b788d12a7c4a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T192611159026Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43890dda93c02cb2e32158b17501b15b54c2f27689d8da5961061a04d1bcc2ba +size 1166834 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192612968859Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192612968859Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f3164d3a9c56788c48360663abc0092f0b859e36 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192612968859Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 3, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:12.968443Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 3, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:11.181461Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 243003392, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 1251874304, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 1887436800, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 1887436800, + "segment_count_current": 1123, + "segment_count_current_delta": 27, + "segment_count_peak": 1123, + "segment_count_peak_delta": 27, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T192612968859Z.pickle", + "timestamp_utc": "2026-08-26T19:26:12.992187Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192612968859Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192612968859Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aa11bc33d8eb5dd03d09675581b8214a15929d5d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T192612968859Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60b22116d6484b152f9ecde166c495988a35b03896e773a1073f6eb477d6dcea +size 1259072 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192613193569Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192613193569Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1f740e3050e32fb860acfe0b840ff5749ab4591a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192613193569Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 3, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.193171Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 3, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:12.994892Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1123, + "segment_count_current_delta": 0, + "segment_count_peak": 1123, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T192613193569Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.199762Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192613193569Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192613193569Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea5949a3e6c3d5444b210d11b740c0c541821c6c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T192613193569Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc6bfc8853891bc282d3d00f8dbe1d5587a256d6ba427e62ff4e88991c5a889c +size 267606 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T192613392440Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T192613392440Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3cce2a86108b7be5bdbd2670ea7d6a3391e2a8bc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T192613392440Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 3, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.392054Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 3, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.200730Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1123, + "segment_count_current_delta": 0, + "segment_count_peak": 1123, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T192613392440Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.398630Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T192613392440Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T192613392440Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2df3fa0ef82b4f10ceec43a1ea8d6857f630bd05 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T192613392440Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b23c181c8930e5707d1b17472a58b58d9f5e589c79550213d4b201db03db80c5 +size 267606 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192610591160Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192610591160Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e77f8a3aa6d0b77f826eeb8e65aceef070978b55 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192610591160Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 26722478592, + "active_bytes_peak": 68186698240, + "active_bytes_peak_delta": 27462896128, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 68186698240, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 68186698240, + "device": 4, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 77043073024, + "reserved_bytes_peak": 77043073024, + "segment_count_current": 985, + "segment_count_peak": 985, + "timestamp_utc": "2026-08-26T19:26:10.590458Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 26722478592, + "allocated_bytes_peak": 68186698240, + "allocated_bytes_peak_delta": 27462896128, + "before": { + "active_bytes_current": 40686053376, + "active_bytes_peak": 40723802112, + "allocated_bytes_current": 40686053376, + "allocated_bytes_peak": 40723802112, + "device": 4, + "inactive_split_bytes_current": 508303360, + "inactive_split_bytes_peak": 728504320, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:25:44.719260Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 1685542400, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 3887474176, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 77043073024, + "reserved_bytes_current_delta": 33179041792, + "reserved_bytes_peak": 77043073024, + "reserved_bytes_peak_delta": 33179041792, + "segment_count_current": 985, + "segment_count_current_delta": 724, + "segment_count_peak": 985, + "segment_count_peak_delta": 724, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T192610591160Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.617663Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192610591160Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192610591160Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b5e6a37748d6b8ea4d5129d2ba5f862b17d68c25 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192610591160Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fde8f1ac287368d9316f1b1d407dae769831792b792a72dfed806e9289cc2664 +size 1449009 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192610914792Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192610914792Z.json new file mode 100644 index 0000000000000000000000000000000000000000..57bc6bfa6c7f102d47c970f799ced1688da6627a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192610914792Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 21505789440, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 4, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:10.914397Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 21505789440, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 68186698240, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 68186698240, + "device": 4, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 77043073024, + "reserved_bytes_peak": 77043073024, + "segment_count_current": 985, + "segment_count_peak": 985, + "timestamp_utc": "2026-08-26T19:26:10.620525Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 91163197440, + "reserved_bytes_current_delta": 14120124416, + "reserved_bytes_peak": 91163197440, + "reserved_bytes_peak_delta": 14120124416, + "segment_count_current": 1096, + "segment_count_current_delta": 111, + "segment_count_peak": 1096, + "segment_count_peak_delta": 111, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T192610914792Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.935398Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192610914792Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192610914792Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c1ac49e6195dd64e20be829feaf425d68b8ff66 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192610914792Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ef7452118a4a6458bfa0cd4085ac29bb7d86b036e29fdb1e60ac109a9cde8fd +size 1174906 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192611159003Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192611159003Z.json new file mode 100644 index 0000000000000000000000000000000000000000..85ccfbd28b89d99a5378eefecae4b9ff4713d3ec --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192611159003Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 4, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:11.158624Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 4, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:10.937960Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 91163197440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 91163197440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1096, + "segment_count_current_delta": 0, + "segment_count_peak": 1096, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T192611159003Z.pickle", + "timestamp_utc": "2026-08-26T19:26:11.179064Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192611159003Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192611159003Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..583da8528b8920ae5fc2292888439847e02f54b1 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192611159003Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34b3e4355812980afce58dc14d336fb3d80687c36ffc3d2a67531f221105e6ed +size 1166830 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192612968853Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192612968853Z.json new file mode 100644 index 0000000000000000000000000000000000000000..da7c218c60bedfb4ec2c527ebfa9463e86d83a7d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192612968853Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 4, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:12.968419Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 4, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:11.181373Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 243003392, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 1251874304, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 1887436800, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 1887436800, + "segment_count_current": 1123, + "segment_count_current_delta": 27, + "segment_count_peak": 1123, + "segment_count_peak_delta": 27, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T192612968853Z.pickle", + "timestamp_utc": "2026-08-26T19:26:12.992383Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192612968853Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192612968853Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..612195e3f5e14e0733eb058f58a5bbb25c4b36f7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192612968853Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b70ca8292034168d086b412a96eb793eb4ffe0724b6166b5ff4ab7ed1002e39 +size 1259070 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192613193550Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192613193550Z.json new file mode 100644 index 0000000000000000000000000000000000000000..498b283d9fef34a9cd6361ce99877e94369aab2c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192613193550Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 4, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.193143Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 4, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:12.995069Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1123, + "segment_count_current_delta": 0, + "segment_count_peak": 1123, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T192613193550Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.199627Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192613193550Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192613193550Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..08778206212b97c6af6e80b34c8c983969e2cc0b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192613193550Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e73b8a1ff1c448bfc3d70d2ead5313211a9a74e5071d8aece7c5d7f4c7919736 +size 267604 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192613392385Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192613392385Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b22b79280e0f7cf4fdbc4811b4a60f92e1450d5d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192613392385Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 4, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.392003Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 4, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.200582Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1123, + "segment_count_current_delta": 0, + "segment_count_peak": 1123, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T192613392385Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.398467Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192613392385Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192613392385Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8cf1e0877ae25babe69a7ed4e90fe23d2d21cf24 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192613392385Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9011475d7a73fda001d77f64a35412e40f761dc66aa643f56ff763ab04007c2a +size 267604 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192610591171Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192610591171Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4f56aa68aa634c2131f4af48ba11edf4324c16db --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192610591171Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 26722478592, + "active_bytes_peak": 68186698240, + "active_bytes_peak_delta": 27462896128, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 68186698240, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 68186698240, + "device": 5, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 77043073024, + "reserved_bytes_peak": 77043073024, + "segment_count_current": 985, + "segment_count_peak": 985, + "timestamp_utc": "2026-08-26T19:26:10.590470Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 26722478592, + "allocated_bytes_peak": 68186698240, + "allocated_bytes_peak_delta": 27462896128, + "before": { + "active_bytes_current": 40686053376, + "active_bytes_peak": 40723802112, + "allocated_bytes_current": 40686053376, + "allocated_bytes_peak": 40723802112, + "device": 5, + "inactive_split_bytes_current": 508303360, + "inactive_split_bytes_peak": 728504320, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:25:44.719193Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 1685542400, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 3887474176, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 77043073024, + "reserved_bytes_current_delta": 33179041792, + "reserved_bytes_peak": 77043073024, + "reserved_bytes_peak_delta": 33179041792, + "segment_count_current": 985, + "segment_count_current_delta": 724, + "segment_count_peak": 985, + "segment_count_peak_delta": 724, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T192610591171Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.617025Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192610591171Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192610591171Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5726cad09bc83314665cac7f458a2f56f6c0f0e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192610591171Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:052c71fe69ecc7e34d3b3181a1f89e19659a4342452b5dcc57f78686327490bb +size 1449009 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192610914849Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192610914849Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6ad67659f8de667bd7c29302fa503fc2d70af12 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192610914849Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 21505789440, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 5, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:10.914444Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 21505789440, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 68186698240, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 68186698240, + "device": 5, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 77043073024, + "reserved_bytes_peak": 77043073024, + "segment_count_current": 985, + "segment_count_peak": 985, + "timestamp_utc": "2026-08-26T19:26:10.619953Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 91163197440, + "reserved_bytes_current_delta": 14120124416, + "reserved_bytes_peak": 91163197440, + "reserved_bytes_peak_delta": 14120124416, + "segment_count_current": 1096, + "segment_count_current_delta": 111, + "segment_count_peak": 1096, + "segment_count_peak_delta": 111, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T192610914849Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.935279Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192610914849Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192610914849Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..78381a3e9dc3227e92821e6d4400518f5f9a2809 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192610914849Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e1074f888d1bf9deaaebbf22730a3fdd59ef174e567c3da24342415ae7ddf0e +size 1174906 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192611159018Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192611159018Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b02dffe9b2432c49065081dd56ea7c463b969c59 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192611159018Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 5, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:11.158631Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 5, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:10.937824Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 91163197440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 91163197440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1096, + "segment_count_current_delta": 0, + "segment_count_peak": 1096, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T192611159018Z.pickle", + "timestamp_utc": "2026-08-26T19:26:11.178650Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192611159018Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192611159018Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8d6999f4a36a38d7dcd1e14dcf754cf61e4c5d02 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192611159018Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27e6369d054851e1457ec7dfa3164f76e1ac105c8fe2210b0610fcf9884d8a70 +size 1166832 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192612968836Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192612968836Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06bc259b38efafc81471a1e6dd56bca2e5c9325f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192612968836Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 5, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:12.968402Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 5, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:11.180932Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 243003392, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 1251874304, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 1887436800, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 1887436800, + "segment_count_current": 1123, + "segment_count_current_delta": 27, + "segment_count_peak": 1123, + "segment_count_peak_delta": 27, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T192612968836Z.pickle", + "timestamp_utc": "2026-08-26T19:26:12.992190Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192612968836Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192612968836Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..37ad255ca4d724c4cd17856e9376801ed94a9f27 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192612968836Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf41c722bfb61b3c2b990e5a8be50f86ad43439a85b347e842d8cdd03e9f1c10 +size 1259070 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192613193545Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192613193545Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ff0be4e1c889a8ea6ae4dc97756a71a03c6ee8d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192613193545Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 5, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.193154Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 5, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:12.994927Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1123, + "segment_count_current_delta": 0, + "segment_count_peak": 1123, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T192613193545Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.199722Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192613193545Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192613193545Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf009cd51ba64aa89906c7cf42aff505396df4da --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192613193545Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c559723f2e3fabaff2576bf5431fb71e60a9c642b996f13d756172a690fafee3 +size 267604 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192613392428Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192613392428Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94b8cade2777097edb3262782c5d1d061eacdf6c --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192613392428Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 5, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.392037Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 5, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.200677Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1123, + "segment_count_current_delta": 0, + "segment_count_peak": 1123, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T192613392428Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.398682Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192613392428Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192613392428Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..65b0fba300a643a93cef80e3b323b18f165662ef --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192613392428Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:328482d45c730b301ea81a15fe431fa4afebeb79d3889dd9c3f801b926c426c4 +size 267604 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192610591168Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192610591168Z.json new file mode 100644 index 0000000000000000000000000000000000000000..88f92c7418b262442dfb90130a1760026a38aec0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192610591168Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 26722478592, + "active_bytes_peak": 68186698240, + "active_bytes_peak_delta": 27462896128, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 68186698240, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 68186698240, + "device": 6, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 77043073024, + "reserved_bytes_peak": 77043073024, + "segment_count_current": 985, + "segment_count_peak": 985, + "timestamp_utc": "2026-08-26T19:26:10.590477Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 26722478592, + "allocated_bytes_peak": 68186698240, + "allocated_bytes_peak_delta": 27462896128, + "before": { + "active_bytes_current": 40686053376, + "active_bytes_peak": 40723802112, + "allocated_bytes_current": 40686053376, + "allocated_bytes_peak": 40723802112, + "device": 6, + "inactive_split_bytes_current": 508303360, + "inactive_split_bytes_peak": 728504320, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:25:44.719260Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 1685542400, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 3887474176, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 77043073024, + "reserved_bytes_current_delta": 33179041792, + "reserved_bytes_peak": 77043073024, + "reserved_bytes_peak_delta": 33179041792, + "segment_count_current": 985, + "segment_count_current_delta": 724, + "segment_count_peak": 985, + "segment_count_peak_delta": 724, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T192610591168Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.617417Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192610591168Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192610591168Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..21df2d29e59bc57071786814e869108ae0fc3319 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192610591168Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f04167d8e10aca771ae96426cee3eb267fbda16c7dc72ae6a918ae185a0b41fe +size 1449009 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192610914797Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192610914797Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8a9205c6d3fa45aaeb84641d3bcdbee7684caf35 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192610914797Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 21505789440, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 6, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:10.914404Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 21505789440, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 68186698240, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 68186698240, + "device": 6, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 77043073024, + "reserved_bytes_peak": 77043073024, + "segment_count_current": 985, + "segment_count_peak": 985, + "timestamp_utc": "2026-08-26T19:26:10.620318Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 91163197440, + "reserved_bytes_current_delta": 14120124416, + "reserved_bytes_peak": 91163197440, + "reserved_bytes_peak_delta": 14120124416, + "segment_count_current": 1096, + "segment_count_current_delta": 111, + "segment_count_peak": 1096, + "segment_count_peak_delta": 111, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T192610914797Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.935444Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192610914797Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192610914797Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f4d0c43a130e17fbfb3d2c9211eee1656a3268d3 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192610914797Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:795aa6761e560a3082feda1a23f1d04afaf92eb1965e0ea63f374ba842d2fc40 +size 1174904 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192611159035Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192611159035Z.json new file mode 100644 index 0000000000000000000000000000000000000000..473140ce43d54cbeffb5e89b02159d9c9898ba6a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192611159035Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 6, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:11.158642Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 6, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:10.937945Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 91163197440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 91163197440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1096, + "segment_count_current_delta": 0, + "segment_count_peak": 1096, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T192611159035Z.pickle", + "timestamp_utc": "2026-08-26T19:26:11.179491Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192611159035Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192611159035Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..653f11fa5546aec58995c89912ef39cb1351b610 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192611159035Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce1b71991cf38ae72bae546cdd359ea969c68e4bdb1fffe0fd7ebc24b4e39c8c +size 1166834 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192612968838Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192612968838Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bbd440cc05371865b3ba212dec698b75260c8aad --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192612968838Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 6, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:12.968413Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 6, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:11.181750Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 243003392, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 1251874304, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 1887436800, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 1887436800, + "segment_count_current": 1123, + "segment_count_current_delta": 27, + "segment_count_peak": 1123, + "segment_count_peak_delta": 27, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T192612968838Z.pickle", + "timestamp_utc": "2026-08-26T19:26:12.992783Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192612968838Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192612968838Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a753ce20bdb512e3b502763b82eeac31e9670a0 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192612968838Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eea5b054e29ec4fb714988cbd689284bb662826f97c9421ad4c1ded35ce8c3ca +size 1259072 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192613193569Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192613193569Z.json new file mode 100644 index 0000000000000000000000000000000000000000..83bb859865414566361c1afa7ea45f3b87dca23e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192613193569Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 6, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.193167Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 6, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:12.995533Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1123, + "segment_count_current_delta": 0, + "segment_count_peak": 1123, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T192613193569Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.199799Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192613193569Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192613193569Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6470531416670b9c18a1d235f17630855affb12a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192613193569Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72ec9a24ba24def137d13b2c385a8ce775389bdfc867f8ec103262342255b278 +size 267606 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192613392427Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192613392427Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0a04672e8f3a104885af6b9099ae22e45910faad --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192613392427Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 6, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.392038Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 6, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.200751Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1123, + "segment_count_current_delta": 0, + "segment_count_peak": 1123, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T192613392427Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.398549Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192613392427Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192613392427Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c56840f3c05dbdc2d3f500205b8bfb2667832ad9 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192613392427Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:823c5dcfb0b6b1fd74fc4806d6826e497b0f12c914237712e36e6b4665a0b502 +size 267606 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192610591169Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192610591169Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b7e451f1b8de763825c4b988094b244964ec0c5b --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192610591169Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 26722478592, + "active_bytes_peak": 68186698240, + "active_bytes_peak_delta": 27462896128, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 68186698240, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 68186698240, + "device": 7, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 77043073024, + "reserved_bytes_peak": 77043073024, + "segment_count_current": 985, + "segment_count_peak": 985, + "timestamp_utc": "2026-08-26T19:26:10.590469Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 26722478592, + "allocated_bytes_peak": 68186698240, + "allocated_bytes_peak_delta": 27462896128, + "before": { + "active_bytes_current": 40686053376, + "active_bytes_peak": 40723802112, + "allocated_bytes_current": 40686053376, + "allocated_bytes_peak": 40723802112, + "device": 7, + "inactive_split_bytes_current": 508303360, + "inactive_split_bytes_peak": 728504320, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 43864031232, + "reserved_bytes_peak": 43864031232, + "segment_count_current": 261, + "segment_count_peak": 261, + "timestamp_utc": "2026-08-26T19:25:44.719142Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 1685542400, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 3887474176, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 77043073024, + "reserved_bytes_current_delta": 33179041792, + "reserved_bytes_peak": 77043073024, + "reserved_bytes_peak_delta": 33179041792, + "segment_count_current": 985, + "segment_count_current_delta": 724, + "segment_count_peak": 985, + "segment_count_peak_delta": 724, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T192610591169Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.617177Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192610591169Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192610591169Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ccaf25071ac0764fdba3cbacf5d9d15957283aca --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192610591169Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:251f5fd1dc52c34b3f774a86b30d408fcf0ce4d200c14eccfd668da5f01be7ec +size 1449009 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192610914848Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192610914848Z.json new file mode 100644 index 0000000000000000000000000000000000000000..389c4ede8b24a5373b6be2b6782ad580020ed423 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192610914848Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 21505789440, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 7, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:10.914451Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 21505789440, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 68186698240, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 68186698240, + "device": 7, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 77043073024, + "reserved_bytes_peak": 77043073024, + "segment_count_current": 985, + "segment_count_peak": 985, + "timestamp_utc": "2026-08-26T19:26:10.620100Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 91163197440, + "reserved_bytes_current_delta": 14120124416, + "reserved_bytes_peak": 91163197440, + "reserved_bytes_peak_delta": 14120124416, + "segment_count_current": 1096, + "segment_count_current_delta": 111, + "segment_count_peak": 1096, + "segment_count_peak_delta": 111, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T192610914848Z.pickle", + "timestamp_utc": "2026-08-26T19:26:10.935959Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192610914848Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192610914848Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cb6e928106883d70c174d8365b41c217e394c20a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192610914848Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45c96a2c15189dd8e3bb7eed795b7aad007723cbab21e9de6b34b8c898744d68 +size 1174902 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192611159068Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192611159068Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c7fa74eca746f9f98205f9344182cd3399b15da5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192611159068Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67408531968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 7, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:11.158684Z", + "world_size": 8 + }, + "allocated_bytes_current": 67408531968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 7, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:10.938504Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4615978496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 91163197440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 91163197440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1096, + "segment_count_current_delta": 0, + "segment_count_peak": 1096, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T192611159068Z.pickle", + "timestamp_utc": "2026-08-26T19:26:11.178797Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192611159068Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192611159068Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2868359594703c0d7df4f3f95379bb35809f9f6a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192611159068Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6325df2188bfd672b4d90ab538be01843bf96bc7c77dece69716c173f228522d +size 1166834 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192612968870Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192612968870Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8996fdcaec33c72fff4f7bc72e9fa48503cc0a6a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192612968870Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 7, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:12.968445Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67408531968, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67408531968, + "allocated_bytes_peak": 89692487680, + "device": 7, + "inactive_split_bytes_current": 2193845760, + "inactive_split_bytes_peak": 4615978496, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 91163197440, + "reserved_bytes_peak": 91163197440, + "segment_count_current": 1096, + "segment_count_peak": 1096, + "timestamp_utc": "2026-08-26T19:26:11.181297Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 243003392, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 1251874304, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 1887436800, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 1887436800, + "segment_count_current": 1123, + "segment_count_current_delta": 27, + "segment_count_peak": 1123, + "segment_count_peak_delta": 27, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T192612968870Z.pickle", + "timestamp_utc": "2026-08-26T19:26:12.992358Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192612968870Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192612968870Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b15bfb1ffc551f23eaa30037d24a348c8b3e3a4d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192612968870Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a89cfa67dc32cd2116eb2dc9263ec96d2b0fbb2579f0fad9f9f32b92c986c3c3 +size 1259072 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192613193590Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192613193590Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5d2e8bb6b2489c49dc48822c06d3132fe2e9bfc2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192613193590Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 7, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.193189Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 7, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:12.995033Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1123, + "segment_count_current_delta": 0, + "segment_count_peak": 1123, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T192613193590Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.199807Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192613193590Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192613193590Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2c54aac050a63cc516e73b556e659794f406f6c8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192613193590Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb1e9ee1519f98b7bc8362c27912d0df1db9e5b9ee2730a410990a7bffdee980 +size 267606 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192613392429Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192613392429Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5416b63d60e2e6040954ee52cfd4aed7f0bc30dc --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192613392429Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67425575424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 89692487680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 7, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.392049Z", + "world_size": 8 + }, + "allocated_bytes_current": 67425575424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 89692487680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67425575424, + "active_bytes_peak": 89692487680, + "allocated_bytes_current": 67425575424, + "allocated_bytes_peak": 89692487680, + "device": 7, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_peak": 5867852800, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 93050634240, + "reserved_bytes_peak": 93050634240, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:26:13.200800Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2436849152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5867852800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93050634240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93050634240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1123, + "segment_count_current_delta": 0, + "segment_count_peak": 1123, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T192613392429Z.pickle", + "timestamp_utc": "2026-08-26T19:26:13.398527Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192613392429Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192613392429Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a8e95eadb51105e5fd677a847d53cd01da58d580 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192613392429Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67771dbef187a7195beaea8c1b62d422e5f40477c9b2913690a40b06099eb569 +size 267606 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/run.env b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..6056e7d82b1bae6fbad0d485aba77d3309bfda65 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN2_57B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-better-memory-dump-support +commit=c3a8a8a7 +num_layers=10 +hidden=3584 +ffn_hidden=2560 +moe_ffn_hidden=2560 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=27525120 +heads=28 +kv_groups=4 +kv_channels= +qk_layernorm=0 +shared_ffn=20480 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=10 +rotary_base=1000000 +rotary_scaling= +qkv_bias=1 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.00 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=512 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/train.log b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8a9467eb3b238f25c9c56e34b95f857172d34785 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/train.log @@ -0,0 +1,1194 @@ +============================================================== + QWEN2_57B / ultraep (slots/rank=8 policy=matched) + E=64 K=8 H'=2560 m=8 + L=10 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.00 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ c3a8a8a7 +============================================================== +W0826 19:25:17.133000 666872 torch/distributed/run.py:766] +W0826 19:25:17.133000 666872 torch/distributed/run.py:766] ***************************************** +W0826 19:25:17.133000 666872 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:25:17.133000 666872 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 10 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 2560 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 3584 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2560 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 20480 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 28 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 10 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 71 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 75 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 0[Gloo] Rank 0 is connected to is connected to 000 is connected to peer ranks. peer ranks. 0Expected number of connected peer ranks is : Expected number of connected peer ranks is : peer ranks. 0[Gloo] Rank 0Expected number of connected peer ranks is : +0 +0 is connected to +0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W826 19:25:35.090592629 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 19:25:35.090612771 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:25:35.093986553 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:25:35.094066854 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:25:35.094117982 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:25:35.094134240 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:25:35.095433162 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:25:35.245812775 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.916 seconds +time to initialize megatron (seconds): 7.272 +[after megatron is initialized] datetime: 2026-08-26 19:25:41 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 7262624256 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22 +Params for bucket 1 (180289536 elements, 180289536 padded size): + module.output_layer.weight +Params for bucket 2 (73403904 elements, 73403904 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (146800640 elements, 146800640 padded size): + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight +Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.bias + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (146800640 elements, 146800640 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight +Params for bucket 7 (146800640 elements, 146800640 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight +Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight +Params for bucket 9 (146800640 elements, 146800640 padded size): + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 10 (103001600 elements, 103001600 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.bias + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (146800640 elements, 146800640 padded size): + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (103001600 elements, 103001600 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_qkv.bias + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.pre_mlp_layernorm.weight +Params for bucket 13 (146800640 elements, 146800640 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight +Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.self_attention.linear_qkv.bias + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight +Params for bucket 15 (146800640 elements, 146800640 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight +Params for bucket 16 (103001600 elements, 103001600 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.self_attention.linear_qkv.bias + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight +Params for bucket 17 (146800640 elements, 146800640 padded size): + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.bias +Params for bucket 19 (146800640 elements, 146800640 padded size): + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 20 (103001600 elements, 103001600 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.bias +Params for bucket 21 (146800640 elements, 146800640 padded size): + module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight +Params for bucket 22 (209890816 elements, 209890816 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 +Params for bucket 1 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 +Params for bucket 2 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 +Params for bucket 3 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 5 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 +Params for bucket 6 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 +Params for bucket 7 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 +Params for bucket 8 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 9 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 10 (45875200 elements, 45875200 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 11 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 12 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 14 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 +Params for bucket 15 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 16 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 +Params for bucket 17 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 18 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 19 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 20 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 21 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 22 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 +Params for bucket 23 (45875200 elements, 45875200 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 24 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 25 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 26 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 27 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 +Params for bucket 28 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 +Params for bucket 30 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 31 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 +Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 33 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 34 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (45875200 elements, 45875200 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 +Params for bucket 37 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 38 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 39 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 40 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 +Params for bucket 41 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 42 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 +Params for bucket 43 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 44 (18350080 elements, 18350080 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:25:44 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:25:44 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3505.16, 3511.78) + train/valid/test-data-iterators-setup ..........: (73.17, 87.34) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:25:44 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 19:26:10] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 25870.2 | throughput per GPU (TFLOP/s/GPU): 4.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097934E+01 | loss scale: 1.0 | grad norm: 29.876 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 32768 | min safe capacity: 1.0000 | +Number of parameters in transformer block in billions: 20.11 +Number of parameters in embedding layers in billions: 0.36 +Total number of parameters in billions: 20.47 +Number of parameters in most loaded shard in billions: 20.4724 +Theoretical memory footprints: weight and optimizer=146430.14 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 64284.79443359375 | max allocated: 65032.31591796875 | reserved: 73540.0 | max reserved: 73540.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 19:26:10] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 324.8 | throughput per GPU (TFLOP/s/GPU): 380.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097897E+01 | loss scale: 1.0 | grad norm: 30.623 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 32768 | min safe capacity: 1.0000 | + [2026-08-26 19:26:11] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 244.2 | throughput per GPU (TFLOP/s/GPU): 505.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.538938E+00 | loss scale: 1.0 | grad norm: 32.389 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 32768 | min safe capacity: 1.0000 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:26:12] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 1809.7 | throughput per GPU (TFLOP/s/GPU): 68.2 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.108247E+01 | loss scale: 1.0 | grad norm: 106.879 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 32768 | min safe capacity: 1.0000 | + [2026-08-26 19:26:13] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 224.7 | throughput per GPU (TFLOP/s/GPU): 549.5 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.000609E+01 | loss scale: 1.0 | grad norm: 26.680 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 32768 | min safe capacity: 1.0000 | + [2026-08-26 19:26:13] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 198.8 | throughput per GPU (TFLOP/s/GPU): 621.1 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.050235E+01 | loss scale: 1.0 | grad norm: 8.734 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 32768 | min safe capacity: 1.0000 | +[after training is done] datetime: 2026-08-26 19:26:13 +[rank0]:[W826 19:26:14.569675178 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 19:26:15.715904720 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 19:26:15.718335267 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 19:26:15.719695743 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:26:15.820000 666872 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:26:15.821000 666872 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 666960 closing signal SIGTERM +W0826 19:26:15.821000 666872 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 666961 closing signal SIGTERM +W0826 19:26:15.825000 666872 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 666962 closing signal SIGTERM +W0826 19:26:15.827000 666872 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 666963 closing signal SIGTERM +W0826 19:26:15.830000 666872 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 666964 closing signal SIGTERM +W0826 19:26:15.832000 666872 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 666965 closing signal SIGTERM +W0826 19:26:15.833000 666872 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 666966 closing signal SIGTERM +W0826 19:26:15.835000 666872 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 666967 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 666872 got signal: 15 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..d10f854919995525f3c7642dc954701f395b43ed --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,200 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 2 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 47 +3, 629 +4, 45 +5, 41 +6, 47 +7, 35 +0, 925 +1, 923 +2, 903 +3, 917 +4, 917 +5, 913 +6, 907 +7, 923 +0, 1803 +1, 1803 +2, 1803 +3, 1803 +4, 1803 +5, 1803 +6, 1803 +7, 1803 +0, 2265 +1, 2247 +2, 2267 +3, 2271 +4, 2263 +5, 2249 +6, 2251 +7, 2249 +0, 46703 +1, 46137 +2, 46137 +3, 46837 +4, 46837 +5, 46837 +6, 46837 +7, 46837 +0, 47125 +1, 46555 +2, 46555 +3, 47255 +4, 47255 +5, 47255 +6, 47255 +7, 47255 +0, 48413 +1, 47843 +2, 47843 +3, 48543 +4, 48543 +5, 48543 +6, 48543 +7, 48543 +0, 48413 +1, 47843 +2, 47843 +3, 48543 +4, 48543 +5, 48543 +6, 48543 +7, 48543 +0, 48665 +1, 48095 +2, 48095 +3, 48795 +4, 48795 +5, 48795 +6, 48795 +7, 48795 +0, 50757 +1, 50187 +2, 50187 +3, 50887 +4, 50887 +5, 50887 +6, 50887 +7, 50887 +0, 73669 +1, 73645 +2, 73645 +3, 73653 +4, 73653 +5, 73653 +6, 73653 +7, 73653 +0, 74315 +1, 74291 +2, 74291 +3, 74299 +4, 74299 +5, 74299 +6, 74299 +7, 74299 +0, 74315 +1, 74291 +2, 74291 +3, 74299 +4, 74299 +5, 74299 +6, 74299 +7, 74299 +0, 74315 +1, 74293 +2, 74293 +3, 74301 +4, 74301 +5, 74301 +6, 74301 +7, 74301 +0, 74669 +1, 74647 +2, 74675 +3, 74667 +4, 74673 +5, 74699 +6, 74669 +7, 74661 +0, 82117 +1, 81831 +2, 81839 +3, 82107 +4, 82067 +5, 82121 +6, 82123 +7, 82081 +0, 96955 +1, 96969 +2, 96959 +3, 96905 +4, 96905 +5, 96905 +6, 96905 +7, 96905 +0, 98849 +1, 98609 +2, 98585 +3, 98961 +4, 98961 +5, 98961 +6, 98961 +7, 98961 +0, 97955 +1, 97715 +2, 97691 +3, 98067 +4, 98067 +5, 98067 +6, 98067 +7, 98067 +0, 217 +1, 94725 +2, 1094 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192716003733Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192716003733Z.json new file mode 100644 index 0000000000000000000000000000000000000000..87d201fa0e8c9ac1bf183da21c2cc1a0584b1b2f --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192716003733Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67395317248, + "active_bytes_current_delta": 26710310400, + "active_bytes_peak": 68869932544, + "active_bytes_peak_delta": 28147176960, + "after": { + "active_bytes_current": 67395317248, + "active_bytes_peak": 68869932544, + "allocated_bytes_current": 67395317248, + "allocated_bytes_peak": 68869932544, + "device": 0, + "inactive_split_bytes_current": 2305626624, + "inactive_split_bytes_peak": 4573583872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 76875300864, + "reserved_bytes_peak": 76875300864, + "segment_count_current": 957, + "segment_count_peak": 957, + "timestamp_utc": "2026-08-26T19:27:16.003050Z", + "world_size": 8 + }, + "allocated_bytes_current": 67395317248, + "allocated_bytes_current_delta": 26710310400, + "allocated_bytes_peak": 68869932544, + "allocated_bytes_peak_delta": 28147176960, + "before": { + "active_bytes_current": 40685006848, + "active_bytes_peak": 40722755584, + "allocated_bytes_current": 40685006848, + "allocated_bytes_peak": 40722755584, + "device": 0, + "inactive_split_bytes_current": 373035008, + "inactive_split_bytes_peak": 593235968, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 43723522048, + "reserved_bytes_peak": 43723522048, + "segment_count_current": 262, + "segment_count_peak": 262, + "timestamp_utc": "2026-08-26T19:26:50.112606Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2305626624, + "inactive_split_bytes_current_delta": 1932591616, + "inactive_split_bytes_peak": 4573583872, + "inactive_split_bytes_peak_delta": 3980347904, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 76875300864, + "reserved_bytes_current_delta": 33151778816, + "reserved_bytes_peak": 76875300864, + "reserved_bytes_peak_delta": 33151778816, + "segment_count_current": 957, + "segment_count_current_delta": 695, + "segment_count_peak": 957, + "segment_count_peak_delta": 695, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T192716003733Z.pickle", + "timestamp_utc": "2026-08-26T19:27:16.030644Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192716003733Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192716003733Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eba822d6503c19d3104829bc6b4f01e9cde43fc5 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T192716003733Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78169d0a068a8f419e76f164edc29443f1efb2fb9064ca4457ab702abd1188b7 +size 1454214 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192716331721Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192716331721Z.json new file mode 100644 index 0000000000000000000000000000000000000000..56f11e372967901a757cda175640f89674178d4a --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192716331721Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67395317248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 90348203008, + "active_bytes_peak_delta": 21478270464, + "after": { + "active_bytes_current": 67395317248, + "active_bytes_peak": 90348203008, + "allocated_bytes_current": 67395317248, + "allocated_bytes_peak": 90348203008, + "device": 0, + "inactive_split_bytes_current": 2305626624, + "inactive_split_bytes_peak": 4573583872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 91924463616, + "reserved_bytes_peak": 91924463616, + "segment_count_current": 1073, + "segment_count_peak": 1073, + "timestamp_utc": "2026-08-26T19:27:16.331286Z", + "world_size": 8 + }, + "allocated_bytes_current": 67395317248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 90348203008, + "allocated_bytes_peak_delta": 21478270464, + "before": { + "active_bytes_current": 67395317248, + "active_bytes_peak": 68869932544, + "allocated_bytes_current": 67395317248, + "allocated_bytes_peak": 68869932544, + "device": 0, + "inactive_split_bytes_current": 2305626624, + "inactive_split_bytes_peak": 4573583872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 76875300864, + "reserved_bytes_peak": 76875300864, + "segment_count_current": 957, + "segment_count_peak": 957, + "timestamp_utc": "2026-08-26T19:27:16.033575Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2305626624, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4573583872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 91924463616, + "reserved_bytes_current_delta": 15049162752, + "reserved_bytes_peak": 91924463616, + "reserved_bytes_peak_delta": 15049162752, + "segment_count_current": 1073, + "segment_count_current_delta": 116, + "segment_count_peak": 1073, + "segment_count_peak_delta": 116, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T192716331721Z.pickle", + "timestamp_utc": "2026-08-26T19:27:16.352757Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192716331721Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192716331721Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..749999d36f7d4ce0d4e25d8f19b6195d112f0a29 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T192716331721Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc3ddce17ef4f0374f79ff3eb721a5ed7fc32cae5714cc25607e880b82dc7f8a +size 1158352 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192716576839Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192716576839Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5f89f0051f212492aae1c8725c2934a90a4f371 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192716576839Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67395317248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 90348203008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67395317248, + "active_bytes_peak": 90348203008, + "allocated_bytes_current": 67395317248, + "allocated_bytes_peak": 90348203008, + "device": 0, + "inactive_split_bytes_current": 2307723776, + "inactive_split_bytes_peak": 4573583872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 91924463616, + "reserved_bytes_peak": 91924463616, + "segment_count_current": 1073, + "segment_count_peak": 1073, + "timestamp_utc": "2026-08-26T19:27:16.576434Z", + "world_size": 8 + }, + "allocated_bytes_current": 67395317248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 90348203008, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67395317248, + "active_bytes_peak": 90348203008, + "allocated_bytes_current": 67395317248, + "allocated_bytes_peak": 90348203008, + "device": 0, + "inactive_split_bytes_current": 2305626624, + "inactive_split_bytes_peak": 4573583872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 91924463616, + "reserved_bytes_peak": 91924463616, + "segment_count_current": 1073, + "segment_count_peak": 1073, + "timestamp_utc": "2026-08-26T19:27:16.355372Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2307723776, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 4573583872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 91924463616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 91924463616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1073, + "segment_count_current_delta": 0, + "segment_count_peak": 1073, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T192716576839Z.pickle", + "timestamp_utc": "2026-08-26T19:27:16.595704Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192716576839Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192716576839Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..64109703430e33c667160a7aa8d1e14cc9996d31 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T192716576839Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c2831d3cc4e3acd43fa0e79d5452177eb17e7ec223ee2dbc8836594a487ab38 +size 1149300 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192718236649Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192718236649Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b19163821acbd5a08d627e910d245d03f9dfd3cf --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192718236649Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67412360704, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 90348203008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67412360704, + "active_bytes_peak": 90348203008, + "allocated_bytes_current": 67412360704, + "allocated_bytes_peak": 90348203008, + "device": 0, + "inactive_split_bytes_current": 2577990144, + "inactive_split_bytes_peak": 5587646464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93891592192, + "reserved_bytes_peak": 93891592192, + "segment_count_current": 1095, + "segment_count_peak": 1095, + "timestamp_utc": "2026-08-26T19:27:18.236151Z", + "world_size": 8 + }, + "allocated_bytes_current": 67412360704, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 90348203008, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67395317248, + "active_bytes_peak": 90348203008, + "allocated_bytes_current": 67395317248, + "allocated_bytes_peak": 90348203008, + "device": 0, + "inactive_split_bytes_current": 2307723776, + "inactive_split_bytes_peak": 4573583872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 91924463616, + "reserved_bytes_peak": 91924463616, + "segment_count_current": 1073, + "segment_count_peak": 1073, + "timestamp_utc": "2026-08-26T19:27:16.598038Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2577990144, + "inactive_split_bytes_current_delta": 270266368, + "inactive_split_bytes_peak": 5587646464, + "inactive_split_bytes_peak_delta": 1014062592, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93891592192, + "reserved_bytes_current_delta": 1967128576, + "reserved_bytes_peak": 93891592192, + "reserved_bytes_peak_delta": 1967128576, + "segment_count_current": 1095, + "segment_count_current_delta": 22, + "segment_count_peak": 1095, + "segment_count_peak_delta": 22, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T192718236649Z.pickle", + "timestamp_utc": "2026-08-26T19:27:18.260479Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192718236649Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192718236649Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..02617e81c831be66a58f381bb40b4f71908227a4 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T192718236649Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4034fa2cb85f9ffbf38ccca91e202a2f8fef53fc8cdbccb24dac4aa8c402ae1 +size 1263597 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192718464385Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192718464385Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2e69b334c475bf73d07ecaf96d38bb1c3c09a26d --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192718464385Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67412360704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 90348203008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67412360704, + "active_bytes_peak": 90348203008, + "allocated_bytes_current": 67412360704, + "allocated_bytes_peak": 90348203008, + "device": 0, + "inactive_split_bytes_current": 2577990144, + "inactive_split_bytes_peak": 5587646464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93891592192, + "reserved_bytes_peak": 93891592192, + "segment_count_current": 1095, + "segment_count_peak": 1095, + "timestamp_utc": "2026-08-26T19:27:18.463980Z", + "world_size": 8 + }, + "allocated_bytes_current": 67412360704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 90348203008, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67412360704, + "active_bytes_peak": 90348203008, + "allocated_bytes_current": 67412360704, + "allocated_bytes_peak": 90348203008, + "device": 0, + "inactive_split_bytes_current": 2577990144, + "inactive_split_bytes_peak": 5587646464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 93891592192, + "reserved_bytes_peak": 93891592192, + "segment_count_current": 1095, + "segment_count_peak": 1095, + "timestamp_utc": "2026-08-26T19:27:18.263306Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2577990144, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5587646464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93891592192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93891592192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1095, + "segment_count_current_delta": 0, + "segment_count_peak": 1095, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T192718464385Z.pickle", + "timestamp_utc": "2026-08-26T19:27:18.470689Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192718464385Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192718464385Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..57b674cdedbcd542ab786b954dea1c8ab2bd61db --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T192718464385Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7c0e981e57fd3e9a8b27789737c88ae23533e6783402f96819524988d791765 +size 274555 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T192718665611Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T192718665611Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e553de100c6436e8ab9d4f80cbaf8e7b387f5301 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T192718665611Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67412360704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 90348203008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67412360704, + "active_bytes_peak": 90348203008, + "allocated_bytes_current": 67412360704, + "allocated_bytes_peak": 90348203008, + "device": 0, + "inactive_split_bytes_current": 2577990144, + "inactive_split_bytes_peak": 5587646464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93891592192, + "reserved_bytes_peak": 93891592192, + "segment_count_current": 1095, + "segment_count_peak": 1095, + "timestamp_utc": "2026-08-26T19:27:18.665225Z", + "world_size": 8 + }, + "allocated_bytes_current": 67412360704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 90348203008, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67412360704, + "active_bytes_peak": 90348203008, + "allocated_bytes_current": 67412360704, + "allocated_bytes_peak": 90348203008, + "device": 0, + "inactive_split_bytes_current": 2577990144, + "inactive_split_bytes_peak": 5587646464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 93891592192, + "reserved_bytes_peak": 93891592192, + "segment_count_current": 1095, + "segment_count_peak": 1095, + "timestamp_utc": "2026-08-26T19:27:18.471668Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2577990144, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5587646464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93891592192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 93891592192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1095, + "segment_count_current_delta": 0, + "segment_count_peak": 1095, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T192718665611Z.pickle", + "timestamp_utc": "2026-08-26T19:27:18.671934Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T192718665611Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T192718665611Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..12e3d94b2cbe7aa292726fdfb4f812acb6b7dd7e --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T192718665611Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3439084110ac73525759b7e6b16819324ecb1407d6d71dca58844e22be9a9f5d +size 274555 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192716003678Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192716003678Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06f759ce71e736d28183516b266d407ab93c64e2 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192716003678Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67369143808, + "active_bytes_current_delta": 26683090432, + "active_bytes_peak": 68870750720, + "active_bytes_peak_delta": 28146948608, + "after": { + "active_bytes_current": 67369143808, + "active_bytes_peak": 68870750720, + "allocated_bytes_current": 67369143808, + "allocated_bytes_peak": 68870750720, + "device": 1, + "inactive_split_bytes_current": 2082238976, + "inactive_split_bytes_peak": 4847862784, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 76587991040, + "reserved_bytes_peak": 76587991040, + "segment_count_current": 950, + "segment_count_peak": 950, + "timestamp_utc": "2026-08-26T19:27:16.002989Z", + "world_size": 8 + }, + "allocated_bytes_current": 67369143808, + "allocated_bytes_current_delta": 26683090432, + "allocated_bytes_peak": 68870750720, + "allocated_bytes_peak_delta": 28146948608, + "before": { + "active_bytes_current": 40686053376, + "active_bytes_peak": 40723802112, + "allocated_bytes_current": 40686053376, + "allocated_bytes_peak": 40723802112, + "device": 1, + "inactive_split_bytes_current": 302782464, + "inactive_split_bytes_peak": 522983424, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 43130028032, + "reserved_bytes_peak": 43130028032, + "segment_count_current": 251, + "segment_count_peak": 251, + "timestamp_utc": "2026-08-26T19:26:50.112580Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2082238976, + "inactive_split_bytes_current_delta": 1779456512, + "inactive_split_bytes_peak": 4847862784, + "inactive_split_bytes_peak_delta": 4324879360, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 76587991040, + "reserved_bytes_current_delta": 33457963008, + "reserved_bytes_peak": 76587991040, + "reserved_bytes_peak_delta": 33457963008, + "segment_count_current": 950, + "segment_count_current_delta": 699, + "segment_count_peak": 950, + "segment_count_peak_delta": 699, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T192716003678Z.pickle", + "timestamp_utc": "2026-08-26T19:27:16.030608Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192716003678Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192716003678Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c6fbc260644b64cab6c3402335198cba49d1e184 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T192716003678Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3072d8f9c3580e8ab97e945b4215f327c6ea33ada3ef6d75d93a2fd513bfd46d +size 1450837 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192716331688Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192716331688Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6e112856c55cf2badb7ae804f47d9fb1ea54320 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192716331688Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67369143808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 90334023168, + "active_bytes_peak_delta": 21463272448, + "after": { + "active_bytes_current": 67369143808, + "active_bytes_peak": 90334023168, + "allocated_bytes_current": 67369143808, + "allocated_bytes_peak": 90334023168, + "device": 1, + "inactive_split_bytes_current": 2084336128, + "inactive_split_bytes_peak": 4847862784, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 91939143680, + "reserved_bytes_peak": 91939143680, + "segment_count_current": 1086, + "segment_count_peak": 1086, + "timestamp_utc": "2026-08-26T19:27:16.331262Z", + "world_size": 8 + }, + "allocated_bytes_current": 67369143808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 90334023168, + "allocated_bytes_peak_delta": 21463272448, + "before": { + "active_bytes_current": 67369143808, + "active_bytes_peak": 68870750720, + "allocated_bytes_current": 67369143808, + "allocated_bytes_peak": 68870750720, + "device": 1, + "inactive_split_bytes_current": 2082238976, + "inactive_split_bytes_peak": 4847862784, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 76587991040, + "reserved_bytes_peak": 76587991040, + "segment_count_current": 950, + "segment_count_peak": 950, + "timestamp_utc": "2026-08-26T19:27:16.033528Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2084336128, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 4847862784, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 91939143680, + "reserved_bytes_current_delta": 15351152640, + "reserved_bytes_peak": 91939143680, + "reserved_bytes_peak_delta": 15351152640, + "segment_count_current": 1086, + "segment_count_current_delta": 136, + "segment_count_peak": 1086, + "segment_count_peak_delta": 136, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T192716331688Z.pickle", + "timestamp_utc": "2026-08-26T19:27:16.352897Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192716331688Z.pickle b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192716331688Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cb988ee5c3516709bed36f8e8fd8f9393d6dfdd7 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T192716331688Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d44e138e4e39d1b7e6008474f677826b1f56150bbc8c1fec56d7b00afba277ba +size 1154234 diff --git a/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192716576859Z.json b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192716576859Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1b2882104a0cf2b767518eeebb0de2f8a43622f8 --- /dev/null +++ b/ultraep/tierbmem/tierbmem_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T192716576859Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 67369143808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 90334023168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 67369143808, + "active_bytes_peak": 90334023168, + "allocated_bytes_current": 67369143808, + "allocated_bytes_peak": 90334023168, + "device": 1, + "inactive_split_bytes_current": 2082238976, + "inactive_split_bytes_peak": 4847862784, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 91939143680, + "reserved_bytes_peak": 91939143680, + "segment_count_current": 1086, + "segment_count_peak": 1086, + "timestamp_utc": "2026-08-26T19:27:16.576453Z", + "world_size": 8 + }, + "allocated_bytes_current": 67369143808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 90334023168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 67369143808, + "active_bytes_peak": 90334023168, + "allocated_bytes_current": 67369143808, + "allocated_bytes_peak": 90334023168, + "device": 1, + "inactive_split_bytes_current": 2084336128, + "inactive_split_bytes_peak": 4847862784, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 91939143680, + "reserved_bytes_peak": 91939143680, + "segment_count_current": 1086, + "segment_count_peak": 1086, + "timestamp_utc": "2026-08-26T19:27:16.355548Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2082238976, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 4847862784, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 91939143680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 91939143680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1086, + "segment_count_current_delta": 0, + "segment_count_peak": 1086, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T192716576859Z.pickle", + "timestamp_utc": "2026-08-26T19:27:16.597421Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file