granularity results ultraep/tierbmem @ 2026-08-26T19:21:12Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 2)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191359673365Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.json +79 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.pickle +3 -0
- ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191359985689Z.json +79 -0
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191359673365Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a9147284e646ca631a49016f8385f79c4d653380bd70f634e89082bb2bcf0789
|
| 3 |
+
size 421723
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77318437376,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115696409600,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77318437376,
|
| 8 |
+
"active_bytes_peak": 115696409600,
|
| 9 |
+
"allocated_bytes_current": 77318437376,
|
| 10 |
+
"allocated_bytes_peak": 115696409600,
|
| 11 |
+
"device": 3,
|
| 12 |
+
"inactive_split_bytes_current": 1278625280,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 3,
|
| 20 |
+
"reserved_bytes_current": 119028056064,
|
| 21 |
+
"reserved_bytes_peak": 119028056064,
|
| 22 |
+
"segment_count_current": 1688,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:59.985370Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77318437376,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115696409600,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77318437376,
|
| 33 |
+
"active_bytes_peak": 115696409600,
|
| 34 |
+
"allocated_bytes_current": 77318437376,
|
| 35 |
+
"allocated_bytes_peak": 115696409600,
|
| 36 |
+
"device": 3,
|
| 37 |
+
"inactive_split_bytes_current": 1278625280,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 3,
|
| 45 |
+
"reserved_bytes_current": 119028056064,
|
| 46 |
+
"reserved_bytes_peak": 119028056064,
|
| 47 |
+
"segment_count_current": 1688,
|
| 48 |
+
"segment_count_peak": 1871,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:59.683225Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 3,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1278625280,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 3,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119028056064,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119028056064,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1688,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:59.994353Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:949994a4fd475feffaa977c358664b1931f7a967f3d2c3aa5f900c0fa78c79ab
|
| 3 |
+
size 421723
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77310399488,
|
| 3 |
+
"active_bytes_current_delta": 32789599232,
|
| 4 |
+
"active_bytes_peak": 84832202240,
|
| 5 |
+
"active_bytes_peak_delta": 40286236160,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77310399488,
|
| 8 |
+
"active_bytes_peak": 84832202240,
|
| 9 |
+
"allocated_bytes_current": 77310399488,
|
| 10 |
+
"allocated_bytes_peak": 84832202240,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 1347480576,
|
| 13 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 90617937920,
|
| 21 |
+
"reserved_bytes_peak": 90617937920,
|
| 22 |
+
"segment_count_current": 1701,
|
| 23 |
+
"segment_count_peak": 1701,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:55.493909Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77310399488,
|
| 28 |
+
"allocated_bytes_current_delta": 32789599232,
|
| 29 |
+
"allocated_bytes_peak": 84832202240,
|
| 30 |
+
"allocated_bytes_peak_delta": 40286236160,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520800256,
|
| 33 |
+
"active_bytes_peak": 44545966080,
|
| 34 |
+
"allocated_bytes_current": 44520800256,
|
| 35 |
+
"allocated_bytes_peak": 44545966080,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 270172160,
|
| 38 |
+
"inactive_split_bytes_peak": 874151936,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 47949283328,
|
| 46 |
+
"reserved_bytes_peak": 47949283328,
|
| 47 |
+
"segment_count_current": 615,
|
| 48 |
+
"segment_count_peak": 615,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:28.769737Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1347480576,
|
| 55 |
+
"inactive_split_bytes_current_delta": 1077308416,
|
| 56 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10678164480,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 90617937920,
|
| 68 |
+
"reserved_bytes_current_delta": 42668654592,
|
| 69 |
+
"reserved_bytes_peak": 90617937920,
|
| 70 |
+
"reserved_bytes_peak_delta": 42668654592,
|
| 71 |
+
"segment_count_current": 1701,
|
| 72 |
+
"segment_count_current_delta": 1086,
|
| 73 |
+
"segment_count_peak": 1701,
|
| 74 |
+
"segment_count_peak_delta": 1086,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:55.534297Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3c0e22735663331ab11cb4896069e2b1a9ff56157fe215c14818c650e9c5325a
|
| 3 |
+
size 2309736
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77310399488,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115718885376,
|
| 5 |
+
"active_bytes_peak_delta": 30886683136,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77310399488,
|
| 8 |
+
"active_bytes_peak": 115718885376,
|
| 9 |
+
"allocated_bytes_current": 77310399488,
|
| 10 |
+
"allocated_bytes_peak": 115718885376,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 1347480576,
|
| 13 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 117964800000,
|
| 21 |
+
"reserved_bytes_peak": 117964800000,
|
| 22 |
+
"segment_count_current": 1875,
|
| 23 |
+
"segment_count_peak": 1875,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:56.330153Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77310399488,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115718885376,
|
| 30 |
+
"allocated_bytes_peak_delta": 30886683136,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77310399488,
|
| 33 |
+
"active_bytes_peak": 84832202240,
|
| 34 |
+
"allocated_bytes_current": 77310399488,
|
| 35 |
+
"allocated_bytes_peak": 84832202240,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 1347480576,
|
| 38 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 90617937920,
|
| 46 |
+
"reserved_bytes_peak": 90617937920,
|
| 47 |
+
"segment_count_current": 1701,
|
| 48 |
+
"segment_count_peak": 1701,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:55.539053Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1347480576,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 117964800000,
|
| 68 |
+
"reserved_bytes_current_delta": 27346862080,
|
| 69 |
+
"reserved_bytes_peak": 117964800000,
|
| 70 |
+
"reserved_bytes_peak_delta": 27346862080,
|
| 71 |
+
"segment_count_current": 1875,
|
| 72 |
+
"segment_count_current_delta": 174,
|
| 73 |
+
"segment_count_peak": 1875,
|
| 74 |
+
"segment_count_peak_delta": 174,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:56.368692Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a433ebf6ad0b6fd862e8367d2a334be15e727899160d2cea2f2f6ef8cf5645bd
|
| 3 |
+
size 1832964
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77310399488,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115722308096,
|
| 5 |
+
"active_bytes_peak_delta": 3422720,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77310399488,
|
| 8 |
+
"active_bytes_peak": 115722308096,
|
| 9 |
+
"allocated_bytes_current": 77310399488,
|
| 10 |
+
"allocated_bytes_peak": 115722308096,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 1347480576,
|
| 13 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 117964800000,
|
| 21 |
+
"reserved_bytes_peak": 117964800000,
|
| 22 |
+
"segment_count_current": 1875,
|
| 23 |
+
"segment_count_peak": 1875,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:56.687840Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77310399488,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115722308096,
|
| 30 |
+
"allocated_bytes_peak_delta": 3422720,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77310399488,
|
| 33 |
+
"active_bytes_peak": 115718885376,
|
| 34 |
+
"allocated_bytes_current": 77310399488,
|
| 35 |
+
"allocated_bytes_peak": 115718885376,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 1347480576,
|
| 38 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 117964800000,
|
| 46 |
+
"reserved_bytes_peak": 117964800000,
|
| 47 |
+
"segment_count_current": 1875,
|
| 48 |
+
"segment_count_peak": 1875,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:56.372779Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1347480576,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 117964800000,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 117964800000,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1875,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1875,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:56.719593Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4339be419f1dfeea9b6f2341159cfa1ae4872d4b1f7db0f8bae6c4890a993b2c
|
| 3 |
+
size 1822365
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77327442944,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 115722308096,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77327442944,
|
| 8 |
+
"active_bytes_peak": 115722308096,
|
| 9 |
+
"allocated_bytes_current": 77327442944,
|
| 10 |
+
"allocated_bytes_peak": 115722308096,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 1355602944,
|
| 13 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 119114039296,
|
| 21 |
+
"reserved_bytes_peak": 119114039296,
|
| 22 |
+
"segment_count_current": 1713,
|
| 23 |
+
"segment_count_peak": 1875,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:59.326873Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77327442944,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 115722308096,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77310399488,
|
| 33 |
+
"active_bytes_peak": 115722308096,
|
| 34 |
+
"allocated_bytes_current": 77310399488,
|
| 35 |
+
"allocated_bytes_peak": 115722308096,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 1347480576,
|
| 38 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 117964800000,
|
| 46 |
+
"reserved_bytes_peak": 117964800000,
|
| 47 |
+
"segment_count_current": 1875,
|
| 48 |
+
"segment_count_peak": 1875,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:56.723364Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1355602944,
|
| 55 |
+
"inactive_split_bytes_current_delta": 8122368,
|
| 56 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119114039296,
|
| 68 |
+
"reserved_bytes_current_delta": 1149239296,
|
| 69 |
+
"reserved_bytes_peak": 119114039296,
|
| 70 |
+
"reserved_bytes_peak_delta": 1149239296,
|
| 71 |
+
"segment_count_current": 1713,
|
| 72 |
+
"segment_count_current_delta": -162,
|
| 73 |
+
"segment_count_peak": 1875,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:59.364705Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:81553d2001b26d037fb0ef08ecce377f2483d9911cde8478a990b930f5ca725f
|
| 3 |
+
size 2180760
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77327442944,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115722308096,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77327442944,
|
| 8 |
+
"active_bytes_peak": 115722308096,
|
| 9 |
+
"allocated_bytes_current": 77327442944,
|
| 10 |
+
"allocated_bytes_peak": 115722308096,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 1355602944,
|
| 13 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 119114039296,
|
| 21 |
+
"reserved_bytes_peak": 119114039296,
|
| 22 |
+
"segment_count_current": 1713,
|
| 23 |
+
"segment_count_peak": 1875,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:59.672899Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77327442944,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115722308096,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77327442944,
|
| 33 |
+
"active_bytes_peak": 115722308096,
|
| 34 |
+
"allocated_bytes_current": 77327442944,
|
| 35 |
+
"allocated_bytes_peak": 115722308096,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 1355602944,
|
| 38 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 119114039296,
|
| 46 |
+
"reserved_bytes_peak": 119114039296,
|
| 47 |
+
"segment_count_current": 1713,
|
| 48 |
+
"segment_count_peak": 1875,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:59.368796Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1355602944,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119114039296,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119114039296,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1713,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1875,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:59.681876Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:78ca0e069ad76904f3dbf2aab22aa93fc2d6158cb98cf3f872b9ad469007c991
|
| 3 |
+
size 427667
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77327442944,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115722308096,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77327442944,
|
| 8 |
+
"active_bytes_peak": 115722308096,
|
| 9 |
+
"allocated_bytes_current": 77327442944,
|
| 10 |
+
"allocated_bytes_peak": 115722308096,
|
| 11 |
+
"device": 4,
|
| 12 |
+
"inactive_split_bytes_current": 1355602944,
|
| 13 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 4,
|
| 20 |
+
"reserved_bytes_current": 119114039296,
|
| 21 |
+
"reserved_bytes_peak": 119114039296,
|
| 22 |
+
"segment_count_current": 1713,
|
| 23 |
+
"segment_count_peak": 1875,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:59.985362Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77327442944,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115722308096,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77327442944,
|
| 33 |
+
"active_bytes_peak": 115722308096,
|
| 34 |
+
"allocated_bytes_current": 77327442944,
|
| 35 |
+
"allocated_bytes_peak": 115722308096,
|
| 36 |
+
"device": 4,
|
| 37 |
+
"inactive_split_bytes_current": 1355602944,
|
| 38 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 4,
|
| 45 |
+
"reserved_bytes_current": 119114039296,
|
| 46 |
+
"reserved_bytes_peak": 119114039296,
|
| 47 |
+
"segment_count_current": 1713,
|
| 48 |
+
"segment_count_peak": 1875,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:59.682999Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 4,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1355602944,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11552316416,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 4,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119114039296,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119114039296,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1713,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1875,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:59.994375Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e0d5e64d3c98af45451f3e73d50e863218980461e14423705e2b3c59f0394ec4
|
| 3 |
+
size 427667
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77301393920,
|
| 3 |
+
"active_bytes_current_delta": 32780593664,
|
| 4 |
+
"active_bytes_peak": 84822148608,
|
| 5 |
+
"active_bytes_peak_delta": 40276182528,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77301393920,
|
| 8 |
+
"active_bytes_peak": 84822148608,
|
| 9 |
+
"allocated_bytes_current": 77301393920,
|
| 10 |
+
"allocated_bytes_peak": 84822148608,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 1270502912,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 90550829056,
|
| 21 |
+
"reserved_bytes_peak": 90550829056,
|
| 22 |
+
"segment_count_current": 1692,
|
| 23 |
+
"segment_count_peak": 1692,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:55.493853Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77301393920,
|
| 28 |
+
"allocated_bytes_current_delta": 32780593664,
|
| 29 |
+
"allocated_bytes_peak": 84822148608,
|
| 30 |
+
"allocated_bytes_peak_delta": 40276182528,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520800256,
|
| 33 |
+
"active_bytes_peak": 44545966080,
|
| 34 |
+
"allocated_bytes_current": 44520800256,
|
| 35 |
+
"allocated_bytes_peak": 44545966080,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 421167104,
|
| 38 |
+
"inactive_split_bytes_peak": 1025146880,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:28.769726Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1270502912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 849335808,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10484395520,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 90550829056,
|
| 68 |
+
"reserved_bytes_current_delta": 42299555840,
|
| 69 |
+
"reserved_bytes_peak": 90550829056,
|
| 70 |
+
"reserved_bytes_peak_delta": 42299555840,
|
| 71 |
+
"segment_count_current": 1692,
|
| 72 |
+
"segment_count_current_delta": 1071,
|
| 73 |
+
"segment_count_peak": 1692,
|
| 74 |
+
"segment_count_peak_delta": 1071,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:55.535234Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c76f4addedaacbcbf678996fe2086b7ee58d9fa92ab2f9a784511a86de2d8943
|
| 3 |
+
size 2301744
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77301393920,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115689589760,
|
| 5 |
+
"active_bytes_peak_delta": 30867441152,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77301393920,
|
| 8 |
+
"active_bytes_peak": 115689589760,
|
| 9 |
+
"allocated_bytes_current": 77301393920,
|
| 10 |
+
"allocated_bytes_peak": 115689589760,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 1270502912,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 118057074688,
|
| 21 |
+
"reserved_bytes_peak": 118057074688,
|
| 22 |
+
"segment_count_current": 1871,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:56.330062Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77301393920,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115689589760,
|
| 30 |
+
"allocated_bytes_peak_delta": 30867441152,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77301393920,
|
| 33 |
+
"active_bytes_peak": 84822148608,
|
| 34 |
+
"allocated_bytes_current": 77301393920,
|
| 35 |
+
"allocated_bytes_peak": 84822148608,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 1270502912,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 90550829056,
|
| 46 |
+
"reserved_bytes_peak": 90550829056,
|
| 47 |
+
"segment_count_current": 1692,
|
| 48 |
+
"segment_count_peak": 1692,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:55.539528Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1270502912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 118057074688,
|
| 68 |
+
"reserved_bytes_current_delta": 27506245632,
|
| 69 |
+
"reserved_bytes_peak": 118057074688,
|
| 70 |
+
"reserved_bytes_peak_delta": 27506245632,
|
| 71 |
+
"segment_count_current": 1871,
|
| 72 |
+
"segment_count_current_delta": 179,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 179,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:56.368425Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ae3d144b7a7aa70ae100304915a5acf40b282558356bb7724a06362bcfe37d0c
|
| 3 |
+
size 1827611
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77301393920,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115689851904,
|
| 5 |
+
"active_bytes_peak_delta": 262144,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77301393920,
|
| 8 |
+
"active_bytes_peak": 115689851904,
|
| 9 |
+
"allocated_bytes_current": 77301393920,
|
| 10 |
+
"allocated_bytes_peak": 115689851904,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 1270502912,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 118057074688,
|
| 21 |
+
"reserved_bytes_peak": 118057074688,
|
| 22 |
+
"segment_count_current": 1871,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:56.687752Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77301393920,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115689851904,
|
| 30 |
+
"allocated_bytes_peak_delta": 262144,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77301393920,
|
| 33 |
+
"active_bytes_peak": 115689589760,
|
| 34 |
+
"allocated_bytes_current": 77301393920,
|
| 35 |
+
"allocated_bytes_peak": 115689589760,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 1270502912,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 118057074688,
|
| 46 |
+
"reserved_bytes_peak": 118057074688,
|
| 47 |
+
"segment_count_current": 1871,
|
| 48 |
+
"segment_count_peak": 1871,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:56.372485Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1270502912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 118057074688,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 118057074688,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1871,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:56.719780Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8afceca630ac9709d43717867212431b78f6cd8c6e489d31b455d6b6405ba57f
|
| 3 |
+
size 1818571
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77318437376,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 115696409600,
|
| 5 |
+
"active_bytes_peak_delta": 6557696,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77318437376,
|
| 8 |
+
"active_bytes_peak": 115696409600,
|
| 9 |
+
"allocated_bytes_current": 77318437376,
|
| 10 |
+
"allocated_bytes_peak": 115696409600,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 1278625280,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 119028056064,
|
| 21 |
+
"reserved_bytes_peak": 119028056064,
|
| 22 |
+
"segment_count_current": 1688,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:59.326814Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77318437376,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 115696409600,
|
| 30 |
+
"allocated_bytes_peak_delta": 6557696,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77301393920,
|
| 33 |
+
"active_bytes_peak": 115689851904,
|
| 34 |
+
"allocated_bytes_current": 77301393920,
|
| 35 |
+
"allocated_bytes_peak": 115689851904,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 1270502912,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 118057074688,
|
| 46 |
+
"reserved_bytes_peak": 118057074688,
|
| 47 |
+
"segment_count_current": 1871,
|
| 48 |
+
"segment_count_peak": 1871,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:56.723479Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1278625280,
|
| 55 |
+
"inactive_split_bytes_current_delta": 8122368,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119028056064,
|
| 68 |
+
"reserved_bytes_current_delta": 970981376,
|
| 69 |
+
"reserved_bytes_peak": 119028056064,
|
| 70 |
+
"reserved_bytes_peak_delta": 970981376,
|
| 71 |
+
"segment_count_current": 1688,
|
| 72 |
+
"segment_count_current_delta": -183,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:59.365472Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4bb280f177e21f491febe7b9350f04063491f550be6141cedaa3bd07e6fd62e8
|
| 3 |
+
size 2188971
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77318437376,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115696409600,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77318437376,
|
| 8 |
+
"active_bytes_peak": 115696409600,
|
| 9 |
+
"allocated_bytes_current": 77318437376,
|
| 10 |
+
"allocated_bytes_peak": 115696409600,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 1278625280,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 119028056064,
|
| 21 |
+
"reserved_bytes_peak": 119028056064,
|
| 22 |
+
"segment_count_current": 1688,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:59.672803Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77318437376,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115696409600,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77318437376,
|
| 33 |
+
"active_bytes_peak": 115696409600,
|
| 34 |
+
"allocated_bytes_current": 77318437376,
|
| 35 |
+
"allocated_bytes_peak": 115696409600,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 1278625280,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 119028056064,
|
| 46 |
+
"reserved_bytes_peak": 119028056064,
|
| 47 |
+
"segment_count_current": 1688,
|
| 48 |
+
"segment_count_peak": 1871,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:59.369653Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1278625280,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119028056064,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119028056064,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1688,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:59.681756Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cc4f8e39145028b10440661429b72d0baa3b625ed880807aeba57acb39ef8785
|
| 3 |
+
size 421723
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77318437376,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115696409600,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77318437376,
|
| 8 |
+
"active_bytes_peak": 115696409600,
|
| 9 |
+
"allocated_bytes_current": 77318437376,
|
| 10 |
+
"allocated_bytes_peak": 115696409600,
|
| 11 |
+
"device": 5,
|
| 12 |
+
"inactive_split_bytes_current": 1278625280,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 5,
|
| 20 |
+
"reserved_bytes_current": 119028056064,
|
| 21 |
+
"reserved_bytes_peak": 119028056064,
|
| 22 |
+
"segment_count_current": 1688,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:59.985251Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77318437376,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115696409600,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77318437376,
|
| 33 |
+
"active_bytes_peak": 115696409600,
|
| 34 |
+
"allocated_bytes_current": 77318437376,
|
| 35 |
+
"allocated_bytes_peak": 115696409600,
|
| 36 |
+
"device": 5,
|
| 37 |
+
"inactive_split_bytes_current": 1278625280,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 5,
|
| 45 |
+
"reserved_bytes_current": 119028056064,
|
| 46 |
+
"reserved_bytes_peak": 119028056064,
|
| 47 |
+
"segment_count_current": 1688,
|
| 48 |
+
"segment_count_peak": 1871,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:59.682961Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 5,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1278625280,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 5,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119028056064,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119028056064,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1688,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:59.994017Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8ee5b558e3780976e2049d345a43cd3c121fad600394fb1ca9cdc3230e8c8c23
|
| 3 |
+
size 421723
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77301393920,
|
| 3 |
+
"active_bytes_current_delta": 32780593664,
|
| 4 |
+
"active_bytes_peak": 84822148608,
|
| 5 |
+
"active_bytes_peak_delta": 40276182528,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77301393920,
|
| 8 |
+
"active_bytes_peak": 84822148608,
|
| 9 |
+
"allocated_bytes_current": 77301393920,
|
| 10 |
+
"allocated_bytes_peak": 84822148608,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 1270502912,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 90550829056,
|
| 21 |
+
"reserved_bytes_peak": 90550829056,
|
| 22 |
+
"segment_count_current": 1692,
|
| 23 |
+
"segment_count_peak": 1692,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:55.493820Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77301393920,
|
| 28 |
+
"allocated_bytes_current_delta": 32780593664,
|
| 29 |
+
"allocated_bytes_peak": 84822148608,
|
| 30 |
+
"allocated_bytes_peak_delta": 40276182528,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520800256,
|
| 33 |
+
"active_bytes_peak": 44545966080,
|
| 34 |
+
"allocated_bytes_current": 44520800256,
|
| 35 |
+
"allocated_bytes_peak": 44545966080,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 421167104,
|
| 38 |
+
"inactive_split_bytes_peak": 1025146880,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:28.769718Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1270502912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 849335808,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10484395520,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 90550829056,
|
| 68 |
+
"reserved_bytes_current_delta": 42299555840,
|
| 69 |
+
"reserved_bytes_peak": 90550829056,
|
| 70 |
+
"reserved_bytes_peak_delta": 42299555840,
|
| 71 |
+
"segment_count_current": 1692,
|
| 72 |
+
"segment_count_current_delta": 1071,
|
| 73 |
+
"segment_count_peak": 1692,
|
| 74 |
+
"segment_count_peak_delta": 1071,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:55.534205Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7546d1ea9f3f0d79614678667f8c94abf0d3be2c7be5c9fc2bcae3181cc78087
|
| 3 |
+
size 2301744
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77301393920,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115689589760,
|
| 5 |
+
"active_bytes_peak_delta": 30867441152,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77301393920,
|
| 8 |
+
"active_bytes_peak": 115689589760,
|
| 9 |
+
"allocated_bytes_current": 77301393920,
|
| 10 |
+
"allocated_bytes_peak": 115689589760,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 1270502912,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 118057074688,
|
| 21 |
+
"reserved_bytes_peak": 118057074688,
|
| 22 |
+
"segment_count_current": 1871,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:56.330030Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77301393920,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115689589760,
|
| 30 |
+
"allocated_bytes_peak_delta": 30867441152,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77301393920,
|
| 33 |
+
"active_bytes_peak": 84822148608,
|
| 34 |
+
"allocated_bytes_current": 77301393920,
|
| 35 |
+
"allocated_bytes_peak": 84822148608,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 1270502912,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 90550829056,
|
| 46 |
+
"reserved_bytes_peak": 90550829056,
|
| 47 |
+
"segment_count_current": 1692,
|
| 48 |
+
"segment_count_peak": 1692,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:55.538393Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1270502912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 118057074688,
|
| 68 |
+
"reserved_bytes_current_delta": 27506245632,
|
| 69 |
+
"reserved_bytes_peak": 118057074688,
|
| 70 |
+
"reserved_bytes_peak_delta": 27506245632,
|
| 71 |
+
"segment_count_current": 1871,
|
| 72 |
+
"segment_count_current_delta": 179,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 179,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:56.368321Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1e773a8d295d08939892c810ab7cedf3e6e3476f039cd840179cf2d3625501a4
|
| 3 |
+
size 1827611
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77301393920,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115689851904,
|
| 5 |
+
"active_bytes_peak_delta": 262144,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77301393920,
|
| 8 |
+
"active_bytes_peak": 115689851904,
|
| 9 |
+
"allocated_bytes_current": 77301393920,
|
| 10 |
+
"allocated_bytes_peak": 115689851904,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 1270502912,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 118057074688,
|
| 21 |
+
"reserved_bytes_peak": 118057074688,
|
| 22 |
+
"segment_count_current": 1871,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:56.687721Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77301393920,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115689851904,
|
| 30 |
+
"allocated_bytes_peak_delta": 262144,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77301393920,
|
| 33 |
+
"active_bytes_peak": 115689589760,
|
| 34 |
+
"allocated_bytes_current": 77301393920,
|
| 35 |
+
"allocated_bytes_peak": 115689589760,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 1270502912,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 118057074688,
|
| 46 |
+
"reserved_bytes_peak": 118057074688,
|
| 47 |
+
"segment_count_current": 1871,
|
| 48 |
+
"segment_count_peak": 1871,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:56.372359Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1270502912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 118057074688,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 118057074688,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1871,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:56.719841Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:477b4999a2bd545526ad3f57e9928a1b11714caacafeec8d642e934dd20026ee
|
| 3 |
+
size 1818569
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77318437376,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 115696409600,
|
| 5 |
+
"active_bytes_peak_delta": 6557696,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77318437376,
|
| 8 |
+
"active_bytes_peak": 115696409600,
|
| 9 |
+
"allocated_bytes_current": 77318437376,
|
| 10 |
+
"allocated_bytes_peak": 115696409600,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 1278625280,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 119028056064,
|
| 21 |
+
"reserved_bytes_peak": 119028056064,
|
| 22 |
+
"segment_count_current": 1688,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:59.326767Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77318437376,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 115696409600,
|
| 30 |
+
"allocated_bytes_peak_delta": 6557696,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77301393920,
|
| 33 |
+
"active_bytes_peak": 115689851904,
|
| 34 |
+
"allocated_bytes_current": 77301393920,
|
| 35 |
+
"allocated_bytes_peak": 115689851904,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 1270502912,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 118057074688,
|
| 46 |
+
"reserved_bytes_peak": 118057074688,
|
| 47 |
+
"segment_count_current": 1871,
|
| 48 |
+
"segment_count_peak": 1871,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:56.723546Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1278625280,
|
| 55 |
+
"inactive_split_bytes_current_delta": 8122368,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119028056064,
|
| 68 |
+
"reserved_bytes_current_delta": 970981376,
|
| 69 |
+
"reserved_bytes_peak": 119028056064,
|
| 70 |
+
"reserved_bytes_peak_delta": 970981376,
|
| 71 |
+
"segment_count_current": 1688,
|
| 72 |
+
"segment_count_current_delta": -183,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:59.365192Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e9b11c7add56a973ef2608477b59db90684f3564f425375500655d32885ac3d2
|
| 3 |
+
size 2188967
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77318437376,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115696409600,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77318437376,
|
| 8 |
+
"active_bytes_peak": 115696409600,
|
| 9 |
+
"allocated_bytes_current": 77318437376,
|
| 10 |
+
"allocated_bytes_peak": 115696409600,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 1278625280,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 119028056064,
|
| 21 |
+
"reserved_bytes_peak": 119028056064,
|
| 22 |
+
"segment_count_current": 1688,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:59.672754Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77318437376,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115696409600,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77318437376,
|
| 33 |
+
"active_bytes_peak": 115696409600,
|
| 34 |
+
"allocated_bytes_current": 77318437376,
|
| 35 |
+
"allocated_bytes_peak": 115696409600,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 1278625280,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 119028056064,
|
| 46 |
+
"reserved_bytes_peak": 119028056064,
|
| 47 |
+
"segment_count_current": 1688,
|
| 48 |
+
"segment_count_peak": 1871,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:59.370060Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1278625280,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119028056064,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119028056064,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1688,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:59.681579Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:46494cb4f75b92861470d7953cc41c7290a9ef1a4b8a8fbbc30d32287de3c77c
|
| 3 |
+
size 421719
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77318437376,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115696409600,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77318437376,
|
| 8 |
+
"active_bytes_peak": 115696409600,
|
| 9 |
+
"allocated_bytes_current": 77318437376,
|
| 10 |
+
"allocated_bytes_peak": 115696409600,
|
| 11 |
+
"device": 6,
|
| 12 |
+
"inactive_split_bytes_current": 1278625280,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 6,
|
| 20 |
+
"reserved_bytes_current": 119028056064,
|
| 21 |
+
"reserved_bytes_peak": 119028056064,
|
| 22 |
+
"segment_count_current": 1688,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:59.985258Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77318437376,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115696409600,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77318437376,
|
| 33 |
+
"active_bytes_peak": 115696409600,
|
| 34 |
+
"allocated_bytes_current": 77318437376,
|
| 35 |
+
"allocated_bytes_peak": 115696409600,
|
| 36 |
+
"device": 6,
|
| 37 |
+
"inactive_split_bytes_current": 1278625280,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 6,
|
| 45 |
+
"reserved_bytes_current": 119028056064,
|
| 46 |
+
"reserved_bytes_peak": 119028056064,
|
| 47 |
+
"segment_count_current": 1688,
|
| 48 |
+
"segment_count_peak": 1871,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:59.682681Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 6,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1278625280,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 6,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119028056064,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119028056064,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1688,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:59.994267Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a8ed52e7fb1ba42cb3dfb7cfe137845aec85ea0965a531b8b3fcf3e7dd3f7e76
|
| 3 |
+
size 421719
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77301393920,
|
| 3 |
+
"active_bytes_current_delta": 32780593664,
|
| 4 |
+
"active_bytes_peak": 84822148608,
|
| 5 |
+
"active_bytes_peak_delta": 40276182528,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77301393920,
|
| 8 |
+
"active_bytes_peak": 84822148608,
|
| 9 |
+
"allocated_bytes_current": 77301393920,
|
| 10 |
+
"allocated_bytes_peak": 84822148608,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 1270502912,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 0,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 90550829056,
|
| 21 |
+
"reserved_bytes_peak": 90550829056,
|
| 22 |
+
"segment_count_current": 1692,
|
| 23 |
+
"segment_count_peak": 1692,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:55.493830Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77301393920,
|
| 28 |
+
"allocated_bytes_current_delta": 32780593664,
|
| 29 |
+
"allocated_bytes_peak": 84822148608,
|
| 30 |
+
"allocated_bytes_peak_delta": 40276182528,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 44520800256,
|
| 33 |
+
"active_bytes_peak": 44545966080,
|
| 34 |
+
"allocated_bytes_current": 44520800256,
|
| 35 |
+
"allocated_bytes_peak": 44545966080,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 421167104,
|
| 38 |
+
"inactive_split_bytes_peak": 1025146880,
|
| 39 |
+
"iteration": 0,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 48251273216,
|
| 46 |
+
"reserved_bytes_peak": 48251273216,
|
| 47 |
+
"segment_count_current": 621,
|
| 48 |
+
"segment_count_peak": 621,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:28.769712Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1270502912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 849335808,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 10484395520,
|
| 58 |
+
"iteration": 0,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 90550829056,
|
| 68 |
+
"reserved_bytes_current_delta": 42299555840,
|
| 69 |
+
"reserved_bytes_peak": 90550829056,
|
| 70 |
+
"reserved_bytes_peak_delta": 42299555840,
|
| 71 |
+
"segment_count_current": 1692,
|
| 72 |
+
"segment_count_current_delta": 1071,
|
| 73 |
+
"segment_count_peak": 1692,
|
| 74 |
+
"segment_count_peak_delta": 1071,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:55.533437Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e3a4a4c320ca7cff52964878a75848b1c44cdceca40f57f8cf772c106d7ed2c3
|
| 3 |
+
size 2301744
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77301393920,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115689589760,
|
| 5 |
+
"active_bytes_peak_delta": 30867441152,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77301393920,
|
| 8 |
+
"active_bytes_peak": 115689589760,
|
| 9 |
+
"allocated_bytes_current": 77301393920,
|
| 10 |
+
"allocated_bytes_peak": 115689589760,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 1270502912,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 1,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 118057074688,
|
| 21 |
+
"reserved_bytes_peak": 118057074688,
|
| 22 |
+
"segment_count_current": 1871,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:56.330084Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77301393920,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115689589760,
|
| 30 |
+
"allocated_bytes_peak_delta": 30867441152,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77301393920,
|
| 33 |
+
"active_bytes_peak": 84822148608,
|
| 34 |
+
"allocated_bytes_current": 77301393920,
|
| 35 |
+
"allocated_bytes_peak": 84822148608,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 1270502912,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 1,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 90550829056,
|
| 46 |
+
"reserved_bytes_peak": 90550829056,
|
| 47 |
+
"segment_count_current": 1692,
|
| 48 |
+
"segment_count_peak": 1692,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:55.537623Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1270502912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 1,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 118057074688,
|
| 68 |
+
"reserved_bytes_current_delta": 27506245632,
|
| 69 |
+
"reserved_bytes_peak": 118057074688,
|
| 70 |
+
"reserved_bytes_peak_delta": 27506245632,
|
| 71 |
+
"segment_count_current": 1871,
|
| 72 |
+
"segment_count_current_delta": 179,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 179,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:56.366764Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b039cf25681691e304df5a1e5986b58c7f46fdda2fa1a9e415857fc5f19d97c1
|
| 3 |
+
size 1827611
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77301393920,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115689851904,
|
| 5 |
+
"active_bytes_peak_delta": 262144,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77301393920,
|
| 8 |
+
"active_bytes_peak": 115689851904,
|
| 9 |
+
"allocated_bytes_current": 77301393920,
|
| 10 |
+
"allocated_bytes_peak": 115689851904,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 1270502912,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 2,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 118057074688,
|
| 21 |
+
"reserved_bytes_peak": 118057074688,
|
| 22 |
+
"segment_count_current": 1871,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:56.687775Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77301393920,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115689851904,
|
| 30 |
+
"allocated_bytes_peak_delta": 262144,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77301393920,
|
| 33 |
+
"active_bytes_peak": 115689589760,
|
| 34 |
+
"allocated_bytes_current": 77301393920,
|
| 35 |
+
"allocated_bytes_peak": 115689589760,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 1270502912,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 2,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 118057074688,
|
| 46 |
+
"reserved_bytes_peak": 118057074688,
|
| 47 |
+
"segment_count_current": 1871,
|
| 48 |
+
"segment_count_peak": 1871,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:56.370614Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1270502912,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 2,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 118057074688,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 118057074688,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1871,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:56.718676Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:83a6c52cd07fca4b9503d38d0e115d42d3d8a779f0d8fd7be8553d584bcce60e
|
| 3 |
+
size 1818569
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77318437376,
|
| 3 |
+
"active_bytes_current_delta": 17043456,
|
| 4 |
+
"active_bytes_peak": 115696409600,
|
| 5 |
+
"active_bytes_peak_delta": 6557696,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77318437376,
|
| 8 |
+
"active_bytes_peak": 115696409600,
|
| 9 |
+
"allocated_bytes_current": 77318437376,
|
| 10 |
+
"allocated_bytes_peak": 115696409600,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 1278625280,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 3,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 119028056064,
|
| 21 |
+
"reserved_bytes_peak": 119028056064,
|
| 22 |
+
"segment_count_current": 1688,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:59.326845Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77318437376,
|
| 28 |
+
"allocated_bytes_current_delta": 17043456,
|
| 29 |
+
"allocated_bytes_peak": 115696409600,
|
| 30 |
+
"allocated_bytes_peak_delta": 6557696,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77301393920,
|
| 33 |
+
"active_bytes_peak": 115689851904,
|
| 34 |
+
"allocated_bytes_current": 77301393920,
|
| 35 |
+
"allocated_bytes_peak": 115689851904,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 1270502912,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 3,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 118057074688,
|
| 46 |
+
"reserved_bytes_peak": 118057074688,
|
| 47 |
+
"segment_count_current": 1871,
|
| 48 |
+
"segment_count_peak": 1871,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:56.722150Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1278625280,
|
| 55 |
+
"inactive_split_bytes_current_delta": 8122368,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 3,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119028056064,
|
| 68 |
+
"reserved_bytes_current_delta": 970981376,
|
| 69 |
+
"reserved_bytes_peak": 119028056064,
|
| 70 |
+
"reserved_bytes_peak_delta": 970981376,
|
| 71 |
+
"segment_count_current": 1688,
|
| 72 |
+
"segment_count_current_delta": -183,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:59.365437Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3f769ed3ca5a81059504fdac9122eee3bc84daf6b28b36b3f4458951c58e6738
|
| 3 |
+
size 2188969
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77318437376,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115696409600,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77318437376,
|
| 8 |
+
"active_bytes_peak": 115696409600,
|
| 9 |
+
"allocated_bytes_current": 77318437376,
|
| 10 |
+
"allocated_bytes_peak": 115696409600,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 1278625280,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 4,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 119028056064,
|
| 21 |
+
"reserved_bytes_peak": 119028056064,
|
| 22 |
+
"segment_count_current": 1688,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:59.672803Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77318437376,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115696409600,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77318437376,
|
| 33 |
+
"active_bytes_peak": 115696409600,
|
| 34 |
+
"allocated_bytes_current": 77318437376,
|
| 35 |
+
"allocated_bytes_peak": 115696409600,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 1278625280,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 4,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 119028056064,
|
| 46 |
+
"reserved_bytes_peak": 119028056064,
|
| 47 |
+
"segment_count_current": 1688,
|
| 48 |
+
"segment_count_peak": 1871,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:59.369446Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1278625280,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 4,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119028056064,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119028056064,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1688,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:59.681882Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.pickle
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2530246e9f8fdaeddf81c262d7186b4d97c1e7010a93d79ebdef32546b960e77
|
| 3 |
+
size 421721
|
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191359985689Z.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"active_bytes_current": 77318437376,
|
| 3 |
+
"active_bytes_current_delta": 0,
|
| 4 |
+
"active_bytes_peak": 115696409600,
|
| 5 |
+
"active_bytes_peak_delta": 0,
|
| 6 |
+
"after": {
|
| 7 |
+
"active_bytes_current": 77318437376,
|
| 8 |
+
"active_bytes_peak": 115696409600,
|
| 9 |
+
"allocated_bytes_current": 77318437376,
|
| 10 |
+
"allocated_bytes_peak": 115696409600,
|
| 11 |
+
"device": 7,
|
| 12 |
+
"inactive_split_bytes_current": 1278625280,
|
| 13 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 14 |
+
"iteration": 5,
|
| 15 |
+
"mode": "iteration",
|
| 16 |
+
"num_alloc_retries": 0,
|
| 17 |
+
"num_ooms": 0,
|
| 18 |
+
"phase": "iteration_end",
|
| 19 |
+
"rank": 7,
|
| 20 |
+
"reserved_bytes_current": 119028056064,
|
| 21 |
+
"reserved_bytes_peak": 119028056064,
|
| 22 |
+
"segment_count_current": 1688,
|
| 23 |
+
"segment_count_peak": 1871,
|
| 24 |
+
"timestamp_utc": "2026-08-26T19:13:59.985310Z",
|
| 25 |
+
"world_size": 8
|
| 26 |
+
},
|
| 27 |
+
"allocated_bytes_current": 77318437376,
|
| 28 |
+
"allocated_bytes_current_delta": 0,
|
| 29 |
+
"allocated_bytes_peak": 115696409600,
|
| 30 |
+
"allocated_bytes_peak_delta": 0,
|
| 31 |
+
"before": {
|
| 32 |
+
"active_bytes_current": 77318437376,
|
| 33 |
+
"active_bytes_peak": 115696409600,
|
| 34 |
+
"allocated_bytes_current": 77318437376,
|
| 35 |
+
"allocated_bytes_peak": 115696409600,
|
| 36 |
+
"device": 7,
|
| 37 |
+
"inactive_split_bytes_current": 1278625280,
|
| 38 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 39 |
+
"iteration": 5,
|
| 40 |
+
"mode": "iteration",
|
| 41 |
+
"num_alloc_retries": 0,
|
| 42 |
+
"num_ooms": 0,
|
| 43 |
+
"phase": "iteration_start",
|
| 44 |
+
"rank": 7,
|
| 45 |
+
"reserved_bytes_current": 119028056064,
|
| 46 |
+
"reserved_bytes_peak": 119028056064,
|
| 47 |
+
"segment_count_current": 1688,
|
| 48 |
+
"segment_count_peak": 1871,
|
| 49 |
+
"timestamp_utc": "2026-08-26T19:13:59.683052Z",
|
| 50 |
+
"world_size": 8
|
| 51 |
+
},
|
| 52 |
+
"device": 7,
|
| 53 |
+
"device_delta": 0,
|
| 54 |
+
"inactive_split_bytes_current": 1278625280,
|
| 55 |
+
"inactive_split_bytes_current_delta": 0,
|
| 56 |
+
"inactive_split_bytes_peak": 11509542400,
|
| 57 |
+
"inactive_split_bytes_peak_delta": 0,
|
| 58 |
+
"iteration": 5,
|
| 59 |
+
"iteration_delta": 0,
|
| 60 |
+
"mode": "iteration",
|
| 61 |
+
"num_alloc_retries": 0,
|
| 62 |
+
"num_alloc_retries_delta": 0,
|
| 63 |
+
"num_ooms": 0,
|
| 64 |
+
"num_ooms_delta": 0,
|
| 65 |
+
"rank": 7,
|
| 66 |
+
"rank_delta": 0,
|
| 67 |
+
"reserved_bytes_current": 119028056064,
|
| 68 |
+
"reserved_bytes_current_delta": 0,
|
| 69 |
+
"reserved_bytes_peak": 119028056064,
|
| 70 |
+
"reserved_bytes_peak_delta": 0,
|
| 71 |
+
"segment_count_current": 1688,
|
| 72 |
+
"segment_count_current_delta": 0,
|
| 73 |
+
"segment_count_peak": 1871,
|
| 74 |
+
"segment_count_peak_delta": 0,
|
| 75 |
+
"snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T191359985689Z.pickle",
|
| 76 |
+
"timestamp_utc": "2026-08-26T19:13:59.994216Z",
|
| 77 |
+
"world_size": 8,
|
| 78 |
+
"world_size_delta": 0
|
| 79 |
+
}
|