AbhishekGhosh1998 commited on
Commit
756e47c
·
verified ·
1 Parent(s): bca31b2

granularity results ultraep/tierbmem @ 2026-08-26T19:21:12Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 2)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191359673365Z.pickle +3 -0
  2. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.json +79 -0
  3. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.pickle +3 -0
  4. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.json +79 -0
  5. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.pickle +3 -0
  6. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.json +79 -0
  7. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.pickle +3 -0
  8. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.json +79 -0
  9. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.pickle +3 -0
  10. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.json +79 -0
  11. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.pickle +3 -0
  12. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.json +79 -0
  13. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.pickle +3 -0
  14. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.json +79 -0
  15. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.pickle +3 -0
  16. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.json +79 -0
  17. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.pickle +3 -0
  18. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.json +79 -0
  19. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.pickle +3 -0
  20. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.json +79 -0
  21. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.pickle +3 -0
  22. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.json +79 -0
  23. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.pickle +3 -0
  24. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.json +79 -0
  25. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.pickle +3 -0
  26. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.json +79 -0
  27. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.pickle +3 -0
  28. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.json +79 -0
  29. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.pickle +3 -0
  30. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.json +79 -0
  31. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.pickle +3 -0
  32. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.json +79 -0
  33. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.pickle +3 -0
  34. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.json +79 -0
  35. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.pickle +3 -0
  36. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.json +79 -0
  37. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.pickle +3 -0
  38. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.json +79 -0
  39. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.pickle +3 -0
  40. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.json +79 -0
  41. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.pickle +3 -0
  42. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.json +79 -0
  43. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.pickle +3 -0
  44. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.json +79 -0
  45. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.pickle +3 -0
  46. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.json +79 -0
  47. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.pickle +3 -0
  48. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.json +79 -0
  49. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.pickle +3 -0
  50. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191359985689Z.json +79 -0
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191359673365Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9147284e646ca631a49016f8385f79c4d653380bd70f634e89082bb2bcf0789
3
+ size 421723
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77318437376,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115696409600,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77318437376,
8
+ "active_bytes_peak": 115696409600,
9
+ "allocated_bytes_current": 77318437376,
10
+ "allocated_bytes_peak": 115696409600,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 1278625280,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 119028056064,
21
+ "reserved_bytes_peak": 119028056064,
22
+ "segment_count_current": 1688,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:59.985370Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77318437376,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115696409600,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77318437376,
33
+ "active_bytes_peak": 115696409600,
34
+ "allocated_bytes_current": 77318437376,
35
+ "allocated_bytes_peak": 115696409600,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 1278625280,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 119028056064,
46
+ "reserved_bytes_peak": 119028056064,
47
+ "segment_count_current": 1688,
48
+ "segment_count_peak": 1871,
49
+ "timestamp_utc": "2026-08-26T19:13:59.683225Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1278625280,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119028056064,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119028056064,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1688,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:59.994353Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191359985763Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:949994a4fd475feffaa977c358664b1931f7a967f3d2c3aa5f900c0fa78c79ab
3
+ size 421723
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77310399488,
3
+ "active_bytes_current_delta": 32789599232,
4
+ "active_bytes_peak": 84832202240,
5
+ "active_bytes_peak_delta": 40286236160,
6
+ "after": {
7
+ "active_bytes_current": 77310399488,
8
+ "active_bytes_peak": 84832202240,
9
+ "allocated_bytes_current": 77310399488,
10
+ "allocated_bytes_peak": 84832202240,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1347480576,
13
+ "inactive_split_bytes_peak": 11552316416,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 90617937920,
21
+ "reserved_bytes_peak": 90617937920,
22
+ "segment_count_current": 1701,
23
+ "segment_count_peak": 1701,
24
+ "timestamp_utc": "2026-08-26T19:13:55.493909Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77310399488,
28
+ "allocated_bytes_current_delta": 32789599232,
29
+ "allocated_bytes_peak": 84832202240,
30
+ "allocated_bytes_peak_delta": 40286236160,
31
+ "before": {
32
+ "active_bytes_current": 44520800256,
33
+ "active_bytes_peak": 44545966080,
34
+ "allocated_bytes_current": 44520800256,
35
+ "allocated_bytes_peak": 44545966080,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 270172160,
38
+ "inactive_split_bytes_peak": 874151936,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 47949283328,
46
+ "reserved_bytes_peak": 47949283328,
47
+ "segment_count_current": 615,
48
+ "segment_count_peak": 615,
49
+ "timestamp_utc": "2026-08-26T19:13:28.769737Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1347480576,
55
+ "inactive_split_bytes_current_delta": 1077308416,
56
+ "inactive_split_bytes_peak": 11552316416,
57
+ "inactive_split_bytes_peak_delta": 10678164480,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 90617937920,
68
+ "reserved_bytes_current_delta": 42668654592,
69
+ "reserved_bytes_peak": 90617937920,
70
+ "reserved_bytes_peak_delta": 42668654592,
71
+ "segment_count_current": 1701,
72
+ "segment_count_current_delta": 1086,
73
+ "segment_count_peak": 1701,
74
+ "segment_count_peak_delta": 1086,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:55.534297Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191355494408Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3c0e22735663331ab11cb4896069e2b1a9ff56157fe215c14818c650e9c5325a
3
+ size 2309736
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77310399488,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115718885376,
5
+ "active_bytes_peak_delta": 30886683136,
6
+ "after": {
7
+ "active_bytes_current": 77310399488,
8
+ "active_bytes_peak": 115718885376,
9
+ "allocated_bytes_current": 77310399488,
10
+ "allocated_bytes_peak": 115718885376,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1347480576,
13
+ "inactive_split_bytes_peak": 11552316416,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 117964800000,
21
+ "reserved_bytes_peak": 117964800000,
22
+ "segment_count_current": 1875,
23
+ "segment_count_peak": 1875,
24
+ "timestamp_utc": "2026-08-26T19:13:56.330153Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77310399488,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115718885376,
30
+ "allocated_bytes_peak_delta": 30886683136,
31
+ "before": {
32
+ "active_bytes_current": 77310399488,
33
+ "active_bytes_peak": 84832202240,
34
+ "allocated_bytes_current": 77310399488,
35
+ "allocated_bytes_peak": 84832202240,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 1347480576,
38
+ "inactive_split_bytes_peak": 11552316416,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 90617937920,
46
+ "reserved_bytes_peak": 90617937920,
47
+ "segment_count_current": 1701,
48
+ "segment_count_peak": 1701,
49
+ "timestamp_utc": "2026-08-26T19:13:55.539053Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1347480576,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11552316416,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 117964800000,
68
+ "reserved_bytes_current_delta": 27346862080,
69
+ "reserved_bytes_peak": 117964800000,
70
+ "reserved_bytes_peak_delta": 27346862080,
71
+ "segment_count_current": 1875,
72
+ "segment_count_current_delta": 174,
73
+ "segment_count_peak": 1875,
74
+ "segment_count_peak_delta": 174,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:56.368692Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191356330609Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a433ebf6ad0b6fd862e8367d2a334be15e727899160d2cea2f2f6ef8cf5645bd
3
+ size 1832964
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77310399488,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115722308096,
5
+ "active_bytes_peak_delta": 3422720,
6
+ "after": {
7
+ "active_bytes_current": 77310399488,
8
+ "active_bytes_peak": 115722308096,
9
+ "allocated_bytes_current": 77310399488,
10
+ "allocated_bytes_peak": 115722308096,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1347480576,
13
+ "inactive_split_bytes_peak": 11552316416,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 117964800000,
21
+ "reserved_bytes_peak": 117964800000,
22
+ "segment_count_current": 1875,
23
+ "segment_count_peak": 1875,
24
+ "timestamp_utc": "2026-08-26T19:13:56.687840Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77310399488,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115722308096,
30
+ "allocated_bytes_peak_delta": 3422720,
31
+ "before": {
32
+ "active_bytes_current": 77310399488,
33
+ "active_bytes_peak": 115718885376,
34
+ "allocated_bytes_current": 77310399488,
35
+ "allocated_bytes_peak": 115718885376,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 1347480576,
38
+ "inactive_split_bytes_peak": 11552316416,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 117964800000,
46
+ "reserved_bytes_peak": 117964800000,
47
+ "segment_count_current": 1875,
48
+ "segment_count_peak": 1875,
49
+ "timestamp_utc": "2026-08-26T19:13:56.372779Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1347480576,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11552316416,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 117964800000,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 117964800000,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1875,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1875,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:56.719593Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191356688261Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4339be419f1dfeea9b6f2341159cfa1ae4872d4b1f7db0f8bae6c4890a993b2c
3
+ size 1822365
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77327442944,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 115722308096,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77327442944,
8
+ "active_bytes_peak": 115722308096,
9
+ "allocated_bytes_current": 77327442944,
10
+ "allocated_bytes_peak": 115722308096,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1355602944,
13
+ "inactive_split_bytes_peak": 11552316416,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 119114039296,
21
+ "reserved_bytes_peak": 119114039296,
22
+ "segment_count_current": 1713,
23
+ "segment_count_peak": 1875,
24
+ "timestamp_utc": "2026-08-26T19:13:59.326873Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77327442944,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 115722308096,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77310399488,
33
+ "active_bytes_peak": 115722308096,
34
+ "allocated_bytes_current": 77310399488,
35
+ "allocated_bytes_peak": 115722308096,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 1347480576,
38
+ "inactive_split_bytes_peak": 11552316416,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 117964800000,
46
+ "reserved_bytes_peak": 117964800000,
47
+ "segment_count_current": 1875,
48
+ "segment_count_peak": 1875,
49
+ "timestamp_utc": "2026-08-26T19:13:56.723364Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1355602944,
55
+ "inactive_split_bytes_current_delta": 8122368,
56
+ "inactive_split_bytes_peak": 11552316416,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119114039296,
68
+ "reserved_bytes_current_delta": 1149239296,
69
+ "reserved_bytes_peak": 119114039296,
70
+ "reserved_bytes_peak_delta": 1149239296,
71
+ "segment_count_current": 1713,
72
+ "segment_count_current_delta": -162,
73
+ "segment_count_peak": 1875,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:59.364705Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191359327343Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:81553d2001b26d037fb0ef08ecce377f2483d9911cde8478a990b930f5ca725f
3
+ size 2180760
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77327442944,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115722308096,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77327442944,
8
+ "active_bytes_peak": 115722308096,
9
+ "allocated_bytes_current": 77327442944,
10
+ "allocated_bytes_peak": 115722308096,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1355602944,
13
+ "inactive_split_bytes_peak": 11552316416,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 119114039296,
21
+ "reserved_bytes_peak": 119114039296,
22
+ "segment_count_current": 1713,
23
+ "segment_count_peak": 1875,
24
+ "timestamp_utc": "2026-08-26T19:13:59.672899Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77327442944,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115722308096,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77327442944,
33
+ "active_bytes_peak": 115722308096,
34
+ "allocated_bytes_current": 77327442944,
35
+ "allocated_bytes_peak": 115722308096,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 1355602944,
38
+ "inactive_split_bytes_peak": 11552316416,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 119114039296,
46
+ "reserved_bytes_peak": 119114039296,
47
+ "segment_count_current": 1713,
48
+ "segment_count_peak": 1875,
49
+ "timestamp_utc": "2026-08-26T19:13:59.368796Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1355602944,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11552316416,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119114039296,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119114039296,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1713,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1875,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:59.681876Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191359673301Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:78ca0e069ad76904f3dbf2aab22aa93fc2d6158cb98cf3f872b9ad469007c991
3
+ size 427667
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77327442944,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115722308096,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77327442944,
8
+ "active_bytes_peak": 115722308096,
9
+ "allocated_bytes_current": 77327442944,
10
+ "allocated_bytes_peak": 115722308096,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1355602944,
13
+ "inactive_split_bytes_peak": 11552316416,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 119114039296,
21
+ "reserved_bytes_peak": 119114039296,
22
+ "segment_count_current": 1713,
23
+ "segment_count_peak": 1875,
24
+ "timestamp_utc": "2026-08-26T19:13:59.985362Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77327442944,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115722308096,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77327442944,
33
+ "active_bytes_peak": 115722308096,
34
+ "allocated_bytes_current": 77327442944,
35
+ "allocated_bytes_peak": 115722308096,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 1355602944,
38
+ "inactive_split_bytes_peak": 11552316416,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 119114039296,
46
+ "reserved_bytes_peak": 119114039296,
47
+ "segment_count_current": 1713,
48
+ "segment_count_peak": 1875,
49
+ "timestamp_utc": "2026-08-26T19:13:59.682999Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1355602944,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11552316416,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119114039296,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119114039296,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1713,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1875,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:59.994375Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191359985734Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0d5e64d3c98af45451f3e73d50e863218980461e14423705e2b3c59f0394ec4
3
+ size 427667
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77301393920,
3
+ "active_bytes_current_delta": 32780593664,
4
+ "active_bytes_peak": 84822148608,
5
+ "active_bytes_peak_delta": 40276182528,
6
+ "after": {
7
+ "active_bytes_current": 77301393920,
8
+ "active_bytes_peak": 84822148608,
9
+ "allocated_bytes_current": 77301393920,
10
+ "allocated_bytes_peak": 84822148608,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1270502912,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 90550829056,
21
+ "reserved_bytes_peak": 90550829056,
22
+ "segment_count_current": 1692,
23
+ "segment_count_peak": 1692,
24
+ "timestamp_utc": "2026-08-26T19:13:55.493853Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77301393920,
28
+ "allocated_bytes_current_delta": 32780593664,
29
+ "allocated_bytes_peak": 84822148608,
30
+ "allocated_bytes_peak_delta": 40276182528,
31
+ "before": {
32
+ "active_bytes_current": 44520800256,
33
+ "active_bytes_peak": 44545966080,
34
+ "allocated_bytes_current": 44520800256,
35
+ "allocated_bytes_peak": 44545966080,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 421167104,
38
+ "inactive_split_bytes_peak": 1025146880,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T19:13:28.769726Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1270502912,
55
+ "inactive_split_bytes_current_delta": 849335808,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 10484395520,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 90550829056,
68
+ "reserved_bytes_current_delta": 42299555840,
69
+ "reserved_bytes_peak": 90550829056,
70
+ "reserved_bytes_peak_delta": 42299555840,
71
+ "segment_count_current": 1692,
72
+ "segment_count_current_delta": 1071,
73
+ "segment_count_peak": 1692,
74
+ "segment_count_peak_delta": 1071,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:55.535234Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191355494327Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c76f4addedaacbcbf678996fe2086b7ee58d9fa92ab2f9a784511a86de2d8943
3
+ size 2301744
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77301393920,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115689589760,
5
+ "active_bytes_peak_delta": 30867441152,
6
+ "after": {
7
+ "active_bytes_current": 77301393920,
8
+ "active_bytes_peak": 115689589760,
9
+ "allocated_bytes_current": 77301393920,
10
+ "allocated_bytes_peak": 115689589760,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1270502912,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 118057074688,
21
+ "reserved_bytes_peak": 118057074688,
22
+ "segment_count_current": 1871,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:56.330062Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77301393920,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115689589760,
30
+ "allocated_bytes_peak_delta": 30867441152,
31
+ "before": {
32
+ "active_bytes_current": 77301393920,
33
+ "active_bytes_peak": 84822148608,
34
+ "allocated_bytes_current": 77301393920,
35
+ "allocated_bytes_peak": 84822148608,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1270502912,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 90550829056,
46
+ "reserved_bytes_peak": 90550829056,
47
+ "segment_count_current": 1692,
48
+ "segment_count_peak": 1692,
49
+ "timestamp_utc": "2026-08-26T19:13:55.539528Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1270502912,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 118057074688,
68
+ "reserved_bytes_current_delta": 27506245632,
69
+ "reserved_bytes_peak": 118057074688,
70
+ "reserved_bytes_peak_delta": 27506245632,
71
+ "segment_count_current": 1871,
72
+ "segment_count_current_delta": 179,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 179,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:56.368425Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191356330504Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ae3d144b7a7aa70ae100304915a5acf40b282558356bb7724a06362bcfe37d0c
3
+ size 1827611
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77301393920,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115689851904,
5
+ "active_bytes_peak_delta": 262144,
6
+ "after": {
7
+ "active_bytes_current": 77301393920,
8
+ "active_bytes_peak": 115689851904,
9
+ "allocated_bytes_current": 77301393920,
10
+ "allocated_bytes_peak": 115689851904,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1270502912,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 118057074688,
21
+ "reserved_bytes_peak": 118057074688,
22
+ "segment_count_current": 1871,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:56.687752Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77301393920,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115689851904,
30
+ "allocated_bytes_peak_delta": 262144,
31
+ "before": {
32
+ "active_bytes_current": 77301393920,
33
+ "active_bytes_peak": 115689589760,
34
+ "allocated_bytes_current": 77301393920,
35
+ "allocated_bytes_peak": 115689589760,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1270502912,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 118057074688,
46
+ "reserved_bytes_peak": 118057074688,
47
+ "segment_count_current": 1871,
48
+ "segment_count_peak": 1871,
49
+ "timestamp_utc": "2026-08-26T19:13:56.372485Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1270502912,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 118057074688,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 118057074688,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1871,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:56.719780Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T191356688169Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8afceca630ac9709d43717867212431b78f6cd8c6e489d31b455d6b6405ba57f
3
+ size 1818571
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77318437376,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 115696409600,
5
+ "active_bytes_peak_delta": 6557696,
6
+ "after": {
7
+ "active_bytes_current": 77318437376,
8
+ "active_bytes_peak": 115696409600,
9
+ "allocated_bytes_current": 77318437376,
10
+ "allocated_bytes_peak": 115696409600,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1278625280,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 119028056064,
21
+ "reserved_bytes_peak": 119028056064,
22
+ "segment_count_current": 1688,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:59.326814Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77318437376,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 115696409600,
30
+ "allocated_bytes_peak_delta": 6557696,
31
+ "before": {
32
+ "active_bytes_current": 77301393920,
33
+ "active_bytes_peak": 115689851904,
34
+ "allocated_bytes_current": 77301393920,
35
+ "allocated_bytes_peak": 115689851904,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1270502912,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 118057074688,
46
+ "reserved_bytes_peak": 118057074688,
47
+ "segment_count_current": 1871,
48
+ "segment_count_peak": 1871,
49
+ "timestamp_utc": "2026-08-26T19:13:56.723479Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1278625280,
55
+ "inactive_split_bytes_current_delta": 8122368,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119028056064,
68
+ "reserved_bytes_current_delta": 970981376,
69
+ "reserved_bytes_peak": 119028056064,
70
+ "reserved_bytes_peak_delta": 970981376,
71
+ "segment_count_current": 1688,
72
+ "segment_count_current_delta": -183,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:59.365472Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T191359327282Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4bb280f177e21f491febe7b9350f04063491f550be6141cedaa3bd07e6fd62e8
3
+ size 2188971
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77318437376,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115696409600,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77318437376,
8
+ "active_bytes_peak": 115696409600,
9
+ "allocated_bytes_current": 77318437376,
10
+ "allocated_bytes_peak": 115696409600,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1278625280,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 119028056064,
21
+ "reserved_bytes_peak": 119028056064,
22
+ "segment_count_current": 1688,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:59.672803Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77318437376,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115696409600,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77318437376,
33
+ "active_bytes_peak": 115696409600,
34
+ "allocated_bytes_current": 77318437376,
35
+ "allocated_bytes_peak": 115696409600,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1278625280,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 119028056064,
46
+ "reserved_bytes_peak": 119028056064,
47
+ "segment_count_current": 1688,
48
+ "segment_count_peak": 1871,
49
+ "timestamp_utc": "2026-08-26T19:13:59.369653Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1278625280,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119028056064,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119028056064,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1688,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:59.681756Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T191359673315Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cc4f8e39145028b10440661429b72d0baa3b625ed880807aeba57acb39ef8785
3
+ size 421723
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77318437376,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115696409600,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77318437376,
8
+ "active_bytes_peak": 115696409600,
9
+ "allocated_bytes_current": 77318437376,
10
+ "allocated_bytes_peak": 115696409600,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1278625280,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 119028056064,
21
+ "reserved_bytes_peak": 119028056064,
22
+ "segment_count_current": 1688,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:59.985251Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77318437376,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115696409600,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77318437376,
33
+ "active_bytes_peak": 115696409600,
34
+ "allocated_bytes_current": 77318437376,
35
+ "allocated_bytes_peak": 115696409600,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1278625280,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 119028056064,
46
+ "reserved_bytes_peak": 119028056064,
47
+ "segment_count_current": 1688,
48
+ "segment_count_peak": 1871,
49
+ "timestamp_utc": "2026-08-26T19:13:59.682961Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1278625280,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119028056064,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119028056064,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1688,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:59.994017Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T191359985624Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8ee5b558e3780976e2049d345a43cd3c121fad600394fb1ca9cdc3230e8c8c23
3
+ size 421723
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77301393920,
3
+ "active_bytes_current_delta": 32780593664,
4
+ "active_bytes_peak": 84822148608,
5
+ "active_bytes_peak_delta": 40276182528,
6
+ "after": {
7
+ "active_bytes_current": 77301393920,
8
+ "active_bytes_peak": 84822148608,
9
+ "allocated_bytes_current": 77301393920,
10
+ "allocated_bytes_peak": 84822148608,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1270502912,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 90550829056,
21
+ "reserved_bytes_peak": 90550829056,
22
+ "segment_count_current": 1692,
23
+ "segment_count_peak": 1692,
24
+ "timestamp_utc": "2026-08-26T19:13:55.493820Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77301393920,
28
+ "allocated_bytes_current_delta": 32780593664,
29
+ "allocated_bytes_peak": 84822148608,
30
+ "allocated_bytes_peak_delta": 40276182528,
31
+ "before": {
32
+ "active_bytes_current": 44520800256,
33
+ "active_bytes_peak": 44545966080,
34
+ "allocated_bytes_current": 44520800256,
35
+ "allocated_bytes_peak": 44545966080,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 421167104,
38
+ "inactive_split_bytes_peak": 1025146880,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T19:13:28.769718Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1270502912,
55
+ "inactive_split_bytes_current_delta": 849335808,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 10484395520,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 90550829056,
68
+ "reserved_bytes_current_delta": 42299555840,
69
+ "reserved_bytes_peak": 90550829056,
70
+ "reserved_bytes_peak_delta": 42299555840,
71
+ "segment_count_current": 1692,
72
+ "segment_count_current_delta": 1071,
73
+ "segment_count_peak": 1692,
74
+ "segment_count_peak_delta": 1071,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:55.534205Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T191355494322Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7546d1ea9f3f0d79614678667f8c94abf0d3be2c7be5c9fc2bcae3181cc78087
3
+ size 2301744
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77301393920,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115689589760,
5
+ "active_bytes_peak_delta": 30867441152,
6
+ "after": {
7
+ "active_bytes_current": 77301393920,
8
+ "active_bytes_peak": 115689589760,
9
+ "allocated_bytes_current": 77301393920,
10
+ "allocated_bytes_peak": 115689589760,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1270502912,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 118057074688,
21
+ "reserved_bytes_peak": 118057074688,
22
+ "segment_count_current": 1871,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:56.330030Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77301393920,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115689589760,
30
+ "allocated_bytes_peak_delta": 30867441152,
31
+ "before": {
32
+ "active_bytes_current": 77301393920,
33
+ "active_bytes_peak": 84822148608,
34
+ "allocated_bytes_current": 77301393920,
35
+ "allocated_bytes_peak": 84822148608,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 1270502912,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 90550829056,
46
+ "reserved_bytes_peak": 90550829056,
47
+ "segment_count_current": 1692,
48
+ "segment_count_peak": 1692,
49
+ "timestamp_utc": "2026-08-26T19:13:55.538393Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1270502912,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 118057074688,
68
+ "reserved_bytes_current_delta": 27506245632,
69
+ "reserved_bytes_peak": 118057074688,
70
+ "reserved_bytes_peak_delta": 27506245632,
71
+ "segment_count_current": 1871,
72
+ "segment_count_current_delta": 179,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 179,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:56.368321Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T191356330452Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1e773a8d295d08939892c810ab7cedf3e6e3476f039cd840179cf2d3625501a4
3
+ size 1827611
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77301393920,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115689851904,
5
+ "active_bytes_peak_delta": 262144,
6
+ "after": {
7
+ "active_bytes_current": 77301393920,
8
+ "active_bytes_peak": 115689851904,
9
+ "allocated_bytes_current": 77301393920,
10
+ "allocated_bytes_peak": 115689851904,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1270502912,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 118057074688,
21
+ "reserved_bytes_peak": 118057074688,
22
+ "segment_count_current": 1871,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:56.687721Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77301393920,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115689851904,
30
+ "allocated_bytes_peak_delta": 262144,
31
+ "before": {
32
+ "active_bytes_current": 77301393920,
33
+ "active_bytes_peak": 115689589760,
34
+ "allocated_bytes_current": 77301393920,
35
+ "allocated_bytes_peak": 115689589760,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 1270502912,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 118057074688,
46
+ "reserved_bytes_peak": 118057074688,
47
+ "segment_count_current": 1871,
48
+ "segment_count_peak": 1871,
49
+ "timestamp_utc": "2026-08-26T19:13:56.372359Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1270502912,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 118057074688,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 118057074688,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1871,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:56.719841Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T191356688117Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:477b4999a2bd545526ad3f57e9928a1b11714caacafeec8d642e934dd20026ee
3
+ size 1818569
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77318437376,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 115696409600,
5
+ "active_bytes_peak_delta": 6557696,
6
+ "after": {
7
+ "active_bytes_current": 77318437376,
8
+ "active_bytes_peak": 115696409600,
9
+ "allocated_bytes_current": 77318437376,
10
+ "allocated_bytes_peak": 115696409600,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1278625280,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 119028056064,
21
+ "reserved_bytes_peak": 119028056064,
22
+ "segment_count_current": 1688,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:59.326767Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77318437376,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 115696409600,
30
+ "allocated_bytes_peak_delta": 6557696,
31
+ "before": {
32
+ "active_bytes_current": 77301393920,
33
+ "active_bytes_peak": 115689851904,
34
+ "allocated_bytes_current": 77301393920,
35
+ "allocated_bytes_peak": 115689851904,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 1270502912,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 118057074688,
46
+ "reserved_bytes_peak": 118057074688,
47
+ "segment_count_current": 1871,
48
+ "segment_count_peak": 1871,
49
+ "timestamp_utc": "2026-08-26T19:13:56.723546Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1278625280,
55
+ "inactive_split_bytes_current_delta": 8122368,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119028056064,
68
+ "reserved_bytes_current_delta": 970981376,
69
+ "reserved_bytes_peak": 119028056064,
70
+ "reserved_bytes_peak_delta": 970981376,
71
+ "segment_count_current": 1688,
72
+ "segment_count_current_delta": -183,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:59.365192Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T191359327210Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e9b11c7add56a973ef2608477b59db90684f3564f425375500655d32885ac3d2
3
+ size 2188967
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77318437376,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115696409600,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77318437376,
8
+ "active_bytes_peak": 115696409600,
9
+ "allocated_bytes_current": 77318437376,
10
+ "allocated_bytes_peak": 115696409600,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1278625280,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 119028056064,
21
+ "reserved_bytes_peak": 119028056064,
22
+ "segment_count_current": 1688,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:59.672754Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77318437376,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115696409600,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77318437376,
33
+ "active_bytes_peak": 115696409600,
34
+ "allocated_bytes_current": 77318437376,
35
+ "allocated_bytes_peak": 115696409600,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 1278625280,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 119028056064,
46
+ "reserved_bytes_peak": 119028056064,
47
+ "segment_count_current": 1688,
48
+ "segment_count_peak": 1871,
49
+ "timestamp_utc": "2026-08-26T19:13:59.370060Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1278625280,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119028056064,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119028056064,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1688,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:59.681579Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T191359673140Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46494cb4f75b92861470d7953cc41c7290a9ef1a4b8a8fbbc30d32287de3c77c
3
+ size 421719
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77318437376,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115696409600,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77318437376,
8
+ "active_bytes_peak": 115696409600,
9
+ "allocated_bytes_current": 77318437376,
10
+ "allocated_bytes_peak": 115696409600,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1278625280,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 119028056064,
21
+ "reserved_bytes_peak": 119028056064,
22
+ "segment_count_current": 1688,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:59.985258Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77318437376,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115696409600,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77318437376,
33
+ "active_bytes_peak": 115696409600,
34
+ "allocated_bytes_current": 77318437376,
35
+ "allocated_bytes_peak": 115696409600,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 1278625280,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 119028056064,
46
+ "reserved_bytes_peak": 119028056064,
47
+ "segment_count_current": 1688,
48
+ "segment_count_peak": 1871,
49
+ "timestamp_utc": "2026-08-26T19:13:59.682681Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1278625280,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119028056064,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119028056064,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1688,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:59.994267Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T191359985640Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a8ed52e7fb1ba42cb3dfb7cfe137845aec85ea0965a531b8b3fcf3e7dd3f7e76
3
+ size 421719
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77301393920,
3
+ "active_bytes_current_delta": 32780593664,
4
+ "active_bytes_peak": 84822148608,
5
+ "active_bytes_peak_delta": 40276182528,
6
+ "after": {
7
+ "active_bytes_current": 77301393920,
8
+ "active_bytes_peak": 84822148608,
9
+ "allocated_bytes_current": 77301393920,
10
+ "allocated_bytes_peak": 84822148608,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1270502912,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 90550829056,
21
+ "reserved_bytes_peak": 90550829056,
22
+ "segment_count_current": 1692,
23
+ "segment_count_peak": 1692,
24
+ "timestamp_utc": "2026-08-26T19:13:55.493830Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77301393920,
28
+ "allocated_bytes_current_delta": 32780593664,
29
+ "allocated_bytes_peak": 84822148608,
30
+ "allocated_bytes_peak_delta": 40276182528,
31
+ "before": {
32
+ "active_bytes_current": 44520800256,
33
+ "active_bytes_peak": 44545966080,
34
+ "allocated_bytes_current": 44520800256,
35
+ "allocated_bytes_peak": 44545966080,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 421167104,
38
+ "inactive_split_bytes_peak": 1025146880,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T19:13:28.769712Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1270502912,
55
+ "inactive_split_bytes_current_delta": 849335808,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 10484395520,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 90550829056,
68
+ "reserved_bytes_current_delta": 42299555840,
69
+ "reserved_bytes_peak": 90550829056,
70
+ "reserved_bytes_peak_delta": 42299555840,
71
+ "segment_count_current": 1692,
72
+ "segment_count_current_delta": 1071,
73
+ "segment_count_peak": 1692,
74
+ "segment_count_peak_delta": 1071,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:55.533437Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T191355494323Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e3a4a4c320ca7cff52964878a75848b1c44cdceca40f57f8cf772c106d7ed2c3
3
+ size 2301744
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77301393920,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115689589760,
5
+ "active_bytes_peak_delta": 30867441152,
6
+ "after": {
7
+ "active_bytes_current": 77301393920,
8
+ "active_bytes_peak": 115689589760,
9
+ "allocated_bytes_current": 77301393920,
10
+ "allocated_bytes_peak": 115689589760,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1270502912,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 118057074688,
21
+ "reserved_bytes_peak": 118057074688,
22
+ "segment_count_current": 1871,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:56.330084Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77301393920,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115689589760,
30
+ "allocated_bytes_peak_delta": 30867441152,
31
+ "before": {
32
+ "active_bytes_current": 77301393920,
33
+ "active_bytes_peak": 84822148608,
34
+ "allocated_bytes_current": 77301393920,
35
+ "allocated_bytes_peak": 84822148608,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1270502912,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 90550829056,
46
+ "reserved_bytes_peak": 90550829056,
47
+ "segment_count_current": 1692,
48
+ "segment_count_peak": 1692,
49
+ "timestamp_utc": "2026-08-26T19:13:55.537623Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1270502912,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 118057074688,
68
+ "reserved_bytes_current_delta": 27506245632,
69
+ "reserved_bytes_peak": 118057074688,
70
+ "reserved_bytes_peak_delta": 27506245632,
71
+ "segment_count_current": 1871,
72
+ "segment_count_current_delta": 179,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 179,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:56.366764Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T191356330503Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b039cf25681691e304df5a1e5986b58c7f46fdda2fa1a9e415857fc5f19d97c1
3
+ size 1827611
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77301393920,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115689851904,
5
+ "active_bytes_peak_delta": 262144,
6
+ "after": {
7
+ "active_bytes_current": 77301393920,
8
+ "active_bytes_peak": 115689851904,
9
+ "allocated_bytes_current": 77301393920,
10
+ "allocated_bytes_peak": 115689851904,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1270502912,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 118057074688,
21
+ "reserved_bytes_peak": 118057074688,
22
+ "segment_count_current": 1871,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:56.687775Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77301393920,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115689851904,
30
+ "allocated_bytes_peak_delta": 262144,
31
+ "before": {
32
+ "active_bytes_current": 77301393920,
33
+ "active_bytes_peak": 115689589760,
34
+ "allocated_bytes_current": 77301393920,
35
+ "allocated_bytes_peak": 115689589760,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1270502912,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 118057074688,
46
+ "reserved_bytes_peak": 118057074688,
47
+ "segment_count_current": 1871,
48
+ "segment_count_peak": 1871,
49
+ "timestamp_utc": "2026-08-26T19:13:56.370614Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1270502912,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 118057074688,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 118057074688,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1871,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:56.718676Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T191356688171Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:83a6c52cd07fca4b9503d38d0e115d42d3d8a779f0d8fd7be8553d584bcce60e
3
+ size 1818569
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77318437376,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 115696409600,
5
+ "active_bytes_peak_delta": 6557696,
6
+ "after": {
7
+ "active_bytes_current": 77318437376,
8
+ "active_bytes_peak": 115696409600,
9
+ "allocated_bytes_current": 77318437376,
10
+ "allocated_bytes_peak": 115696409600,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1278625280,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 119028056064,
21
+ "reserved_bytes_peak": 119028056064,
22
+ "segment_count_current": 1688,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:59.326845Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77318437376,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 115696409600,
30
+ "allocated_bytes_peak_delta": 6557696,
31
+ "before": {
32
+ "active_bytes_current": 77301393920,
33
+ "active_bytes_peak": 115689851904,
34
+ "allocated_bytes_current": 77301393920,
35
+ "allocated_bytes_peak": 115689851904,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1270502912,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 118057074688,
46
+ "reserved_bytes_peak": 118057074688,
47
+ "segment_count_current": 1871,
48
+ "segment_count_peak": 1871,
49
+ "timestamp_utc": "2026-08-26T19:13:56.722150Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1278625280,
55
+ "inactive_split_bytes_current_delta": 8122368,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119028056064,
68
+ "reserved_bytes_current_delta": 970981376,
69
+ "reserved_bytes_peak": 119028056064,
70
+ "reserved_bytes_peak_delta": 970981376,
71
+ "segment_count_current": 1688,
72
+ "segment_count_current_delta": -183,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:59.365437Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T191359327349Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f769ed3ca5a81059504fdac9122eee3bc84daf6b28b36b3f4458951c58e6738
3
+ size 2188969
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77318437376,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115696409600,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77318437376,
8
+ "active_bytes_peak": 115696409600,
9
+ "allocated_bytes_current": 77318437376,
10
+ "allocated_bytes_peak": 115696409600,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1278625280,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 119028056064,
21
+ "reserved_bytes_peak": 119028056064,
22
+ "segment_count_current": 1688,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:59.672803Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77318437376,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115696409600,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77318437376,
33
+ "active_bytes_peak": 115696409600,
34
+ "allocated_bytes_current": 77318437376,
35
+ "allocated_bytes_peak": 115696409600,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1278625280,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 119028056064,
46
+ "reserved_bytes_peak": 119028056064,
47
+ "segment_count_current": 1688,
48
+ "segment_count_peak": 1871,
49
+ "timestamp_utc": "2026-08-26T19:13:59.369446Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1278625280,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119028056064,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119028056064,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1688,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:59.681882Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T191359673272Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2530246e9f8fdaeddf81c262d7186b4d97c1e7010a93d79ebdef32546b960e77
3
+ size 421721
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T191359985689Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77318437376,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 115696409600,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77318437376,
8
+ "active_bytes_peak": 115696409600,
9
+ "allocated_bytes_current": 77318437376,
10
+ "allocated_bytes_peak": 115696409600,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1278625280,
13
+ "inactive_split_bytes_peak": 11509542400,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 119028056064,
21
+ "reserved_bytes_peak": 119028056064,
22
+ "segment_count_current": 1688,
23
+ "segment_count_peak": 1871,
24
+ "timestamp_utc": "2026-08-26T19:13:59.985310Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77318437376,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 115696409600,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77318437376,
33
+ "active_bytes_peak": 115696409600,
34
+ "allocated_bytes_current": 77318437376,
35
+ "allocated_bytes_peak": 115696409600,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1278625280,
38
+ "inactive_split_bytes_peak": 11509542400,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 119028056064,
46
+ "reserved_bytes_peak": 119028056064,
47
+ "segment_count_current": 1688,
48
+ "segment_count_peak": 1871,
49
+ "timestamp_utc": "2026-08-26T19:13:59.683052Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1278625280,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 11509542400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119028056064,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119028056064,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1688,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1871,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T191359985689Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:13:59.994216Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }