AbhishekGhosh1998 commited on
Commit
bc3a4bb
·
verified ·
1 Parent(s): 756e47c

granularity results ultraep/tierbmem @ 2026-08-26T19:21:12Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 3)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.json +79 -0
  2. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.pickle +3 -0
  3. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.json +79 -0
  4. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.pickle +3 -0
  5. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.json +79 -0
  6. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.pickle +3 -0
  7. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.json +79 -0
  8. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.pickle +3 -0
  9. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.json +79 -0
  10. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.pickle +3 -0
  11. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.json +79 -0
  12. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.pickle +3 -0
  13. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.json +79 -0
  14. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.pickle +3 -0
  15. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.json +79 -0
  16. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.pickle +3 -0
  17. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.json +79 -0
  18. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.pickle +3 -0
  19. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.json +79 -0
  20. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.pickle +3 -0
  21. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.json +79 -0
  22. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.pickle +3 -0
  23. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.json +79 -0
  24. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.pickle +3 -0
  25. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.json +79 -0
  26. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.pickle +3 -0
  27. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.json +79 -0
  28. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.pickle +3 -0
  29. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.json +79 -0
  30. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.pickle +3 -0
  31. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.json +79 -0
  32. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.pickle +3 -0
  33. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.json +79 -0
  34. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.pickle +3 -0
  35. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.json +79 -0
  36. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.pickle +3 -0
  37. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.json +79 -0
  38. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.pickle +3 -0
  39. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.json +79 -0
  40. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.pickle +3 -0
  41. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.json +79 -0
  42. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.pickle +3 -0
  43. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.json +79 -0
  44. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.pickle +3 -0
  45. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.json +79 -0
  46. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.pickle +3 -0
  47. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.json +79 -0
  48. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.pickle +3 -0
  49. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.json +79 -0
  50. ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.pickle +3 -0
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75901254656,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113456552448,
5
+ "active_bytes_peak_delta": 30836036096,
6
+ "after": {
7
+ "active_bytes_current": 75901254656,
8
+ "active_bytes_peak": 113456552448,
9
+ "allocated_bytes_current": 75901254656,
10
+ "allocated_bytes_peak": 113456552448,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 5912839168,
13
+ "inactive_split_bytes_peak": 14139202560,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 113873256448,
21
+ "reserved_bytes_peak": 113873256448,
22
+ "segment_count_current": 2836,
23
+ "segment_count_peak": 2836,
24
+ "timestamp_utc": "2026-08-26T19:17:37.348676Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75901254656,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113456552448,
30
+ "allocated_bytes_peak_delta": 30836036096,
31
+ "before": {
32
+ "active_bytes_current": 75901254656,
33
+ "active_bytes_peak": 82620516352,
34
+ "allocated_bytes_current": 75901254656,
35
+ "allocated_bytes_peak": 82620516352,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 5912839168,
38
+ "inactive_split_bytes_peak": 14139202560,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 83022053376,
46
+ "reserved_bytes_peak": 83022053376,
47
+ "segment_count_current": 2125,
48
+ "segment_count_peak": 2125,
49
+ "timestamp_utc": "2026-08-26T19:17:36.067939Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5912839168,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 14139202560,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113873256448,
68
+ "reserved_bytes_current_delta": 30851203072,
69
+ "reserved_bytes_peak": 113873256448,
70
+ "reserved_bytes_peak_delta": 30851203072,
71
+ "segment_count_current": 2836,
72
+ "segment_count_current_delta": 711,
73
+ "segment_count_peak": 2836,
74
+ "segment_count_peak_delta": 711,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:37.476498Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191737349095Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd22d0b31afd0255b04015d9a7b4e9df333063e40f147439b23bec488ad4cdcf
3
+ size 7113911
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75901254656,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113495366144,
5
+ "active_bytes_peak_delta": 38813696,
6
+ "after": {
7
+ "active_bytes_current": 75901254656,
8
+ "active_bytes_peak": 113495366144,
9
+ "allocated_bytes_current": 75901254656,
10
+ "allocated_bytes_peak": 113495366144,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 5912839168,
13
+ "inactive_split_bytes_peak": 14139202560,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 113873256448,
21
+ "reserved_bytes_peak": 113873256448,
22
+ "segment_count_current": 2836,
23
+ "segment_count_peak": 2836,
24
+ "timestamp_utc": "2026-08-26T19:17:38.517239Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75901254656,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113495366144,
30
+ "allocated_bytes_peak_delta": 38813696,
31
+ "before": {
32
+ "active_bytes_current": 75901254656,
33
+ "active_bytes_peak": 113456552448,
34
+ "allocated_bytes_current": 75901254656,
35
+ "allocated_bytes_peak": 113456552448,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 5912839168,
38
+ "inactive_split_bytes_peak": 14139202560,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 113873256448,
46
+ "reserved_bytes_peak": 113873256448,
47
+ "segment_count_current": 2836,
48
+ "segment_count_peak": 2836,
49
+ "timestamp_utc": "2026-08-26T19:17:37.492292Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5912839168,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 14139202560,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113873256448,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 113873256448,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2836,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2836,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:38.647254Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191738517674Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:982257ddd1e947c99f2570403f65f61ca2f011471cd1462fb9c5af7cffa8c375
3
+ size 6892313
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75918298112,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 113495366144,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75918298112,
8
+ "active_bytes_peak": 113495366144,
9
+ "allocated_bytes_current": 75918298112,
10
+ "allocated_bytes_peak": 113495366144,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 5950321664,
13
+ "inactive_split_bytes_peak": 15467503104,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 119042736128,
21
+ "reserved_bytes_peak": 119042736128,
22
+ "segment_count_current": 3126,
23
+ "segment_count_peak": 3126,
24
+ "timestamp_utc": "2026-08-26T19:17:41.821331Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75918298112,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 113495366144,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75901254656,
33
+ "active_bytes_peak": 113495366144,
34
+ "allocated_bytes_current": 75901254656,
35
+ "allocated_bytes_peak": 113495366144,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 5912839168,
38
+ "inactive_split_bytes_peak": 14139202560,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 113873256448,
46
+ "reserved_bytes_peak": 113873256448,
47
+ "segment_count_current": 2836,
48
+ "segment_count_peak": 2836,
49
+ "timestamp_utc": "2026-08-26T19:17:38.662900Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5950321664,
55
+ "inactive_split_bytes_current_delta": 37482496,
56
+ "inactive_split_bytes_peak": 15467503104,
57
+ "inactive_split_bytes_peak_delta": 1328300544,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119042736128,
68
+ "reserved_bytes_current_delta": 5169479680,
69
+ "reserved_bytes_peak": 119042736128,
70
+ "reserved_bytes_peak_delta": 5169479680,
71
+ "segment_count_current": 3126,
72
+ "segment_count_current_delta": 290,
73
+ "segment_count_peak": 3126,
74
+ "segment_count_peak_delta": 290,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:41.960602Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191741821837Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7488c72f8849c7acc6825f47d531d1216b71cfd482bf98b9a9258ad406183644
3
+ size 7508905
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75918298112,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113495366144,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75918298112,
8
+ "active_bytes_peak": 113495366144,
9
+ "allocated_bytes_current": 75918298112,
10
+ "allocated_bytes_peak": 113495366144,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 5950321664,
13
+ "inactive_split_bytes_peak": 15467503104,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 119042736128,
21
+ "reserved_bytes_peak": 119042736128,
22
+ "segment_count_current": 3126,
23
+ "segment_count_peak": 3126,
24
+ "timestamp_utc": "2026-08-26T19:17:42.351495Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75918298112,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113495366144,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75918298112,
33
+ "active_bytes_peak": 113495366144,
34
+ "allocated_bytes_current": 75918298112,
35
+ "allocated_bytes_peak": 113495366144,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 5950321664,
38
+ "inactive_split_bytes_peak": 15467503104,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 119042736128,
46
+ "reserved_bytes_peak": 119042736128,
47
+ "segment_count_current": 3126,
48
+ "segment_count_peak": 3126,
49
+ "timestamp_utc": "2026-08-26T19:17:41.977258Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5950321664,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 15467503104,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119042736128,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119042736128,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3126,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3126,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:42.371970Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191742351912Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92651b2e399d31cbb87b64130deeff9d01389b8e0872afd4185c48f5b885c6f6
3
+ size 902755
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75918298112,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113495366144,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75918298112,
8
+ "active_bytes_peak": 113495366144,
9
+ "allocated_bytes_current": 75918298112,
10
+ "allocated_bytes_peak": 113495366144,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 5950321664,
13
+ "inactive_split_bytes_peak": 15467503104,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 119042736128,
21
+ "reserved_bytes_peak": 119042736128,
22
+ "segment_count_current": 3126,
23
+ "segment_count_peak": 3126,
24
+ "timestamp_utc": "2026-08-26T19:17:42.732476Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75918298112,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113495366144,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75918298112,
33
+ "active_bytes_peak": 113495366144,
34
+ "allocated_bytes_current": 75918298112,
35
+ "allocated_bytes_peak": 113495366144,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 5950321664,
38
+ "inactive_split_bytes_peak": 15467503104,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 119042736128,
46
+ "reserved_bytes_peak": 119042736128,
47
+ "segment_count_current": 3126,
48
+ "segment_count_peak": 3126,
49
+ "timestamp_utc": "2026-08-26T19:17:42.373557Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5950321664,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 15467503104,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119042736128,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119042736128,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3126,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3126,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:42.751054Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191742732859Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:87358302e71f8050e4d1f4e0c8b125bcd8f2e0539bc811ebd33be2336b197cb5
3
+ size 902755
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75883125760,
3
+ "active_bytes_current_delta": 32328931328,
4
+ "active_bytes_peak": 82624266240,
5
+ "active_bytes_peak_delta": 39063780352,
6
+ "after": {
7
+ "active_bytes_current": 75883125760,
8
+ "active_bytes_peak": 82624266240,
9
+ "allocated_bytes_current": 75883125760,
10
+ "allocated_bytes_peak": 82624266240,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 5901607936,
13
+ "inactive_split_bytes_peak": 14149293568,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 82996887552,
21
+ "reserved_bytes_peak": 82996887552,
22
+ "segment_count_current": 2112,
23
+ "segment_count_peak": 2112,
24
+ "timestamp_utc": "2026-08-26T19:17:35.910227Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75883125760,
28
+ "allocated_bytes_current_delta": 32328931328,
29
+ "allocated_bytes_peak": 82624266240,
30
+ "allocated_bytes_peak_delta": 39063780352,
31
+ "before": {
32
+ "active_bytes_current": 43554194432,
33
+ "active_bytes_peak": 43560485888,
34
+ "allocated_bytes_current": 43554194432,
35
+ "allocated_bytes_peak": 43560485888,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1255652352,
38
+ "inactive_split_bytes_peak": 1612261376,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 44835012608,
46
+ "reserved_bytes_peak": 44835012608,
47
+ "segment_count_current": 883,
48
+ "segment_count_peak": 883,
49
+ "timestamp_utc": "2026-08-26T19:17:09.289391Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5901607936,
55
+ "inactive_split_bytes_current_delta": 4645955584,
56
+ "inactive_split_bytes_peak": 14149293568,
57
+ "inactive_split_bytes_peak_delta": 12537032192,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 82996887552,
68
+ "reserved_bytes_current_delta": 38161874944,
69
+ "reserved_bytes_peak": 82996887552,
70
+ "reserved_bytes_peak_delta": 38161874944,
71
+ "segment_count_current": 2112,
72
+ "segment_count_current_delta": 1229,
73
+ "segment_count_peak": 2112,
74
+ "segment_count_peak_delta": 1229,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:36.047916Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191735910683Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:feb161bcba0cd15af96cbddd476e5bbbbcc7702ae29e002495a161d94622b2b6
3
+ size 7001386
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75883125760,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113455617536,
5
+ "active_bytes_peak_delta": 30831351296,
6
+ "after": {
7
+ "active_bytes_current": 75883125760,
8
+ "active_bytes_peak": 113455617536,
9
+ "allocated_bytes_current": 75883125760,
10
+ "allocated_bytes_peak": 113455617536,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 5901607936,
13
+ "inactive_split_bytes_peak": 14149293568,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 113841799168,
21
+ "reserved_bytes_peak": 113841799168,
22
+ "segment_count_current": 2812,
23
+ "segment_count_peak": 2812,
24
+ "timestamp_utc": "2026-08-26T19:17:37.348673Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75883125760,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113455617536,
30
+ "allocated_bytes_peak_delta": 30831351296,
31
+ "before": {
32
+ "active_bytes_current": 75883125760,
33
+ "active_bytes_peak": 82624266240,
34
+ "allocated_bytes_current": 75883125760,
35
+ "allocated_bytes_peak": 82624266240,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 5901607936,
38
+ "inactive_split_bytes_peak": 14149293568,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 82996887552,
46
+ "reserved_bytes_peak": 82996887552,
47
+ "segment_count_current": 2112,
48
+ "segment_count_peak": 2112,
49
+ "timestamp_utc": "2026-08-26T19:17:36.069015Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5901607936,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 14149293568,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113841799168,
68
+ "reserved_bytes_current_delta": 30844911616,
69
+ "reserved_bytes_peak": 113841799168,
70
+ "reserved_bytes_peak_delta": 30844911616,
71
+ "segment_count_current": 2812,
72
+ "segment_count_current_delta": 700,
73
+ "segment_count_peak": 2812,
74
+ "segment_count_peak_delta": 700,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:37.477367Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191737349096Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb407cb73c7f6b58ae60b7500ac00821299baa5e1cabe4319498795df9c8c375
3
+ size 6492477
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75883125760,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113498658304,
5
+ "active_bytes_peak_delta": 43040768,
6
+ "after": {
7
+ "active_bytes_current": 75883125760,
8
+ "active_bytes_peak": 113498658304,
9
+ "allocated_bytes_current": 75883125760,
10
+ "allocated_bytes_peak": 113498658304,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 5901607936,
13
+ "inactive_split_bytes_peak": 14149293568,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 113841799168,
21
+ "reserved_bytes_peak": 113841799168,
22
+ "segment_count_current": 2812,
23
+ "segment_count_peak": 2812,
24
+ "timestamp_utc": "2026-08-26T19:17:38.517252Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75883125760,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113498658304,
30
+ "allocated_bytes_peak_delta": 43040768,
31
+ "before": {
32
+ "active_bytes_current": 75883125760,
33
+ "active_bytes_peak": 113455617536,
34
+ "allocated_bytes_current": 75883125760,
35
+ "allocated_bytes_peak": 113455617536,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 5901607936,
38
+ "inactive_split_bytes_peak": 14149293568,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 113841799168,
46
+ "reserved_bytes_peak": 113841799168,
47
+ "segment_count_current": 2812,
48
+ "segment_count_peak": 2812,
49
+ "timestamp_utc": "2026-08-26T19:17:37.492866Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5901607936,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 14149293568,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113841799168,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 113841799168,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2812,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2812,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:38.646112Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191738517674Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f0b8e83304e96a7c1cdebf66dfd35177382faa8f65936acf9ca3f1ab51cd3c30
3
+ size 6457970
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75900169216,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 113498658304,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75900169216,
8
+ "active_bytes_peak": 113498658304,
9
+ "allocated_bytes_current": 75900169216,
10
+ "allocated_bytes_peak": 113498658304,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 5939090432,
13
+ "inactive_split_bytes_peak": 15456271872,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 119013376000,
21
+ "reserved_bytes_peak": 119013376000,
22
+ "segment_count_current": 3111,
23
+ "segment_count_peak": 3111,
24
+ "timestamp_utc": "2026-08-26T19:17:41.821366Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75900169216,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 113498658304,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75883125760,
33
+ "active_bytes_peak": 113498658304,
34
+ "allocated_bytes_current": 75883125760,
35
+ "allocated_bytes_peak": 113498658304,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 5901607936,
38
+ "inactive_split_bytes_peak": 14149293568,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 113841799168,
46
+ "reserved_bytes_peak": 113841799168,
47
+ "segment_count_current": 2812,
48
+ "segment_count_peak": 2812,
49
+ "timestamp_utc": "2026-08-26T19:17:38.661216Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5939090432,
55
+ "inactive_split_bytes_current_delta": 37482496,
56
+ "inactive_split_bytes_peak": 15456271872,
57
+ "inactive_split_bytes_peak_delta": 1306978304,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119013376000,
68
+ "reserved_bytes_current_delta": 5171576832,
69
+ "reserved_bytes_peak": 119013376000,
70
+ "reserved_bytes_peak_delta": 5171576832,
71
+ "segment_count_current": 3111,
72
+ "segment_count_current_delta": 299,
73
+ "segment_count_peak": 3111,
74
+ "segment_count_peak_delta": 299,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:41.961683Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191741821886Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:10ba6b6f670b2f05f5b3a2ba8a6b103fc236a236a656023fe21288c42e3b6458
3
+ size 7491891
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75900169216,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113498658304,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75900169216,
8
+ "active_bytes_peak": 113498658304,
9
+ "allocated_bytes_current": 75900169216,
10
+ "allocated_bytes_peak": 113498658304,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 5939090432,
13
+ "inactive_split_bytes_peak": 15456271872,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 119013376000,
21
+ "reserved_bytes_peak": 119013376000,
22
+ "segment_count_current": 3111,
23
+ "segment_count_peak": 3111,
24
+ "timestamp_utc": "2026-08-26T19:17:42.351498Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75900169216,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113498658304,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75900169216,
33
+ "active_bytes_peak": 113498658304,
34
+ "allocated_bytes_current": 75900169216,
35
+ "allocated_bytes_peak": 113498658304,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 5939090432,
38
+ "inactive_split_bytes_peak": 15456271872,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 119013376000,
46
+ "reserved_bytes_peak": 119013376000,
47
+ "segment_count_current": 3111,
48
+ "segment_count_peak": 3111,
49
+ "timestamp_utc": "2026-08-26T19:17:41.978110Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5939090432,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 15456271872,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119013376000,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119013376000,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3111,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3111,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:42.372147Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191742351940Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e4e61955093733498e6a3566b7a8c688e2dc8fe535215b82c6c6ecefde0af079
3
+ size 886237
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75900169216,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113498658304,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75900169216,
8
+ "active_bytes_peak": 113498658304,
9
+ "allocated_bytes_current": 75900169216,
10
+ "allocated_bytes_peak": 113498658304,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 5939090432,
13
+ "inactive_split_bytes_peak": 15456271872,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 119013376000,
21
+ "reserved_bytes_peak": 119013376000,
22
+ "segment_count_current": 3111,
23
+ "segment_count_peak": 3111,
24
+ "timestamp_utc": "2026-08-26T19:17:42.732501Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75900169216,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113498658304,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75900169216,
33
+ "active_bytes_peak": 113498658304,
34
+ "allocated_bytes_current": 75900169216,
35
+ "allocated_bytes_peak": 113498658304,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 5939090432,
38
+ "inactive_split_bytes_peak": 15456271872,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 119013376000,
46
+ "reserved_bytes_peak": 119013376000,
47
+ "segment_count_current": 3111,
48
+ "segment_count_peak": 3111,
49
+ "timestamp_utc": "2026-08-26T19:17:42.373743Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5939090432,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 15456271872,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119013376000,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119013376000,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3111,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3111,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:42.751076Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191742732884Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05dc5dc4571eeeccdfe5f49c6101cb5aa84f61118e3598b3855f030e6e837266
3
+ size 886237
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75906373632,
3
+ "active_bytes_current_delta": 32348899840,
4
+ "active_bytes_peak": 82622775808,
5
+ "active_bytes_peak_delta": 39059010560,
6
+ "after": {
7
+ "active_bytes_current": 75906373632,
8
+ "active_bytes_peak": 82622775808,
9
+ "allocated_bytes_current": 75906373632,
10
+ "allocated_bytes_peak": 82622775808,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 5924497408,
13
+ "inactive_split_bytes_peak": 14129433600,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 83022053376,
21
+ "reserved_bytes_peak": 83022053376,
22
+ "segment_count_current": 2127,
23
+ "segment_count_peak": 2127,
24
+ "timestamp_utc": "2026-08-26T19:17:35.910251Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75906373632,
28
+ "allocated_bytes_current_delta": 32348899840,
29
+ "allocated_bytes_peak": 82622775808,
30
+ "allocated_bytes_peak_delta": 39059010560,
31
+ "before": {
32
+ "active_bytes_current": 43557473792,
33
+ "active_bytes_peak": 43563765248,
34
+ "allocated_bytes_current": 43557473792,
35
+ "allocated_bytes_peak": 43563765248,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 1223012864,
38
+ "inactive_split_bytes_peak": 1612261376,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 44795166720,
46
+ "reserved_bytes_peak": 44795166720,
47
+ "segment_count_current": 887,
48
+ "segment_count_peak": 887,
49
+ "timestamp_utc": "2026-08-26T19:17:09.289271Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5924497408,
55
+ "inactive_split_bytes_current_delta": 4701484544,
56
+ "inactive_split_bytes_peak": 14129433600,
57
+ "inactive_split_bytes_peak_delta": 12517172224,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 83022053376,
68
+ "reserved_bytes_current_delta": 38226886656,
69
+ "reserved_bytes_peak": 83022053376,
70
+ "reserved_bytes_peak_delta": 38226886656,
71
+ "segment_count_current": 2127,
72
+ "segment_count_current_delta": 1240,
73
+ "segment_count_peak": 2127,
74
+ "segment_count_peak_delta": 1240,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:36.045282Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191735910704Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b3a54f0eb6432a48d3702bccd3e8e9f0d611e5cb2449b369de98b9d04d56876
3
+ size 7018235
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75906373632,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113460826112,
5
+ "active_bytes_peak_delta": 30838050304,
6
+ "after": {
7
+ "active_bytes_current": 75906373632,
8
+ "active_bytes_peak": 113460826112,
9
+ "allocated_bytes_current": 75906373632,
10
+ "allocated_bytes_peak": 113460826112,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 5926594560,
13
+ "inactive_split_bytes_peak": 14129433600,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 113862770688,
21
+ "reserved_bytes_peak": 113862770688,
22
+ "segment_count_current": 2841,
23
+ "segment_count_peak": 2841,
24
+ "timestamp_utc": "2026-08-26T19:17:37.348664Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75906373632,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113460826112,
30
+ "allocated_bytes_peak_delta": 30838050304,
31
+ "before": {
32
+ "active_bytes_current": 75906373632,
33
+ "active_bytes_peak": 82622775808,
34
+ "allocated_bytes_current": 75906373632,
35
+ "allocated_bytes_peak": 82622775808,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 5924497408,
38
+ "inactive_split_bytes_peak": 14129433600,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 83022053376,
46
+ "reserved_bytes_peak": 83022053376,
47
+ "segment_count_current": 2127,
48
+ "segment_count_peak": 2127,
49
+ "timestamp_utc": "2026-08-26T19:17:36.066185Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5926594560,
55
+ "inactive_split_bytes_current_delta": 2097152,
56
+ "inactive_split_bytes_peak": 14129433600,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113862770688,
68
+ "reserved_bytes_current_delta": 30840717312,
69
+ "reserved_bytes_peak": 113862770688,
70
+ "reserved_bytes_peak_delta": 30840717312,
71
+ "segment_count_current": 2841,
72
+ "segment_count_current_delta": 714,
73
+ "segment_count_peak": 2841,
74
+ "segment_count_peak_delta": 714,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:37.477356Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191737349091Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b66639ce8538a0bb11755f6ccfcdb7faa6fabbb3ce191fd07e1245df4e09489c
3
+ size 6806262
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75906373632,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113474031616,
5
+ "active_bytes_peak_delta": 13205504,
6
+ "after": {
7
+ "active_bytes_current": 75906373632,
8
+ "active_bytes_peak": 113474031616,
9
+ "allocated_bytes_current": 75906373632,
10
+ "allocated_bytes_peak": 113474031616,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 5924497408,
13
+ "inactive_split_bytes_peak": 14129433600,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 113862770688,
21
+ "reserved_bytes_peak": 113862770688,
22
+ "segment_count_current": 2841,
23
+ "segment_count_peak": 2841,
24
+ "timestamp_utc": "2026-08-26T19:17:38.517239Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75906373632,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113474031616,
30
+ "allocated_bytes_peak_delta": 13205504,
31
+ "before": {
32
+ "active_bytes_current": 75906373632,
33
+ "active_bytes_peak": 113460826112,
34
+ "allocated_bytes_current": 75906373632,
35
+ "allocated_bytes_peak": 113460826112,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 5926594560,
38
+ "inactive_split_bytes_peak": 14129433600,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 113862770688,
46
+ "reserved_bytes_peak": 113862770688,
47
+ "segment_count_current": 2841,
48
+ "segment_count_peak": 2841,
49
+ "timestamp_utc": "2026-08-26T19:17:37.492920Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5924497408,
55
+ "inactive_split_bytes_current_delta": -2097152,
56
+ "inactive_split_bytes_peak": 14129433600,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113862770688,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 113862770688,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2841,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2841,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:38.645475Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191738517674Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e8efaadcf1709cdc1b13914564edcdc3e5346f5010e53e50c204a57c22acdbd1
3
+ size 6771814
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75923417088,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 113474031616,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75923417088,
8
+ "active_bytes_peak": 113474031616,
9
+ "allocated_bytes_current": 75923417088,
10
+ "allocated_bytes_peak": 113474031616,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 5961979904,
13
+ "inactive_split_bytes_peak": 15479161344,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 119059513344,
21
+ "reserved_bytes_peak": 119059513344,
22
+ "segment_count_current": 3136,
23
+ "segment_count_peak": 3136,
24
+ "timestamp_utc": "2026-08-26T19:17:41.821350Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75923417088,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 113474031616,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75906373632,
33
+ "active_bytes_peak": 113474031616,
34
+ "allocated_bytes_current": 75906373632,
35
+ "allocated_bytes_peak": 113474031616,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 5924497408,
38
+ "inactive_split_bytes_peak": 14129433600,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 113862770688,
46
+ "reserved_bytes_peak": 113862770688,
47
+ "segment_count_current": 2841,
48
+ "segment_count_peak": 2841,
49
+ "timestamp_utc": "2026-08-26T19:17:38.660609Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5961979904,
55
+ "inactive_split_bytes_current_delta": 37482496,
56
+ "inactive_split_bytes_peak": 15479161344,
57
+ "inactive_split_bytes_peak_delta": 1349727744,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119059513344,
68
+ "reserved_bytes_current_delta": 5196742656,
69
+ "reserved_bytes_peak": 119059513344,
70
+ "reserved_bytes_peak_delta": 5196742656,
71
+ "segment_count_current": 3136,
72
+ "segment_count_current_delta": 295,
73
+ "segment_count_peak": 3136,
74
+ "segment_count_peak_delta": 295,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:41.963989Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191741821918Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cec3c71cc3c2b7963ee074aebfd39e9437869d8a12b22312c2c2b052a35902bc
3
+ size 7466230
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75923417088,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113474031616,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75923417088,
8
+ "active_bytes_peak": 113474031616,
9
+ "allocated_bytes_current": 75923417088,
10
+ "allocated_bytes_peak": 113474031616,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 5961979904,
13
+ "inactive_split_bytes_peak": 15479161344,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 119059513344,
21
+ "reserved_bytes_peak": 119059513344,
22
+ "segment_count_current": 3136,
23
+ "segment_count_peak": 3136,
24
+ "timestamp_utc": "2026-08-26T19:17:42.351569Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75923417088,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113474031616,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75923417088,
33
+ "active_bytes_peak": 113474031616,
34
+ "allocated_bytes_current": 75923417088,
35
+ "allocated_bytes_peak": 113474031616,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 5961979904,
38
+ "inactive_split_bytes_peak": 15479161344,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 119059513344,
46
+ "reserved_bytes_peak": 119059513344,
47
+ "segment_count_current": 3136,
48
+ "segment_count_peak": 3136,
49
+ "timestamp_utc": "2026-08-26T19:17:41.980683Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5961979904,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 15479161344,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119059513344,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119059513344,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3136,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3136,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:42.372512Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191742352017Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a8c8275abfb2f2937daa8e5310c30653f639027b80d6cd792edc076f091751a4
3
+ size 901220
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75923417088,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113474031616,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75923417088,
8
+ "active_bytes_peak": 113474031616,
9
+ "allocated_bytes_current": 75923417088,
10
+ "allocated_bytes_peak": 113474031616,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 5961979904,
13
+ "inactive_split_bytes_peak": 15479161344,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 119059513344,
21
+ "reserved_bytes_peak": 119059513344,
22
+ "segment_count_current": 3136,
23
+ "segment_count_peak": 3136,
24
+ "timestamp_utc": "2026-08-26T19:17:42.732501Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75923417088,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113474031616,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75923417088,
33
+ "active_bytes_peak": 113474031616,
34
+ "allocated_bytes_current": 75923417088,
35
+ "allocated_bytes_peak": 113474031616,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 5961979904,
38
+ "inactive_split_bytes_peak": 15479161344,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 119059513344,
46
+ "reserved_bytes_peak": 119059513344,
47
+ "segment_count_current": 3136,
48
+ "segment_count_peak": 3136,
49
+ "timestamp_utc": "2026-08-26T19:17:42.374142Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5961979904,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 15479161344,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119059513344,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119059513344,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3136,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3136,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:42.751336Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T191742732888Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70cf0db8ddacad01116ec4c48b801841bbb386f001903ce288f1f9829c621bff
3
+ size 901220
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75889472000,
3
+ "active_bytes_current_delta": 32338143232,
4
+ "active_bytes_peak": 82618485248,
5
+ "active_bytes_peak_delta": 39060865024,
6
+ "after": {
7
+ "active_bytes_current": 75889472000,
8
+ "active_bytes_peak": 82618485248,
9
+ "allocated_bytes_current": 75889472000,
10
+ "allocated_bytes_peak": 82618485248,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 5903650304,
13
+ "inactive_split_bytes_peak": 14135910400,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 83001081856,
21
+ "reserved_bytes_peak": 83001081856,
22
+ "segment_count_current": 2127,
23
+ "segment_count_peak": 2127,
24
+ "timestamp_utc": "2026-08-26T19:17:35.910248Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75889472000,
28
+ "allocated_bytes_current_delta": 32338143232,
29
+ "allocated_bytes_peak": 82618485248,
30
+ "allocated_bytes_peak_delta": 39060865024,
31
+ "before": {
32
+ "active_bytes_current": 43551328768,
33
+ "active_bytes_peak": 43557620224,
34
+ "allocated_bytes_current": 43551328768,
35
+ "allocated_bytes_peak": 43557620224,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 1254323712,
38
+ "inactive_split_bytes_peak": 1612261376,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 44820332544,
46
+ "reserved_bytes_peak": 44820332544,
47
+ "segment_count_current": 889,
48
+ "segment_count_peak": 889,
49
+ "timestamp_utc": "2026-08-26T19:17:09.289064Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5903650304,
55
+ "inactive_split_bytes_current_delta": 4649326592,
56
+ "inactive_split_bytes_peak": 14135910400,
57
+ "inactive_split_bytes_peak_delta": 12523649024,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 83001081856,
68
+ "reserved_bytes_current_delta": 38180749312,
69
+ "reserved_bytes_peak": 83001081856,
70
+ "reserved_bytes_peak_delta": 38180749312,
71
+ "segment_count_current": 2127,
72
+ "segment_count_current_delta": 1238,
73
+ "segment_count_peak": 2127,
74
+ "segment_count_peak_delta": 1238,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:36.047700Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T191735910710Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:938c759b9247f4d8c27dc09ea87f60048674572c7c241818b3c846af4811b62d
3
+ size 7019264
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75889472000,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113449531904,
5
+ "active_bytes_peak_delta": 30831046656,
6
+ "after": {
7
+ "active_bytes_current": 75889472000,
8
+ "active_bytes_peak": 113449531904,
9
+ "allocated_bytes_current": 75889472000,
10
+ "allocated_bytes_peak": 113449531904,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 5903650304,
13
+ "inactive_split_bytes_peak": 14135910400,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 113894227968,
21
+ "reserved_bytes_peak": 113894227968,
22
+ "segment_count_current": 2840,
23
+ "segment_count_peak": 2840,
24
+ "timestamp_utc": "2026-08-26T19:17:37.348755Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75889472000,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113449531904,
30
+ "allocated_bytes_peak_delta": 30831046656,
31
+ "before": {
32
+ "active_bytes_current": 75889472000,
33
+ "active_bytes_peak": 82618485248,
34
+ "allocated_bytes_current": 75889472000,
35
+ "allocated_bytes_peak": 82618485248,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 5903650304,
38
+ "inactive_split_bytes_peak": 14135910400,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 83001081856,
46
+ "reserved_bytes_peak": 83001081856,
47
+ "segment_count_current": 2127,
48
+ "segment_count_peak": 2127,
49
+ "timestamp_utc": "2026-08-26T19:17:36.068725Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5903650304,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 14135910400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113894227968,
68
+ "reserved_bytes_current_delta": 30893146112,
69
+ "reserved_bytes_peak": 113894227968,
70
+ "reserved_bytes_peak_delta": 30893146112,
71
+ "segment_count_current": 2840,
72
+ "segment_count_current_delta": 713,
73
+ "segment_count_peak": 2840,
74
+ "segment_count_peak_delta": 713,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:37.478575Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T191737349184Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17a59db31fcb1178c6c17f0052dcce04b3b1b625978b1186d12783ae15ce5696
3
+ size 6887605
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75889472000,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113484609024,
5
+ "active_bytes_peak_delta": 35077120,
6
+ "after": {
7
+ "active_bytes_current": 75889472000,
8
+ "active_bytes_peak": 113484609024,
9
+ "allocated_bytes_current": 75889472000,
10
+ "allocated_bytes_peak": 113484609024,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 5903650304,
13
+ "inactive_split_bytes_peak": 14135910400,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 113894227968,
21
+ "reserved_bytes_peak": 113894227968,
22
+ "segment_count_current": 2840,
23
+ "segment_count_peak": 2840,
24
+ "timestamp_utc": "2026-08-26T19:17:38.517318Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75889472000,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113484609024,
30
+ "allocated_bytes_peak_delta": 35077120,
31
+ "before": {
32
+ "active_bytes_current": 75889472000,
33
+ "active_bytes_peak": 113449531904,
34
+ "allocated_bytes_current": 75889472000,
35
+ "allocated_bytes_peak": 113449531904,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 5903650304,
38
+ "inactive_split_bytes_peak": 14135910400,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 113894227968,
46
+ "reserved_bytes_peak": 113894227968,
47
+ "segment_count_current": 2840,
48
+ "segment_count_peak": 2840,
49
+ "timestamp_utc": "2026-08-26T19:17:37.494110Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5903650304,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 14135910400,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113894227968,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 113894227968,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2840,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2840,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:38.650019Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T191738517766Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b915e4bfb453cd576b0d714de462afc17aeec39ebeec84f050668f313c22dcc9
3
+ size 6853737
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75906515456,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 113484609024,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75906515456,
8
+ "active_bytes_peak": 113484609024,
9
+ "allocated_bytes_current": 75906515456,
10
+ "allocated_bytes_peak": 113484609024,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 5941132800,
13
+ "inactive_split_bytes_peak": 15458314240,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 119021764608,
21
+ "reserved_bytes_peak": 119021764608,
22
+ "segment_count_current": 3128,
23
+ "segment_count_peak": 3128,
24
+ "timestamp_utc": "2026-08-26T19:17:41.821352Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75906515456,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 113484609024,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75889472000,
33
+ "active_bytes_peak": 113484609024,
34
+ "allocated_bytes_current": 75889472000,
35
+ "allocated_bytes_peak": 113484609024,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 5903650304,
38
+ "inactive_split_bytes_peak": 14135910400,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 113894227968,
46
+ "reserved_bytes_peak": 113894227968,
47
+ "segment_count_current": 2840,
48
+ "segment_count_peak": 2840,
49
+ "timestamp_utc": "2026-08-26T19:17:38.664950Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5941132800,
55
+ "inactive_split_bytes_current_delta": 37482496,
56
+ "inactive_split_bytes_peak": 15458314240,
57
+ "inactive_split_bytes_peak_delta": 1322403840,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119021764608,
68
+ "reserved_bytes_current_delta": 5127536640,
69
+ "reserved_bytes_peak": 119021764608,
70
+ "reserved_bytes_peak_delta": 5127536640,
71
+ "segment_count_current": 3128,
72
+ "segment_count_current_delta": 288,
73
+ "segment_count_peak": 3128,
74
+ "segment_count_peak_delta": 288,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:41.966789Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T191741821837Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2dad172e0d2c6dc1c5c2a863553b608c87e1edb0f663f8f11233e102566d2f0f
3
+ size 7509462
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75906515456,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113484609024,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75906515456,
8
+ "active_bytes_peak": 113484609024,
9
+ "allocated_bytes_current": 75906515456,
10
+ "allocated_bytes_peak": 113484609024,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 5941132800,
13
+ "inactive_split_bytes_peak": 15458314240,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 119021764608,
21
+ "reserved_bytes_peak": 119021764608,
22
+ "segment_count_current": 3128,
23
+ "segment_count_peak": 3128,
24
+ "timestamp_utc": "2026-08-26T19:17:42.351542Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75906515456,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113484609024,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75906515456,
33
+ "active_bytes_peak": 113484609024,
34
+ "allocated_bytes_current": 75906515456,
35
+ "allocated_bytes_peak": 113484609024,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 5941132800,
38
+ "inactive_split_bytes_peak": 15458314240,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 119021764608,
46
+ "reserved_bytes_peak": 119021764608,
47
+ "segment_count_current": 3128,
48
+ "segment_count_peak": 3128,
49
+ "timestamp_utc": "2026-08-26T19:17:41.987871Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5941132800,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 15458314240,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119021764608,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119021764608,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3128,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3128,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:42.372555Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T191742351973Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bfc88224475f580c8895dcd920a7e741377540aa089ea53514dc70213fe34d75
3
+ size 903196
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75906515456,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113484609024,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 75906515456,
8
+ "active_bytes_peak": 113484609024,
9
+ "allocated_bytes_current": 75906515456,
10
+ "allocated_bytes_peak": 113484609024,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 5941132800,
13
+ "inactive_split_bytes_peak": 15458314240,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 119021764608,
21
+ "reserved_bytes_peak": 119021764608,
22
+ "segment_count_current": 3128,
23
+ "segment_count_peak": 3128,
24
+ "timestamp_utc": "2026-08-26T19:17:42.732549Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75906515456,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113484609024,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 75906515456,
33
+ "active_bytes_peak": 113484609024,
34
+ "allocated_bytes_current": 75906515456,
35
+ "allocated_bytes_peak": 113484609024,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 5941132800,
38
+ "inactive_split_bytes_peak": 15458314240,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 119021764608,
46
+ "reserved_bytes_peak": 119021764608,
47
+ "segment_count_current": 3128,
48
+ "segment_count_peak": 3128,
49
+ "timestamp_utc": "2026-08-26T19:17:42.374192Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5941132800,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 15458314240,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 119021764608,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 119021764608,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3128,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3128,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:42.751576Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T191742732944Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:860eb9dcd108badc7a929a9055d4932e004bca7ae75c76ec1aaf6dc2dd7e394f
3
+ size 903196
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75883125760,
3
+ "active_bytes_current_delta": 32328931328,
4
+ "active_bytes_peak": 82624266240,
5
+ "active_bytes_peak_delta": 39063780352,
6
+ "after": {
7
+ "active_bytes_current": 75883125760,
8
+ "active_bytes_peak": 82624266240,
9
+ "allocated_bytes_current": 75883125760,
10
+ "allocated_bytes_peak": 82624266240,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 5901607936,
13
+ "inactive_split_bytes_peak": 14149293568,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 82996887552,
21
+ "reserved_bytes_peak": 82996887552,
22
+ "segment_count_current": 2112,
23
+ "segment_count_peak": 2112,
24
+ "timestamp_utc": "2026-08-26T19:17:35.910208Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75883125760,
28
+ "allocated_bytes_current_delta": 32328931328,
29
+ "allocated_bytes_peak": 82624266240,
30
+ "allocated_bytes_peak_delta": 39063780352,
31
+ "before": {
32
+ "active_bytes_current": 43554194432,
33
+ "active_bytes_peak": 43560485888,
34
+ "allocated_bytes_current": 43554194432,
35
+ "allocated_bytes_peak": 43560485888,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1255652352,
38
+ "inactive_split_bytes_peak": 1612261376,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 44835012608,
46
+ "reserved_bytes_peak": 44835012608,
47
+ "segment_count_current": 883,
48
+ "segment_count_peak": 883,
49
+ "timestamp_utc": "2026-08-26T19:17:09.289231Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5901607936,
55
+ "inactive_split_bytes_current_delta": 4645955584,
56
+ "inactive_split_bytes_peak": 14149293568,
57
+ "inactive_split_bytes_peak_delta": 12537032192,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 82996887552,
68
+ "reserved_bytes_current_delta": 38161874944,
69
+ "reserved_bytes_peak": 82996887552,
70
+ "reserved_bytes_peak_delta": 38161874944,
71
+ "segment_count_current": 2112,
72
+ "segment_count_current_delta": 1229,
73
+ "segment_count_peak": 2112,
74
+ "segment_count_peak_delta": 1229,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:36.046760Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T191735910677Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1873ea447f9791d697e95e1b4efbbaf57a09fbc11a968d7d1450883233f61756
3
+ size 7001386
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 75883125760,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113455617536,
5
+ "active_bytes_peak_delta": 30831351296,
6
+ "after": {
7
+ "active_bytes_current": 75883125760,
8
+ "active_bytes_peak": 113455617536,
9
+ "allocated_bytes_current": 75883125760,
10
+ "allocated_bytes_peak": 113455617536,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 5901607936,
13
+ "inactive_split_bytes_peak": 14149293568,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 113841799168,
21
+ "reserved_bytes_peak": 113841799168,
22
+ "segment_count_current": 2812,
23
+ "segment_count_peak": 2812,
24
+ "timestamp_utc": "2026-08-26T19:17:37.348746Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 75883125760,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113455617536,
30
+ "allocated_bytes_peak_delta": 30831351296,
31
+ "before": {
32
+ "active_bytes_current": 75883125760,
33
+ "active_bytes_peak": 82624266240,
34
+ "allocated_bytes_current": 75883125760,
35
+ "allocated_bytes_peak": 82624266240,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 5901607936,
38
+ "inactive_split_bytes_peak": 14149293568,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 82996887552,
46
+ "reserved_bytes_peak": 82996887552,
47
+ "segment_count_current": 2112,
48
+ "segment_count_peak": 2112,
49
+ "timestamp_utc": "2026-08-26T19:17:36.066321Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 5901607936,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 14149293568,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 113841799168,
68
+ "reserved_bytes_current_delta": 30844911616,
69
+ "reserved_bytes_peak": 113841799168,
70
+ "reserved_bytes_peak_delta": 30844911616,
71
+ "segment_count_current": 2812,
72
+ "segment_count_current_delta": 700,
73
+ "segment_count_peak": 2812,
74
+ "segment_count_peak_delta": 700,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:17:37.477811Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T191737349163Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6bd4983fbce8115d5dfac2488b6a014fa7ba53a90610645caf927c4081abcbe3
3
+ size 6492481