AbhishekGhosh1998 commited on
Commit
3dfc114
·
verified ·
1 Parent(s): 1470ed2

granularity results ultraep/tierbmem @ 2026-08-26T19:31:45Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 2)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.json +79 -0
  2. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.pickle +3 -0
  3. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.json +79 -0
  4. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.pickle +3 -0
  5. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.json +79 -0
  6. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.pickle +3 -0
  7. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.json +79 -0
  8. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.pickle +3 -0
  9. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.json +79 -0
  10. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.pickle +3 -0
  11. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.json +79 -0
  12. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.pickle +3 -0
  13. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.json +79 -0
  14. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.pickle +3 -0
  15. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.json +79 -0
  16. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.pickle +3 -0
  17. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.json +79 -0
  18. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.pickle +3 -0
  19. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.json +79 -0
  20. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.pickle +3 -0
  21. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.json +79 -0
  22. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.pickle +3 -0
  23. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.json +79 -0
  24. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.pickle +3 -0
  25. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.json +79 -0
  26. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.pickle +3 -0
  27. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.json +79 -0
  28. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.pickle +3 -0
  29. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.json +79 -0
  30. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.pickle +3 -0
  31. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.json +79 -0
  32. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.pickle +3 -0
  33. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.json +79 -0
  34. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.pickle +3 -0
  35. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.json +79 -0
  36. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.pickle +3 -0
  37. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.json +79 -0
  38. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.pickle +3 -0
  39. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.json +79 -0
  40. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.pickle +3 -0
  41. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.json +79 -0
  42. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.pickle +3 -0
  43. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.json +79 -0
  44. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.pickle +3 -0
  45. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.json +79 -0
  46. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.pickle +3 -0
  47. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.json +79 -0
  48. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.pickle +3 -0
  49. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env +61 -0
  50. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log +0 -0
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38993883648,
3
+ "active_bytes_current_delta": 15945322496,
4
+ "active_bytes_peak": 109588680192,
5
+ "active_bytes_peak_delta": 86527536128,
6
+ "after": {
7
+ "active_bytes_current": 38993883648,
8
+ "active_bytes_peak": 109588680192,
9
+ "allocated_bytes_current": 38993883648,
10
+ "allocated_bytes_peak": 109588680192,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 14059867648,
13
+ "inactive_split_bytes_peak": 27846074880,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 112956801024,
21
+ "reserved_bytes_peak": 112956801024,
22
+ "segment_count_current": 847,
23
+ "segment_count_peak": 847,
24
+ "timestamp_utc": "2026-08-26T19:23:45.258808Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38993883648,
28
+ "allocated_bytes_current_delta": 15945322496,
29
+ "allocated_bytes_peak": 109588680192,
30
+ "allocated_bytes_peak_delta": 86527536128,
31
+ "before": {
32
+ "active_bytes_current": 23048561152,
33
+ "active_bytes_peak": 23061144064,
34
+ "allocated_bytes_current": 23048561152,
35
+ "allocated_bytes_peak": 23061144064,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 389209600,
38
+ "inactive_split_bytes_peak": 883057664,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 25623003136,
46
+ "reserved_bytes_peak": 25623003136,
47
+ "segment_count_current": 551,
48
+ "segment_count_peak": 551,
49
+ "timestamp_utc": "2026-08-26T19:23:17.758985Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 14059867648,
55
+ "inactive_split_bytes_current_delta": 13670658048,
56
+ "inactive_split_bytes_peak": 27846074880,
57
+ "inactive_split_bytes_peak_delta": 26963017216,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 112956801024,
68
+ "reserved_bytes_current_delta": 87333797888,
69
+ "reserved_bytes_peak": 112956801024,
70
+ "reserved_bytes_peak_delta": 87333797888,
71
+ "segment_count_current": 847,
72
+ "segment_count_current_delta": 296,
73
+ "segment_count_peak": 847,
74
+ "segment_count_peak_delta": 296,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:45.320281Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T192345259308Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:83a7120506693b5adcf9d91a1bca38a7b521d79c9f2046a36149f5c375e2c804
3
+ size 3054015
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38993883648,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123802562048,
5
+ "active_bytes_peak_delta": 14213881856,
6
+ "after": {
7
+ "active_bytes_current": 38993883648,
8
+ "active_bytes_peak": 123802562048,
9
+ "allocated_bytes_current": 38993883648,
10
+ "allocated_bytes_peak": 123802562048,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 14059867648,
13
+ "inactive_split_bytes_peak": 27846074880,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 127500550144,
21
+ "reserved_bytes_peak": 127500550144,
22
+ "segment_count_current": 859,
23
+ "segment_count_peak": 859,
24
+ "timestamp_utc": "2026-08-26T19:23:45.929126Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38993883648,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123802562048,
30
+ "allocated_bytes_peak_delta": 14213881856,
31
+ "before": {
32
+ "active_bytes_current": 38993883648,
33
+ "active_bytes_peak": 109588680192,
34
+ "allocated_bytes_current": 38993883648,
35
+ "allocated_bytes_peak": 109588680192,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 14059867648,
38
+ "inactive_split_bytes_peak": 27846074880,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 112956801024,
46
+ "reserved_bytes_peak": 112956801024,
47
+ "segment_count_current": 847,
48
+ "segment_count_peak": 847,
49
+ "timestamp_utc": "2026-08-26T19:23:45.329151Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 14059867648,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 27846074880,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127500550144,
68
+ "reserved_bytes_current_delta": 14543749120,
69
+ "reserved_bytes_peak": 127500550144,
70
+ "reserved_bytes_peak_delta": 14543749120,
71
+ "segment_count_current": 859,
72
+ "segment_count_current_delta": 12,
73
+ "segment_count_peak": 859,
74
+ "segment_count_peak_delta": 12,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:45.980787Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T192345929555Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cc4f989a013e267c2cfaf9ce568bff5374f09480dedd764567cb3e2664e09bdb
3
+ size 3049002
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38993883648,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123803930112,
5
+ "active_bytes_peak_delta": 1368064,
6
+ "after": {
7
+ "active_bytes_current": 38993883648,
8
+ "active_bytes_peak": 123803930112,
9
+ "allocated_bytes_current": 38993883648,
10
+ "allocated_bytes_peak": 123803930112,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 14059867648,
13
+ "inactive_split_bytes_peak": 27846074880,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 127500550144,
21
+ "reserved_bytes_peak": 127500550144,
22
+ "segment_count_current": 859,
23
+ "segment_count_peak": 859,
24
+ "timestamp_utc": "2026-08-26T19:23:46.587185Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38993883648,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123803930112,
30
+ "allocated_bytes_peak_delta": 1368064,
31
+ "before": {
32
+ "active_bytes_current": 38993883648,
33
+ "active_bytes_peak": 123802562048,
34
+ "allocated_bytes_current": 38993883648,
35
+ "allocated_bytes_peak": 123802562048,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 14059867648,
38
+ "inactive_split_bytes_peak": 27846074880,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 127500550144,
46
+ "reserved_bytes_peak": 127500550144,
47
+ "segment_count_current": 859,
48
+ "segment_count_peak": 859,
49
+ "timestamp_utc": "2026-08-26T19:23:45.986674Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 14059867648,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 27846074880,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127500550144,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 127500550144,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 859,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 859,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:46.638228Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T192346587622Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a31cb9f06e39cd018076e2932e645017551c8ecdf23505ffa7b8d84aae3810c2
3
+ size 3065016
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 39011844608,
3
+ "active_bytes_current_delta": 17960960,
4
+ "active_bytes_peak": 123803930112,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 39011844608,
8
+ "active_bytes_peak": 123803930112,
9
+ "allocated_bytes_current": 39011844608,
10
+ "allocated_bytes_peak": 123803930112,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 16214556160,
13
+ "inactive_split_bytes_peak": 39152205824,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 140314148864,
21
+ "reserved_bytes_peak": 140314148864,
22
+ "segment_count_current": 970,
23
+ "segment_count_peak": 970,
24
+ "timestamp_utc": "2026-08-26T19:23:48.442889Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 39011844608,
28
+ "allocated_bytes_current_delta": 17960960,
29
+ "allocated_bytes_peak": 123803930112,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 38993883648,
33
+ "active_bytes_peak": 123803930112,
34
+ "allocated_bytes_current": 38993883648,
35
+ "allocated_bytes_peak": 123803930112,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 14059867648,
38
+ "inactive_split_bytes_peak": 27846074880,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 127500550144,
46
+ "reserved_bytes_peak": 127500550144,
47
+ "segment_count_current": 859,
48
+ "segment_count_peak": 859,
49
+ "timestamp_utc": "2026-08-26T19:23:46.644129Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16214556160,
55
+ "inactive_split_bytes_current_delta": 2154688512,
56
+ "inactive_split_bytes_peak": 39152205824,
57
+ "inactive_split_bytes_peak_delta": 11306130944,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 140314148864,
68
+ "reserved_bytes_current_delta": 12813598720,
69
+ "reserved_bytes_peak": 140314148864,
70
+ "reserved_bytes_peak_delta": 12813598720,
71
+ "segment_count_current": 970,
72
+ "segment_count_current_delta": 111,
73
+ "segment_count_peak": 970,
74
+ "segment_count_peak_delta": 111,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:48.501132Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T192348443475Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17efcabdc174de09d2a136bf51ce42521716d10ed82c4020b784e72e341b1763
3
+ size 3226224
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 39011844608,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123803930112,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 39011844608,
8
+ "active_bytes_peak": 123803930112,
9
+ "allocated_bytes_current": 39011844608,
10
+ "allocated_bytes_peak": 123803930112,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 16214556160,
13
+ "inactive_split_bytes_peak": 39152205824,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 140314148864,
21
+ "reserved_bytes_peak": 140314148864,
22
+ "segment_count_current": 970,
23
+ "segment_count_peak": 970,
24
+ "timestamp_utc": "2026-08-26T19:23:49.089210Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 39011844608,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123803930112,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 39011844608,
33
+ "active_bytes_peak": 123803930112,
34
+ "allocated_bytes_current": 39011844608,
35
+ "allocated_bytes_peak": 123803930112,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 16214556160,
38
+ "inactive_split_bytes_peak": 39152205824,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 140314148864,
46
+ "reserved_bytes_peak": 140314148864,
47
+ "segment_count_current": 970,
48
+ "segment_count_peak": 970,
49
+ "timestamp_utc": "2026-08-26T19:23:48.508069Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16214556160,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 39152205824,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 140314148864,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 140314148864,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 970,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 970,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:49.097417Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T192349089611Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cec6e5732ef24b557f71d1208d1d83a838a2c444e47941d558eb4b8c6d85b91a
3
+ size 370864
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 39011844608,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123803930112,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 39011844608,
8
+ "active_bytes_peak": 123803930112,
9
+ "allocated_bytes_current": 39011844608,
10
+ "allocated_bytes_peak": 123803930112,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 16214556160,
13
+ "inactive_split_bytes_peak": 39152205824,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 140314148864,
21
+ "reserved_bytes_peak": 140314148864,
22
+ "segment_count_current": 970,
23
+ "segment_count_peak": 970,
24
+ "timestamp_utc": "2026-08-26T19:23:49.684045Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 39011844608,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123803930112,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 39011844608,
33
+ "active_bytes_peak": 123803930112,
34
+ "allocated_bytes_current": 39011844608,
35
+ "allocated_bytes_peak": 123803930112,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 16214556160,
38
+ "inactive_split_bytes_peak": 39152205824,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 140314148864,
46
+ "reserved_bytes_peak": 140314148864,
47
+ "segment_count_current": 970,
48
+ "segment_count_peak": 970,
49
+ "timestamp_utc": "2026-08-26T19:23:49.098450Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16214556160,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 39152205824,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 140314148864,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 140314148864,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 970,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 970,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:49.692107Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T192349684438Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50d5a1c42eda960f09f3c2d51a8efd01aafd7a1c7a2f33260dd3fb2f9ea3d71a
3
+ size 370864
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38984414720,
3
+ "active_bytes_current_delta": 15931054080,
4
+ "active_bytes_peak": 109604034048,
5
+ "active_bytes_peak_delta": 86538090496,
6
+ "after": {
7
+ "active_bytes_current": 38984414720,
8
+ "active_bytes_peak": 109604034048,
9
+ "allocated_bytes_current": 38984414720,
10
+ "allocated_bytes_peak": 109604034048,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 13970770432,
13
+ "inactive_split_bytes_peak": 27816583680,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 112885497856,
21
+ "reserved_bytes_peak": 112885497856,
22
+ "segment_count_current": 844,
23
+ "segment_count_peak": 844,
24
+ "timestamp_utc": "2026-08-26T19:23:45.258776Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38984414720,
28
+ "allocated_bytes_current_delta": 15931054080,
29
+ "allocated_bytes_peak": 109604034048,
30
+ "allocated_bytes_peak_delta": 86538090496,
31
+ "before": {
32
+ "active_bytes_current": 23053360640,
33
+ "active_bytes_peak": 23065943552,
34
+ "allocated_bytes_current": 23053360640,
35
+ "allocated_bytes_peak": 23065943552,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 474587648,
38
+ "inactive_split_bytes_peak": 883057664,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 25623003136,
46
+ "reserved_bytes_peak": 25623003136,
47
+ "segment_count_current": 551,
48
+ "segment_count_peak": 551,
49
+ "timestamp_utc": "2026-08-26T19:23:17.759118Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 13970770432,
55
+ "inactive_split_bytes_current_delta": 13496182784,
56
+ "inactive_split_bytes_peak": 27816583680,
57
+ "inactive_split_bytes_peak_delta": 26933526016,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 112885497856,
68
+ "reserved_bytes_current_delta": 87262494720,
69
+ "reserved_bytes_peak": 112885497856,
70
+ "reserved_bytes_peak_delta": 87262494720,
71
+ "segment_count_current": 844,
72
+ "segment_count_current_delta": 293,
73
+ "segment_count_peak": 844,
74
+ "segment_count_peak_delta": 293,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:45.319604Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T192345259282Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9341d9d5fa4db59064b5f59b8a55b4c55fde01c00f22771fd9bf802548f669d5
3
+ size 3011418
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38984414720,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123775942144,
5
+ "active_bytes_peak_delta": 14171908096,
6
+ "after": {
7
+ "active_bytes_current": 38984414720,
8
+ "active_bytes_peak": 123775942144,
9
+ "allocated_bytes_current": 38984414720,
10
+ "allocated_bytes_peak": 123775942144,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 13970770432,
13
+ "inactive_split_bytes_peak": 27816583680,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 127425052672,
21
+ "reserved_bytes_peak": 127425052672,
22
+ "segment_count_current": 854,
23
+ "segment_count_peak": 854,
24
+ "timestamp_utc": "2026-08-26T19:23:45.929130Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38984414720,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123775942144,
30
+ "allocated_bytes_peak_delta": 14171908096,
31
+ "before": {
32
+ "active_bytes_current": 38984414720,
33
+ "active_bytes_peak": 109604034048,
34
+ "allocated_bytes_current": 38984414720,
35
+ "allocated_bytes_peak": 109604034048,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 13970770432,
38
+ "inactive_split_bytes_peak": 27816583680,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 112885497856,
46
+ "reserved_bytes_peak": 112885497856,
47
+ "segment_count_current": 844,
48
+ "segment_count_peak": 844,
49
+ "timestamp_utc": "2026-08-26T19:23:45.328437Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 13970770432,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 27816583680,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127425052672,
68
+ "reserved_bytes_current_delta": 14539554816,
69
+ "reserved_bytes_peak": 127425052672,
70
+ "reserved_bytes_peak_delta": 14539554816,
71
+ "segment_count_current": 854,
72
+ "segment_count_current_delta": 10,
73
+ "segment_count_peak": 854,
74
+ "segment_count_peak_delta": 10,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:45.980410Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T192345929555Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ed801a69e72e5224d35aa1d873046563c8e396d2748ad13c4ea89e09b92a6a6
3
+ size 2977330
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38984414720,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123783904768,
5
+ "active_bytes_peak_delta": 7962624,
6
+ "after": {
7
+ "active_bytes_current": 38984414720,
8
+ "active_bytes_peak": 123783904768,
9
+ "allocated_bytes_current": 38984414720,
10
+ "allocated_bytes_peak": 123783904768,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 13970770432,
13
+ "inactive_split_bytes_peak": 27816583680,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 127425052672,
21
+ "reserved_bytes_peak": 127425052672,
22
+ "segment_count_current": 854,
23
+ "segment_count_peak": 854,
24
+ "timestamp_utc": "2026-08-26T19:23:46.587182Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38984414720,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123783904768,
30
+ "allocated_bytes_peak_delta": 7962624,
31
+ "before": {
32
+ "active_bytes_current": 38984414720,
33
+ "active_bytes_peak": 123775942144,
34
+ "allocated_bytes_current": 38984414720,
35
+ "allocated_bytes_peak": 123775942144,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 13970770432,
38
+ "inactive_split_bytes_peak": 27816583680,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 127425052672,
46
+ "reserved_bytes_peak": 127425052672,
47
+ "segment_count_current": 854,
48
+ "segment_count_peak": 854,
49
+ "timestamp_utc": "2026-08-26T19:23:45.986292Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 13970770432,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 27816583680,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127425052672,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 127425052672,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 854,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 854,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:46.638582Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T192346587624Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ddcccc20560180b7aa3d3f3772e82fdc11a13e38838401f8f6f5aec5b52d1e2f
3
+ size 2954732
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 39002375680,
3
+ "active_bytes_current_delta": 17960960,
4
+ "active_bytes_peak": 123783904768,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 39002375680,
8
+ "active_bytes_peak": 123783904768,
9
+ "allocated_bytes_current": 39002375680,
10
+ "allocated_bytes_peak": 123783904768,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 16125458944,
13
+ "inactive_split_bytes_peak": 39063108608,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 140215582720,
21
+ "reserved_bytes_peak": 140215582720,
22
+ "segment_count_current": 954,
23
+ "segment_count_peak": 954,
24
+ "timestamp_utc": "2026-08-26T19:23:48.442887Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 39002375680,
28
+ "allocated_bytes_current_delta": 17960960,
29
+ "allocated_bytes_peak": 123783904768,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 38984414720,
33
+ "active_bytes_peak": 123783904768,
34
+ "allocated_bytes_current": 38984414720,
35
+ "allocated_bytes_peak": 123783904768,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 13970770432,
38
+ "inactive_split_bytes_peak": 27816583680,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 127425052672,
46
+ "reserved_bytes_peak": 127425052672,
47
+ "segment_count_current": 854,
48
+ "segment_count_peak": 854,
49
+ "timestamp_utc": "2026-08-26T19:23:46.644412Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16125458944,
55
+ "inactive_split_bytes_current_delta": 2154688512,
56
+ "inactive_split_bytes_peak": 39063108608,
57
+ "inactive_split_bytes_peak_delta": 11246524928,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 140215582720,
68
+ "reserved_bytes_current_delta": 12790530048,
69
+ "reserved_bytes_peak": 140215582720,
70
+ "reserved_bytes_peak_delta": 12790530048,
71
+ "segment_count_current": 954,
72
+ "segment_count_current_delta": 100,
73
+ "segment_count_peak": 954,
74
+ "segment_count_peak_delta": 100,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:48.500293Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T192348443389Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4fb213e3645e5468f55929eff49a7c322a3e786f17fee99dfde3863d7ee5289
3
+ size 3107068
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 39002375680,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123783904768,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 39002375680,
8
+ "active_bytes_peak": 123783904768,
9
+ "allocated_bytes_current": 39002375680,
10
+ "allocated_bytes_peak": 123783904768,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 16125458944,
13
+ "inactive_split_bytes_peak": 39063108608,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 140215582720,
21
+ "reserved_bytes_peak": 140215582720,
22
+ "segment_count_current": 954,
23
+ "segment_count_peak": 954,
24
+ "timestamp_utc": "2026-08-26T19:23:49.089156Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 39002375680,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123783904768,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 39002375680,
33
+ "active_bytes_peak": 123783904768,
34
+ "allocated_bytes_current": 39002375680,
35
+ "allocated_bytes_peak": 123783904768,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 16125458944,
38
+ "inactive_split_bytes_peak": 39063108608,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 140215582720,
46
+ "reserved_bytes_peak": 140215582720,
47
+ "segment_count_current": 954,
48
+ "segment_count_peak": 954,
49
+ "timestamp_utc": "2026-08-26T19:23:48.507139Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16125458944,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 39063108608,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 140215582720,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 140215582720,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 954,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 954,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:49.097290Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T192349089621Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:86ee3b1e2b4ba11de5e21e6070dd5cc3455d6c71933b42b8b11cb08a5089ce7a
3
+ size 342390
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 39002375680,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123783904768,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 39002375680,
8
+ "active_bytes_peak": 123783904768,
9
+ "allocated_bytes_current": 39002375680,
10
+ "allocated_bytes_peak": 123783904768,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 16125458944,
13
+ "inactive_split_bytes_peak": 39063108608,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 140215582720,
21
+ "reserved_bytes_peak": 140215582720,
22
+ "segment_count_current": 954,
23
+ "segment_count_peak": 954,
24
+ "timestamp_utc": "2026-08-26T19:23:49.684055Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 39002375680,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123783904768,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 39002375680,
33
+ "active_bytes_peak": 123783904768,
34
+ "allocated_bytes_current": 39002375680,
35
+ "allocated_bytes_peak": 123783904768,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 16125458944,
38
+ "inactive_split_bytes_peak": 39063108608,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 140215582720,
46
+ "reserved_bytes_peak": 140215582720,
47
+ "segment_count_current": 954,
48
+ "segment_count_peak": 954,
49
+ "timestamp_utc": "2026-08-26T19:23:49.098305Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16125458944,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 39063108608,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 140215582720,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 140215582720,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 954,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 954,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:49.691896Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T192349684439Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4e269c98f92de8a97b417be2464a4e702baebd152c70326d4eee25ebc3edef8f
3
+ size 342390
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38969498112,
3
+ "active_bytes_current_delta": 15923573248,
4
+ "active_bytes_peak": 109581087232,
5
+ "active_bytes_peak_delta": 86522579456,
6
+ "after": {
7
+ "active_bytes_current": 38969498112,
8
+ "active_bytes_peak": 109581087232,
9
+ "allocated_bytes_current": 38969498112,
10
+ "allocated_bytes_peak": 109581087232,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 14163944960,
13
+ "inactive_split_bytes_peak": 27855711232,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 110880620544,
21
+ "reserved_bytes_peak": 110880620544,
22
+ "segment_count_current": 851,
23
+ "segment_count_peak": 851,
24
+ "timestamp_utc": "2026-08-26T19:23:45.258693Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38969498112,
28
+ "allocated_bytes_current_delta": 15923573248,
29
+ "allocated_bytes_peak": 109581087232,
30
+ "allocated_bytes_peak_delta": 86522579456,
31
+ "before": {
32
+ "active_bytes_current": 23045924864,
33
+ "active_bytes_peak": 23058507776,
34
+ "allocated_bytes_current": 23045924864,
35
+ "allocated_bytes_peak": 23058507776,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 370874368,
38
+ "inactive_split_bytes_peak": 883057664,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 25761415168,
46
+ "reserved_bytes_peak": 25761415168,
47
+ "segment_count_current": 557,
48
+ "segment_count_peak": 557,
49
+ "timestamp_utc": "2026-08-26T19:23:17.759100Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 14163944960,
55
+ "inactive_split_bytes_current_delta": 13793070592,
56
+ "inactive_split_bytes_peak": 27855711232,
57
+ "inactive_split_bytes_peak_delta": 26972653568,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 110880620544,
68
+ "reserved_bytes_current_delta": 85119205376,
69
+ "reserved_bytes_peak": 110880620544,
70
+ "reserved_bytes_peak_delta": 85119205376,
71
+ "segment_count_current": 851,
72
+ "segment_count_current_delta": 294,
73
+ "segment_count_peak": 851,
74
+ "segment_count_peak_delta": 294,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:45.318867Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T192345259233Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab9d452f6c8ec9d84dfc01c9f6afa1d0424918b0ff568ccb7d0cf88afb520f03
3
+ size 3059007
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38969498112,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123804231168,
5
+ "active_bytes_peak_delta": 14223143936,
6
+ "after": {
7
+ "active_bytes_current": 38969498112,
8
+ "active_bytes_peak": 123804231168,
9
+ "allocated_bytes_current": 38969498112,
10
+ "allocated_bytes_peak": 123804231168,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 14163944960,
13
+ "inactive_split_bytes_peak": 27855711232,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 127641059328,
21
+ "reserved_bytes_peak": 127641059328,
22
+ "segment_count_current": 866,
23
+ "segment_count_peak": 866,
24
+ "timestamp_utc": "2026-08-26T19:23:45.929006Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38969498112,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123804231168,
30
+ "allocated_bytes_peak_delta": 14223143936,
31
+ "before": {
32
+ "active_bytes_current": 38969498112,
33
+ "active_bytes_peak": 109581087232,
34
+ "allocated_bytes_current": 38969498112,
35
+ "allocated_bytes_peak": 109581087232,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 14163944960,
38
+ "inactive_split_bytes_peak": 27855711232,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 110880620544,
46
+ "reserved_bytes_peak": 110880620544,
47
+ "segment_count_current": 851,
48
+ "segment_count_peak": 851,
49
+ "timestamp_utc": "2026-08-26T19:23:45.327625Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 14163944960,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 27855711232,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127641059328,
68
+ "reserved_bytes_current_delta": 16760438784,
69
+ "reserved_bytes_peak": 127641059328,
70
+ "reserved_bytes_peak_delta": 16760438784,
71
+ "segment_count_current": 866,
72
+ "segment_count_current_delta": 15,
73
+ "segment_count_peak": 866,
74
+ "segment_count_peak_delta": 15,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:45.980292Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T192345929415Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1e36a3bd906118e6570f5b6d01261ce332954e8ca22ee517fec097c83996e02
3
+ size 2973374
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38969498112,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123805369856,
5
+ "active_bytes_peak_delta": 1138688,
6
+ "after": {
7
+ "active_bytes_current": 38969498112,
8
+ "active_bytes_peak": 123805369856,
9
+ "allocated_bytes_current": 38969498112,
10
+ "allocated_bytes_peak": 123805369856,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 14163944960,
13
+ "inactive_split_bytes_peak": 27855711232,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 127641059328,
21
+ "reserved_bytes_peak": 127641059328,
22
+ "segment_count_current": 866,
23
+ "segment_count_peak": 866,
24
+ "timestamp_utc": "2026-08-26T19:23:46.587092Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38969498112,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123805369856,
30
+ "allocated_bytes_peak_delta": 1138688,
31
+ "before": {
32
+ "active_bytes_current": 38969498112,
33
+ "active_bytes_peak": 123804231168,
34
+ "allocated_bytes_current": 38969498112,
35
+ "allocated_bytes_peak": 123804231168,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 14163944960,
38
+ "inactive_split_bytes_peak": 27855711232,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 127641059328,
46
+ "reserved_bytes_peak": 127641059328,
47
+ "segment_count_current": 866,
48
+ "segment_count_peak": 866,
49
+ "timestamp_utc": "2026-08-26T19:23:45.986039Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 14163944960,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 27855711232,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127641059328,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 127641059328,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 866,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 866,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:46.637391Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T192346587495Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:842e23b86e4332c4ad01fd6a72563c2fb7ea3e22c09e87c938975ad7e18151a6
3
+ size 2972521
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38987459072,
3
+ "active_bytes_current_delta": 17960960,
4
+ "active_bytes_peak": 123805369856,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 38987459072,
8
+ "active_bytes_peak": 123805369856,
9
+ "allocated_bytes_current": 38987459072,
10
+ "allocated_bytes_peak": 123805369856,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 16318633472,
13
+ "inactive_split_bytes_peak": 39258380288,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 140393840640,
21
+ "reserved_bytes_peak": 140393840640,
22
+ "segment_count_current": 979,
23
+ "segment_count_peak": 979,
24
+ "timestamp_utc": "2026-08-26T19:23:48.442740Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38987459072,
28
+ "allocated_bytes_current_delta": 17960960,
29
+ "allocated_bytes_peak": 123805369856,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 38969498112,
33
+ "active_bytes_peak": 123805369856,
34
+ "allocated_bytes_current": 38969498112,
35
+ "allocated_bytes_peak": 123805369856,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 14163944960,
38
+ "inactive_split_bytes_peak": 27855711232,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 127641059328,
46
+ "reserved_bytes_peak": 127641059328,
47
+ "segment_count_current": 866,
48
+ "segment_count_peak": 866,
49
+ "timestamp_utc": "2026-08-26T19:23:46.643091Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16318633472,
55
+ "inactive_split_bytes_current_delta": 2154688512,
56
+ "inactive_split_bytes_peak": 39258380288,
57
+ "inactive_split_bytes_peak_delta": 11402669056,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 140393840640,
68
+ "reserved_bytes_current_delta": 12752781312,
69
+ "reserved_bytes_peak": 140393840640,
70
+ "reserved_bytes_peak_delta": 12752781312,
71
+ "segment_count_current": 979,
72
+ "segment_count_current_delta": 113,
73
+ "segment_count_peak": 979,
74
+ "segment_count_peak_delta": 113,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:48.500266Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T192348443272Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:88135e920b0d584c0c8bd4e63216f4476de7dd75c98632e43ac7c3e4dc526a16
3
+ size 3125976
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38987459072,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123805369856,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 38987459072,
8
+ "active_bytes_peak": 123805369856,
9
+ "allocated_bytes_current": 38987459072,
10
+ "allocated_bytes_peak": 123805369856,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 16318633472,
13
+ "inactive_split_bytes_peak": 39258380288,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 140393840640,
21
+ "reserved_bytes_peak": 140393840640,
22
+ "segment_count_current": 979,
23
+ "segment_count_peak": 979,
24
+ "timestamp_utc": "2026-08-26T19:23:49.089066Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38987459072,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123805369856,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 38987459072,
33
+ "active_bytes_peak": 123805369856,
34
+ "allocated_bytes_current": 38987459072,
35
+ "allocated_bytes_peak": 123805369856,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 16318633472,
38
+ "inactive_split_bytes_peak": 39258380288,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 140393840640,
46
+ "reserved_bytes_peak": 140393840640,
47
+ "segment_count_current": 979,
48
+ "segment_count_peak": 979,
49
+ "timestamp_utc": "2026-08-26T19:23:48.507012Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16318633472,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 39258380288,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 140393840640,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 140393840640,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 979,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 979,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:49.097344Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T192349089458Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6335d55966a4a3dc0c04469505d09d68dcc1b64c470488b99ed688dbf506b084
3
+ size 356377
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38987459072,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123805369856,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 38987459072,
8
+ "active_bytes_peak": 123805369856,
9
+ "allocated_bytes_current": 38987459072,
10
+ "allocated_bytes_peak": 123805369856,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 16318633472,
13
+ "inactive_split_bytes_peak": 39258380288,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 140393840640,
21
+ "reserved_bytes_peak": 140393840640,
22
+ "segment_count_current": 979,
23
+ "segment_count_peak": 979,
24
+ "timestamp_utc": "2026-08-26T19:23:49.683944Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38987459072,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123805369856,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 38987459072,
33
+ "active_bytes_peak": 123805369856,
34
+ "allocated_bytes_current": 38987459072,
35
+ "allocated_bytes_peak": 123805369856,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 16318633472,
38
+ "inactive_split_bytes_peak": 39258380288,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 140393840640,
46
+ "reserved_bytes_peak": 140393840640,
47
+ "segment_count_current": 979,
48
+ "segment_count_peak": 979,
49
+ "timestamp_utc": "2026-08-26T19:23:49.098370Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16318633472,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 39258380288,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 140393840640,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 140393840640,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 979,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 979,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:49.691999Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T192349684323Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:78de216bceed7727e49d5802db3c33c9402b572d8949a983c77023509c976944
3
+ size 356377
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38973895680,
3
+ "active_bytes_current_delta": 15925412352,
4
+ "active_bytes_peak": 109599972864,
5
+ "active_bytes_peak_delta": 86538906624,
6
+ "after": {
7
+ "active_bytes_current": 38973895680,
8
+ "active_bytes_peak": 109599972864,
9
+ "allocated_bytes_current": 38973895680,
10
+ "allocated_bytes_peak": 109599972864,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 16290253824,
13
+ "inactive_split_bytes_peak": 28493658112,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 110899494912,
21
+ "reserved_bytes_peak": 110899494912,
22
+ "segment_count_current": 850,
23
+ "segment_count_peak": 850,
24
+ "timestamp_utc": "2026-08-26T19:23:45.258776Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38973895680,
28
+ "allocated_bytes_current_delta": 15925412352,
29
+ "allocated_bytes_peak": 109599972864,
30
+ "allocated_bytes_peak_delta": 86538906624,
31
+ "before": {
32
+ "active_bytes_current": 23048483328,
33
+ "active_bytes_peak": 23061066240,
34
+ "allocated_bytes_current": 23048483328,
35
+ "allocated_bytes_peak": 23061066240,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 515116544,
38
+ "inactive_split_bytes_peak": 883057664,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 25784483840,
46
+ "reserved_bytes_peak": 25784483840,
47
+ "segment_count_current": 558,
48
+ "segment_count_peak": 558,
49
+ "timestamp_utc": "2026-08-26T19:23:17.758943Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16290253824,
55
+ "inactive_split_bytes_current_delta": 15775137280,
56
+ "inactive_split_bytes_peak": 28493658112,
57
+ "inactive_split_bytes_peak_delta": 27610600448,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 110899494912,
68
+ "reserved_bytes_current_delta": 85115011072,
69
+ "reserved_bytes_peak": 110899494912,
70
+ "reserved_bytes_peak_delta": 85115011072,
71
+ "segment_count_current": 850,
72
+ "segment_count_current_delta": 292,
73
+ "segment_count_peak": 850,
74
+ "segment_count_peak_delta": 292,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:45.318988Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T192345259266Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7091e3a842cdfd425213707c3f5267d552da2f9e66e04529c51e6eaf79d149de
3
+ size 3038267
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38973895680,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123793538560,
5
+ "active_bytes_peak_delta": 14193565696,
6
+ "after": {
7
+ "active_bytes_current": 38973895680,
8
+ "active_bytes_peak": 123793538560,
9
+ "allocated_bytes_current": 38973895680,
10
+ "allocated_bytes_peak": 123793538560,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 16290253824,
13
+ "inactive_split_bytes_peak": 28493658112,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 127586533376,
21
+ "reserved_bytes_peak": 127586533376,
22
+ "segment_count_current": 861,
23
+ "segment_count_peak": 861,
24
+ "timestamp_utc": "2026-08-26T19:23:45.929094Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38973895680,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123793538560,
30
+ "allocated_bytes_peak_delta": 14193565696,
31
+ "before": {
32
+ "active_bytes_current": 38973895680,
33
+ "active_bytes_peak": 109599972864,
34
+ "allocated_bytes_current": 38973895680,
35
+ "allocated_bytes_peak": 109599972864,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 16290253824,
38
+ "inactive_split_bytes_peak": 28493658112,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 110899494912,
46
+ "reserved_bytes_peak": 110899494912,
47
+ "segment_count_current": 850,
48
+ "segment_count_peak": 850,
49
+ "timestamp_utc": "2026-08-26T19:23:45.327786Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16290253824,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 28493658112,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127586533376,
68
+ "reserved_bytes_current_delta": 16687038464,
69
+ "reserved_bytes_peak": 127586533376,
70
+ "reserved_bytes_peak_delta": 16687038464,
71
+ "segment_count_current": 861,
72
+ "segment_count_current_delta": 11,
73
+ "segment_count_peak": 861,
74
+ "segment_count_peak_delta": 11,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:45.981568Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T192345929520Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:921ccb5f50ad1a49b8d0912c1cc8e07ff0bbfad73b6f251c7525b20aa860cf18
3
+ size 2983046
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38973895680,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123795095040,
5
+ "active_bytes_peak_delta": 1556480,
6
+ "after": {
7
+ "active_bytes_current": 38973895680,
8
+ "active_bytes_peak": 123795095040,
9
+ "allocated_bytes_current": 38973895680,
10
+ "allocated_bytes_peak": 123795095040,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 16290253824,
13
+ "inactive_split_bytes_peak": 28493658112,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 127586533376,
21
+ "reserved_bytes_peak": 127586533376,
22
+ "segment_count_current": 861,
23
+ "segment_count_peak": 861,
24
+ "timestamp_utc": "2026-08-26T19:23:46.587246Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38973895680,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123795095040,
30
+ "allocated_bytes_peak_delta": 1556480,
31
+ "before": {
32
+ "active_bytes_current": 38973895680,
33
+ "active_bytes_peak": 123793538560,
34
+ "allocated_bytes_current": 38973895680,
35
+ "allocated_bytes_peak": 123793538560,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 16290253824,
38
+ "inactive_split_bytes_peak": 28493658112,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 127586533376,
46
+ "reserved_bytes_peak": 127586533376,
47
+ "segment_count_current": 861,
48
+ "segment_count_peak": 861,
49
+ "timestamp_utc": "2026-08-26T19:23:45.987587Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 16290253824,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 28493658112,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 127586533376,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 127586533376,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 861,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 861,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:46.639856Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T192346587694Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9531abba1c87d4d6b1014745dfad6fdcfdb5457308bed0c61f600d1cc55ec5c4
3
+ size 2959464
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38991856640,
3
+ "active_bytes_current_delta": 17960960,
4
+ "active_bytes_peak": 123795095040,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 38991856640,
8
+ "active_bytes_peak": 123795095040,
9
+ "allocated_bytes_current": 38991856640,
10
+ "allocated_bytes_peak": 123795095040,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 18444942336,
13
+ "inactive_split_bytes_peak": 41384689152,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 142524547072,
21
+ "reserved_bytes_peak": 142524547072,
22
+ "segment_count_current": 962,
23
+ "segment_count_peak": 962,
24
+ "timestamp_utc": "2026-08-26T19:23:48.442905Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38991856640,
28
+ "allocated_bytes_current_delta": 17960960,
29
+ "allocated_bytes_peak": 123795095040,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 38973895680,
33
+ "active_bytes_peak": 123795095040,
34
+ "allocated_bytes_current": 38973895680,
35
+ "allocated_bytes_peak": 123795095040,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 16290253824,
38
+ "inactive_split_bytes_peak": 28493658112,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 127586533376,
46
+ "reserved_bytes_peak": 127586533376,
47
+ "segment_count_current": 861,
48
+ "segment_count_peak": 861,
49
+ "timestamp_utc": "2026-08-26T19:23:46.645936Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18444942336,
55
+ "inactive_split_bytes_current_delta": 2154688512,
56
+ "inactive_split_bytes_peak": 41384689152,
57
+ "inactive_split_bytes_peak_delta": 12891031040,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142524547072,
68
+ "reserved_bytes_current_delta": 14938013696,
69
+ "reserved_bytes_peak": 142524547072,
70
+ "reserved_bytes_peak_delta": 14938013696,
71
+ "segment_count_current": 962,
72
+ "segment_count_current_delta": 101,
73
+ "segment_count_peak": 962,
74
+ "segment_count_peak_delta": 101,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:48.502282Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T192348443490Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc766681d2a1724bc84a2bead9bfdc7b53d38c247c32707168acff7efc391325
3
+ size 3106732
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38991856640,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123795095040,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 38991856640,
8
+ "active_bytes_peak": 123795095040,
9
+ "allocated_bytes_current": 38991856640,
10
+ "allocated_bytes_peak": 123795095040,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 18444942336,
13
+ "inactive_split_bytes_peak": 41384689152,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 142524547072,
21
+ "reserved_bytes_peak": 142524547072,
22
+ "segment_count_current": 962,
23
+ "segment_count_peak": 962,
24
+ "timestamp_utc": "2026-08-26T19:23:49.089185Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38991856640,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123795095040,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 38991856640,
33
+ "active_bytes_peak": 123795095040,
34
+ "allocated_bytes_current": 38991856640,
35
+ "allocated_bytes_peak": 123795095040,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 18444942336,
38
+ "inactive_split_bytes_peak": 41384689152,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 142524547072,
46
+ "reserved_bytes_peak": 142524547072,
47
+ "segment_count_current": 962,
48
+ "segment_count_peak": 962,
49
+ "timestamp_utc": "2026-08-26T19:23:48.509296Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18444942336,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 41384689152,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142524547072,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 142524547072,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 962,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 962,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:49.097168Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T192349089590Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12265aafa532e83b466dd405387b287217c43b5f574287368cbd7661d56b9682
3
+ size 342163
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 38991856640,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123795095040,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 38991856640,
8
+ "active_bytes_peak": 123795095040,
9
+ "allocated_bytes_current": 38991856640,
10
+ "allocated_bytes_peak": 123795095040,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 18444942336,
13
+ "inactive_split_bytes_peak": 41384689152,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 142524547072,
21
+ "reserved_bytes_peak": 142524547072,
22
+ "segment_count_current": 962,
23
+ "segment_count_peak": 962,
24
+ "timestamp_utc": "2026-08-26T19:23:49.684043Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 38991856640,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123795095040,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 38991856640,
33
+ "active_bytes_peak": 123795095040,
34
+ "allocated_bytes_current": 38991856640,
35
+ "allocated_bytes_peak": 123795095040,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 18444942336,
38
+ "inactive_split_bytes_peak": 41384689152,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 142524547072,
46
+ "reserved_bytes_peak": 142524547072,
47
+ "segment_count_current": 962,
48
+ "segment_count_peak": 962,
49
+ "timestamp_utc": "2026-08-26T19:23:49.098209Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18444942336,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 41384689152,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142524547072,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 142524547072,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 962,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 962,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:23:49.692003Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T192349684418Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0ecac16ad4203410bfbb2d55f36c425f98f921cc60078918955a8745939a1f7
3
+ size 342163
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-better-memory-dump-support
6
+ commit=c3a8a8a7
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=1.25
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=memory
48
+ memory_dump_mode=iteration
49
+ memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=6
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=768
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff