AbhishekGhosh1998 commited on
Commit
bca31b2
·
verified ·
1 Parent(s): 738718b

granularity results ultraep/tierbmem @ 2026-08-26T19:21:12Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv +152 -0
  2. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/run.env +61 -0
  3. ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log +0 -0
  4. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv +176 -0
  5. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.json +79 -0
  6. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.pickle +3 -0
  7. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.json +79 -0
  8. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.pickle +3 -0
  9. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.json +79 -0
  10. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.pickle +3 -0
  11. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.json +79 -0
  12. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.pickle +3 -0
  13. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.json +79 -0
  14. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.pickle +3 -0
  15. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.json +79 -0
  16. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.pickle +3 -0
  17. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.json +79 -0
  18. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.pickle +3 -0
  19. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.json +79 -0
  20. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.pickle +3 -0
  21. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.json +79 -0
  22. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.pickle +3 -0
  23. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.json +79 -0
  24. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.pickle +3 -0
  25. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.json +79 -0
  26. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.pickle +3 -0
  27. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.json +79 -0
  28. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.pickle +3 -0
  29. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.json +79 -0
  30. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.pickle +3 -0
  31. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.json +79 -0
  32. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.pickle +3 -0
  33. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.json +79 -0
  34. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.pickle +3 -0
  35. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.json +79 -0
  36. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.pickle +3 -0
  37. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.json +79 -0
  38. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.pickle +3 -0
  39. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.json +79 -0
  40. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.pickle +3 -0
  41. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.json +79 -0
  42. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.pickle +3 -0
  43. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.json +79 -0
  44. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.pickle +3 -0
  45. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.json +79 -0
  46. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.pickle +3 -0
  47. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.json +79 -0
  48. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.pickle +3 -0
  49. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.json +79 -0
  50. ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.pickle +3 -0
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 0
10
+ 1, 0
11
+ 2, 0
12
+ 3, 0
13
+ 4, 0
14
+ 5, 0
15
+ 6, 0
16
+ 7, 0
17
+ 0, 1
18
+ 1, 1
19
+ 2, 1
20
+ 3, 1
21
+ 4, 1
22
+ 5, 1
23
+ 6, 1
24
+ 7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 4
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 4
42
+ 1, 4
43
+ 2, 4
44
+ 3, 4
45
+ 4, 4
46
+ 5, 4
47
+ 6, 4
48
+ 7, 24
49
+ 0, 899
50
+ 1, 899
51
+ 2, 901
52
+ 3, 913
53
+ 4, 903
54
+ 5, 909
55
+ 6, 909
56
+ 7, 901
57
+ 0, 1955
58
+ 1, 1955
59
+ 2, 1955
60
+ 3, 1955
61
+ 4, 1955
62
+ 5, 1955
63
+ 6, 1955
64
+ 7, 1955
65
+ 0, 2411
66
+ 1, 2405
67
+ 2, 2413
68
+ 3, 2409
69
+ 4, 2417
70
+ 5, 2427
71
+ 6, 2417
72
+ 7, 2413
73
+ 0, 25239
74
+ 1, 25239
75
+ 2, 25239
76
+ 3, 25239
77
+ 4, 25239
78
+ 5, 25239
79
+ 6, 25239
80
+ 7, 25239
81
+ 0, 28863
82
+ 1, 28929
83
+ 2, 28773
84
+ 3, 28787
85
+ 4, 28781
86
+ 5, 28761
87
+ 6, 28915
88
+ 7, 28997
89
+ 0, 38975
90
+ 1, 39047
91
+ 2, 38905
92
+ 3, 38869
93
+ 4, 38897
94
+ 5, 38895
95
+ 6, 39009
96
+ 7, 39049
97
+ 0, 39773
98
+ 1, 39857
99
+ 2, 39707
100
+ 3, 39705
101
+ 4, 39705
102
+ 5, 39703
103
+ 6, 39839
104
+ 7, 39857
105
+ 0, 39773
106
+ 1, 39857
107
+ 2, 39707
108
+ 3, 39705
109
+ 4, 39705
110
+ 5, 39703
111
+ 6, 39839
112
+ 7, 39857
113
+ 0, 39783
114
+ 1, 39867
115
+ 2, 39717
116
+ 3, 39717
117
+ 4, 39717
118
+ 5, 39713
119
+ 6, 39849
120
+ 7, 39867
121
+ 0, 39783
122
+ 1, 39867
123
+ 2, 39717
124
+ 3, 39717
125
+ 4, 39717
126
+ 5, 39713
127
+ 6, 39849
128
+ 7, 39867
129
+ 0, 109499
130
+ 1, 109597
131
+ 2, 109441
132
+ 3, 109445
133
+ 4, 109451
134
+ 5, 109447
135
+ 6, 109573
136
+ 7, 109607
137
+ 0, 110101
138
+ 1, 110185
139
+ 2, 110035
140
+ 3, 110035
141
+ 4, 110035
142
+ 5, 110031
143
+ 6, 110167
144
+ 7, 110185
145
+ 0, 110105
146
+ 1, 110191
147
+ 2, 110041
148
+ 3, 110039
149
+ 4, 110039
150
+ 5, 110037
151
+ 6, 110171
152
+ 7, 110191
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-UltraEP
5
+ branch=ultraep-better-memory-dump-support
6
+ commit=c3a8a8a7
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=1.00
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=memory
48
+ memory_dump_mode=iteration
49
+ memory_snapshot_path=/workspace/tierb_study_20260826_144644/ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot.pickle
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=6
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=768
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
ultraep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/mem_samples.csv CHANGED
@@ -22,3 +22,179 @@
22
  5, 1
23
  6, 1
24
  7, 1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  5, 1
23
  6, 1
24
  7, 1
25
+ 0, 4
26
+ 1, 4
27
+ 2, 4
28
+ 3, 4
29
+ 4, 1
30
+ 5, 1
31
+ 6, 1
32
+ 7, 1
33
+ 0, 4
34
+ 1, 4
35
+ 2, 4
36
+ 3, 27
37
+ 4, 4
38
+ 5, 4
39
+ 6, 4
40
+ 7, 4
41
+ 0, 821
42
+ 1, 819
43
+ 2, 819
44
+ 3, 819
45
+ 4, 819
46
+ 5, 819
47
+ 6, 819
48
+ 7, 819
49
+ 0, 1537
50
+ 1, 1537
51
+ 2, 1537
52
+ 3, 1537
53
+ 4, 1537
54
+ 5, 1537
55
+ 6, 1537
56
+ 7, 1537
57
+ 0, 2551
58
+ 1, 2551
59
+ 2, 2551
60
+ 3, 2551
61
+ 4, 2551
62
+ 5, 2551
63
+ 6, 2551
64
+ 7, 2551
65
+ 0, 2959
66
+ 1, 2959
67
+ 2, 2959
68
+ 3, 2959
69
+ 4, 2959
70
+ 5, 2959
71
+ 6, 2959
72
+ 7, 2959
73
+ 0, 51537
74
+ 1, 51537
75
+ 2, 51537
76
+ 3, 51537
77
+ 4, 51249
78
+ 5, 51537
79
+ 6, 51537
80
+ 7, 51537
81
+ 0, 53019
82
+ 1, 53019
83
+ 2, 53017
84
+ 3, 53019
85
+ 4, 52729
86
+ 5, 53019
87
+ 6, 53017
88
+ 7, 53019
89
+ 0, 54651
90
+ 1, 54651
91
+ 2, 54651
92
+ 3, 54651
93
+ 4, 54363
94
+ 5, 54651
95
+ 6, 54651
96
+ 7, 54651
97
+ 0, 55163
98
+ 1, 55163
99
+ 2, 55163
100
+ 3, 55163
101
+ 4, 54875
102
+ 5, 55163
103
+ 6, 55163
104
+ 7, 55163
105
+ 0, 55163
106
+ 1, 55163
107
+ 2, 55163
108
+ 3, 55163
109
+ 4, 54875
110
+ 5, 55163
111
+ 6, 55163
112
+ 7, 55163
113
+ 0, 56699
114
+ 1, 56699
115
+ 2, 56699
116
+ 3, 56699
117
+ 4, 56411
118
+ 5, 56699
119
+ 6, 56699
120
+ 7, 56699
121
+ 0, 87805
122
+ 1, 87845
123
+ 2, 87845
124
+ 3, 87845
125
+ 4, 87841
126
+ 5, 87845
127
+ 6, 87845
128
+ 7, 87845
129
+ 0, 88517
130
+ 1, 88557
131
+ 2, 88557
132
+ 3, 88557
133
+ 4, 88553
134
+ 5, 88557
135
+ 6, 88557
136
+ 7, 88557
137
+ 0, 88517
138
+ 1, 88557
139
+ 2, 88557
140
+ 3, 88557
141
+ 4, 88553
142
+ 5, 88557
143
+ 6, 88557
144
+ 7, 88557
145
+ 0, 88517
146
+ 1, 88557
147
+ 2, 88557
148
+ 3, 88557
149
+ 4, 88553
150
+ 5, 88557
151
+ 6, 88557
152
+ 7, 88557
153
+ 0, 88921
154
+ 1, 88961
155
+ 2, 88961
156
+ 3, 88961
157
+ 4, 88957
158
+ 5, 88961
159
+ 6, 88961
160
+ 7, 88961
161
+ 0, 95725
162
+ 1, 95789
163
+ 2, 95721
164
+ 3, 95743
165
+ 4, 95923
166
+ 5, 95779
167
+ 6, 95741
168
+ 7, 95759
169
+ 0, 111083
170
+ 1, 113027
171
+ 2, 114123
172
+ 3, 111903
173
+ 4, 110463
174
+ 5, 116199
175
+ 6, 108963
176
+ 7, 111075
177
+ 0, 89083
178
+ 1, 89237
179
+ 2, 87617
180
+ 3, 92987
181
+ 4, 91423
182
+ 5, 90119
183
+ 6, 90181
184
+ 7, 88643
185
+ 0, 121025
186
+ 1, 121181
187
+ 2, 121181
188
+ 3, 121181
189
+ 4, 121321
190
+ 5, 121181
191
+ 6, 121181
192
+ 7, 121181
193
+ 0, 5275
194
+ 1, 3678
195
+ 2, 3678
196
+ 3, 3678
197
+ 4, 3678
198
+ 5, 3678
199
+ 6, 3998
200
+ 7, 121179
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77298383360,
3
+ "active_bytes_current_delta": 32777583104,
4
+ "active_bytes_peak": 82477897216,
5
+ "active_bytes_peak_delta": 37931931136,
6
+ "after": {
7
+ "active_bytes_current": 77298383360,
8
+ "active_bytes_peak": 82477897216,
9
+ "allocated_bytes_current": 77298383360,
10
+ "allocated_bytes_peak": 82477897216,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1206404608,
13
+ "inactive_split_bytes_peak": 9362196992,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 90250936320,
21
+ "reserved_bytes_peak": 90250936320,
22
+ "segment_count_current": 1714,
23
+ "segment_count_peak": 1714,
24
+ "timestamp_utc": "2026-08-26T19:11:43.968857Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77298383360,
28
+ "allocated_bytes_current_delta": 32777583104,
29
+ "allocated_bytes_peak": 82477897216,
30
+ "allocated_bytes_peak_delta": 37931931136,
31
+ "before": {
32
+ "active_bytes_current": 44520800256,
33
+ "active_bytes_peak": 44545966080,
34
+ "allocated_bytes_current": 44520800256,
35
+ "allocated_bytes_peak": 44545966080,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 446332928,
38
+ "inactive_split_bytes_peak": 1050312704,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T19:11:17.937284Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1206404608,
55
+ "inactive_split_bytes_current_delta": 760071680,
56
+ "inactive_split_bytes_peak": 9362196992,
57
+ "inactive_split_bytes_peak_delta": 8311884288,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 90250936320,
68
+ "reserved_bytes_current_delta": 41999663104,
69
+ "reserved_bytes_peak": 90250936320,
70
+ "reserved_bytes_peak_delta": 41999663104,
71
+ "segment_count_current": 1714,
72
+ "segment_count_current_delta": 1093,
73
+ "segment_count_peak": 1714,
74
+ "segment_count_peak_delta": 1093,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:44.009882Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T191143969332Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:366506f1eecdad1148894ecfb8f4a0105384eba9793cdc997f78dc0f25a97f89
3
+ size 2316617
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77298383360,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113343831040,
5
+ "active_bytes_peak_delta": 30865933824,
6
+ "after": {
7
+ "active_bytes_current": 77298383360,
8
+ "active_bytes_peak": 113343831040,
9
+ "allocated_bytes_current": 77298383360,
10
+ "allocated_bytes_peak": 113343831040,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1206404608,
13
+ "inactive_split_bytes_peak": 9362196992,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 115242696704,
21
+ "reserved_bytes_peak": 115242696704,
22
+ "segment_count_current": 1923,
23
+ "segment_count_peak": 1923,
24
+ "timestamp_utc": "2026-08-26T19:11:44.855042Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77298383360,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113343831040,
30
+ "allocated_bytes_peak_delta": 30865933824,
31
+ "before": {
32
+ "active_bytes_current": 77298383360,
33
+ "active_bytes_peak": 82477897216,
34
+ "allocated_bytes_current": 77298383360,
35
+ "allocated_bytes_peak": 82477897216,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1206404608,
38
+ "inactive_split_bytes_peak": 9362196992,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 90250936320,
46
+ "reserved_bytes_peak": 90250936320,
47
+ "segment_count_current": 1714,
48
+ "segment_count_peak": 1714,
49
+ "timestamp_utc": "2026-08-26T19:11:44.014581Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1206404608,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 9362196992,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 115242696704,
68
+ "reserved_bytes_current_delta": 24991760384,
69
+ "reserved_bytes_peak": 115242696704,
70
+ "reserved_bytes_peak_delta": 24991760384,
71
+ "segment_count_current": 1923,
72
+ "segment_count_current_delta": 209,
73
+ "segment_count_peak": 1923,
74
+ "segment_count_peak_delta": 209,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:44.891762Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T191144855486Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9d644fd17905670a38a6b94b76cb0b9e6d9eeb7d69c1392488c0adbc3ebceca
3
+ size 1844220
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77298383360,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113343831040,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77298383360,
8
+ "active_bytes_peak": 113343831040,
9
+ "allocated_bytes_current": 77298383360,
10
+ "allocated_bytes_peak": 113343831040,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1206404608,
13
+ "inactive_split_bytes_peak": 9362196992,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 115242696704,
21
+ "reserved_bytes_peak": 115242696704,
22
+ "segment_count_current": 1923,
23
+ "segment_count_peak": 1923,
24
+ "timestamp_utc": "2026-08-26T19:11:45.209078Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77298383360,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113343831040,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77298383360,
33
+ "active_bytes_peak": 113343831040,
34
+ "allocated_bytes_current": 77298383360,
35
+ "allocated_bytes_peak": 113343831040,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1206404608,
38
+ "inactive_split_bytes_peak": 9362196992,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 115242696704,
46
+ "reserved_bytes_peak": 115242696704,
47
+ "segment_count_current": 1923,
48
+ "segment_count_peak": 1923,
49
+ "timestamp_utc": "2026-08-26T19:11:44.895755Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1206404608,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 9362196992,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 115242696704,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 115242696704,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1923,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1923,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:45.240454Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T191145209517Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac7edbe46c6155a1d9a064c5c24884e487feb90b7fc1401f2f098c7baecb476d
3
+ size 1839099
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77315426816,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 113351470080,
5
+ "active_bytes_peak_delta": 7639040,
6
+ "after": {
7
+ "active_bytes_current": 77315426816,
8
+ "active_bytes_peak": 113351470080,
9
+ "allocated_bytes_current": 77315426816,
10
+ "allocated_bytes_peak": 113351470080,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1315190272,
13
+ "inactive_split_bytes_peak": 10511450112,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 116230455296,
21
+ "reserved_bytes_peak": 116230455296,
22
+ "segment_count_current": 1696,
23
+ "segment_count_peak": 1923,
24
+ "timestamp_utc": "2026-08-26T19:11:47.859107Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77315426816,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 113351470080,
30
+ "allocated_bytes_peak_delta": 7639040,
31
+ "before": {
32
+ "active_bytes_current": 77298383360,
33
+ "active_bytes_peak": 113343831040,
34
+ "allocated_bytes_current": 77298383360,
35
+ "allocated_bytes_peak": 113343831040,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1206404608,
38
+ "inactive_split_bytes_peak": 9362196992,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 115242696704,
46
+ "reserved_bytes_peak": 115242696704,
47
+ "segment_count_current": 1923,
48
+ "segment_count_peak": 1923,
49
+ "timestamp_utc": "2026-08-26T19:11:45.244071Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1315190272,
55
+ "inactive_split_bytes_current_delta": 108785664,
56
+ "inactive_split_bytes_peak": 10511450112,
57
+ "inactive_split_bytes_peak_delta": 1149253120,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 116230455296,
68
+ "reserved_bytes_current_delta": 987758592,
69
+ "reserved_bytes_peak": 116230455296,
70
+ "reserved_bytes_peak_delta": 987758592,
71
+ "segment_count_current": 1696,
72
+ "segment_count_current_delta": -227,
73
+ "segment_count_peak": 1923,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:47.897560Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T191147859661Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:189e0599ddd5f751d4966b5bc19cf04a1f32edf44107bc89f91ca9060c9bee95
3
+ size 2181131
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77315426816,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113351470080,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77315426816,
8
+ "active_bytes_peak": 113351470080,
9
+ "allocated_bytes_current": 77315426816,
10
+ "allocated_bytes_peak": 113351470080,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1315190272,
13
+ "inactive_split_bytes_peak": 10511450112,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 116230455296,
21
+ "reserved_bytes_peak": 116230455296,
22
+ "segment_count_current": 1696,
23
+ "segment_count_peak": 1923,
24
+ "timestamp_utc": "2026-08-26T19:11:48.200437Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77315426816,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113351470080,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77315426816,
33
+ "active_bytes_peak": 113351470080,
34
+ "allocated_bytes_current": 77315426816,
35
+ "allocated_bytes_peak": 113351470080,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1315190272,
38
+ "inactive_split_bytes_peak": 10511450112,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 116230455296,
46
+ "reserved_bytes_peak": 116230455296,
47
+ "segment_count_current": 1696,
48
+ "segment_count_peak": 1923,
49
+ "timestamp_utc": "2026-08-26T19:11:47.901776Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1315190272,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 10511450112,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 116230455296,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 116230455296,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1696,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1923,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:48.209298Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T191148200823Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4eecba693a4d72b91d813578a83b637d48f90ddfe5cfcbd277eecd5701aa4ce2
3
+ size 418854
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77315426816,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113351470080,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77315426816,
8
+ "active_bytes_peak": 113351470080,
9
+ "allocated_bytes_current": 77315426816,
10
+ "allocated_bytes_peak": 113351470080,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1315190272,
13
+ "inactive_split_bytes_peak": 10511450112,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 116230455296,
21
+ "reserved_bytes_peak": 116230455296,
22
+ "segment_count_current": 1696,
23
+ "segment_count_peak": 1923,
24
+ "timestamp_utc": "2026-08-26T19:11:48.509226Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77315426816,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113351470080,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77315426816,
33
+ "active_bytes_peak": 113351470080,
34
+ "allocated_bytes_current": 77315426816,
35
+ "allocated_bytes_peak": 113351470080,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1315190272,
38
+ "inactive_split_bytes_peak": 10511450112,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 116230455296,
46
+ "reserved_bytes_peak": 116230455296,
47
+ "segment_count_current": 1696,
48
+ "segment_count_peak": 1923,
49
+ "timestamp_utc": "2026-08-26T19:11:48.210418Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1315190272,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 10511450112,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 116230455296,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 116230455296,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1696,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1923,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:48.518016Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T191148509603Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a6184337a698a26e0528082a885307cd77aaee32329aa3ad51f78c8bc74eb1ac
3
+ size 418854
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77299558912,
3
+ "active_bytes_current_delta": 32778758656,
4
+ "active_bytes_peak": 82480875008,
5
+ "active_bytes_peak_delta": 37934908928,
6
+ "after": {
7
+ "active_bytes_current": 77299558912,
8
+ "active_bytes_peak": 82480875008,
9
+ "allocated_bytes_current": 77299558912,
10
+ "allocated_bytes_peak": 82480875008,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1368806912,
13
+ "inactive_split_bytes_peak": 9548864000,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 90267713536,
21
+ "reserved_bytes_peak": 90267713536,
22
+ "segment_count_current": 1718,
23
+ "segment_count_peak": 1718,
24
+ "timestamp_utc": "2026-08-26T19:11:43.968739Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77299558912,
28
+ "allocated_bytes_current_delta": 32778758656,
29
+ "allocated_bytes_peak": 82480875008,
30
+ "allocated_bytes_peak_delta": 37934908928,
31
+ "before": {
32
+ "active_bytes_current": 44520800256,
33
+ "active_bytes_peak": 44545966080,
34
+ "allocated_bytes_current": 44520800256,
35
+ "allocated_bytes_peak": 44545966080,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 421167104,
38
+ "inactive_split_bytes_peak": 1025146880,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T19:11:17.937156Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1368806912,
55
+ "inactive_split_bytes_current_delta": 947639808,
56
+ "inactive_split_bytes_peak": 9548864000,
57
+ "inactive_split_bytes_peak_delta": 8523717120,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 90267713536,
68
+ "reserved_bytes_current_delta": 42016440320,
69
+ "reserved_bytes_peak": 90267713536,
70
+ "reserved_bytes_peak_delta": 42016440320,
71
+ "segment_count_current": 1718,
72
+ "segment_count_current_delta": 1097,
73
+ "segment_count_peak": 1718,
74
+ "segment_count_peak_delta": 1097,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:44.009111Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T191143969245Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd63bf0418f543d422370e8d8d4fe5ffdc6f159df955028160cd3a2d0c88c29f
3
+ size 2304358
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77299558912,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113344452608,
5
+ "active_bytes_peak_delta": 30863577600,
6
+ "after": {
7
+ "active_bytes_current": 77299558912,
8
+ "active_bytes_peak": 113344452608,
9
+ "allocated_bytes_current": 77299558912,
10
+ "allocated_bytes_peak": 113344452608,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1368806912,
13
+ "inactive_split_bytes_peak": 9548864000,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 115364331520,
21
+ "reserved_bytes_peak": 115364331520,
22
+ "segment_count_current": 1934,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:44.855039Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77299558912,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113344452608,
30
+ "allocated_bytes_peak_delta": 30863577600,
31
+ "before": {
32
+ "active_bytes_current": 77299558912,
33
+ "active_bytes_peak": 82480875008,
34
+ "allocated_bytes_current": 77299558912,
35
+ "allocated_bytes_peak": 82480875008,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 1368806912,
38
+ "inactive_split_bytes_peak": 9548864000,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 90267713536,
46
+ "reserved_bytes_peak": 90267713536,
47
+ "segment_count_current": 1718,
48
+ "segment_count_peak": 1718,
49
+ "timestamp_utc": "2026-08-26T19:11:44.013716Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1368806912,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 9548864000,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 115364331520,
68
+ "reserved_bytes_current_delta": 25096617984,
69
+ "reserved_bytes_peak": 115364331520,
70
+ "reserved_bytes_peak_delta": 25096617984,
71
+ "segment_count_current": 1934,
72
+ "segment_count_current_delta": 216,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 216,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:44.892389Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T191144855476Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ecc694e58f210966e555161d3a63de0fdffbbb234ab434109bef0b153934e320
3
+ size 1833396
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77299558912,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113344452608,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77299558912,
8
+ "active_bytes_peak": 113344452608,
9
+ "allocated_bytes_current": 77299558912,
10
+ "allocated_bytes_peak": 113344452608,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1368806912,
13
+ "inactive_split_bytes_peak": 9548864000,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 115364331520,
21
+ "reserved_bytes_peak": 115364331520,
22
+ "segment_count_current": 1934,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:45.209053Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77299558912,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113344452608,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77299558912,
33
+ "active_bytes_peak": 113344452608,
34
+ "allocated_bytes_current": 77299558912,
35
+ "allocated_bytes_peak": 113344452608,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 1368806912,
38
+ "inactive_split_bytes_peak": 9548864000,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 115364331520,
46
+ "reserved_bytes_peak": 115364331520,
47
+ "segment_count_current": 1934,
48
+ "segment_count_peak": 1934,
49
+ "timestamp_utc": "2026-08-26T19:11:44.896325Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1368806912,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 9548864000,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 115364331520,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 115364331520,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1934,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:45.239689Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T191145209458Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb9fe89f6b89e5abf53df921b3b39cce8a047024604744279461e1fd96a10be0
3
+ size 1828450
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77316602368,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 113352645632,
5
+ "active_bytes_peak_delta": 8193024,
6
+ "after": {
7
+ "active_bytes_current": 77316602368,
8
+ "active_bytes_peak": 113352645632,
9
+ "allocated_bytes_current": 77316602368,
10
+ "allocated_bytes_peak": 113352645632,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1477592576,
13
+ "inactive_split_bytes_peak": 10673852416,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 116394033152,
21
+ "reserved_bytes_peak": 116394033152,
22
+ "segment_count_current": 1701,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:47.858997Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77316602368,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 113352645632,
30
+ "allocated_bytes_peak_delta": 8193024,
31
+ "before": {
32
+ "active_bytes_current": 77299558912,
33
+ "active_bytes_peak": 113344452608,
34
+ "allocated_bytes_current": 77299558912,
35
+ "allocated_bytes_peak": 113344452608,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 1368806912,
38
+ "inactive_split_bytes_peak": 9548864000,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 115364331520,
46
+ "reserved_bytes_peak": 115364331520,
47
+ "segment_count_current": 1934,
48
+ "segment_count_peak": 1934,
49
+ "timestamp_utc": "2026-08-26T19:11:45.243253Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1477592576,
55
+ "inactive_split_bytes_current_delta": 108785664,
56
+ "inactive_split_bytes_peak": 10673852416,
57
+ "inactive_split_bytes_peak_delta": 1124988416,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 116394033152,
68
+ "reserved_bytes_current_delta": 1029701632,
69
+ "reserved_bytes_peak": 116394033152,
70
+ "reserved_bytes_peak_delta": 1029701632,
71
+ "segment_count_current": 1701,
72
+ "segment_count_current_delta": -233,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:47.897085Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T191147859527Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:45316b56ee65848ab6595cb2c9fe9646bc514f9e4526201f8bd7952224ce759a
3
+ size 2193727
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77316602368,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113352645632,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77316602368,
8
+ "active_bytes_peak": 113352645632,
9
+ "allocated_bytes_current": 77316602368,
10
+ "allocated_bytes_peak": 113352645632,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1477592576,
13
+ "inactive_split_bytes_peak": 10673852416,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 116394033152,
21
+ "reserved_bytes_peak": 116394033152,
22
+ "segment_count_current": 1701,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:48.200462Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77316602368,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113352645632,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77316602368,
33
+ "active_bytes_peak": 113352645632,
34
+ "allocated_bytes_current": 77316602368,
35
+ "allocated_bytes_peak": 113352645632,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 1477592576,
38
+ "inactive_split_bytes_peak": 10673852416,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 116394033152,
46
+ "reserved_bytes_peak": 116394033152,
47
+ "segment_count_current": 1701,
48
+ "segment_count_peak": 1934,
49
+ "timestamp_utc": "2026-08-26T19:11:47.901088Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1477592576,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 10673852416,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 116394033152,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 116394033152,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1701,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:48.209181Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T191148200859Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05272f421f21a224b022b0c8f2193e5cd7d2b5f296dee60b14863343931dce7b
3
+ size 411770
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77316602368,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113352645632,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77316602368,
8
+ "active_bytes_peak": 113352645632,
9
+ "allocated_bytes_current": 77316602368,
10
+ "allocated_bytes_peak": 113352645632,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1477592576,
13
+ "inactive_split_bytes_peak": 10673852416,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 116394033152,
21
+ "reserved_bytes_peak": 116394033152,
22
+ "segment_count_current": 1701,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:48.509218Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77316602368,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113352645632,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77316602368,
33
+ "active_bytes_peak": 113352645632,
34
+ "allocated_bytes_current": 77316602368,
35
+ "allocated_bytes_peak": 113352645632,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 1477592576,
38
+ "inactive_split_bytes_peak": 10673852416,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 116394033152,
46
+ "reserved_bytes_peak": 116394033152,
47
+ "segment_count_current": 1701,
48
+ "segment_count_peak": 1934,
49
+ "timestamp_utc": "2026-08-26T19:11:48.210270Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1477592576,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 10673852416,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 116394033152,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 116394033152,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1701,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:48.518027Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T191148509603Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dc4c76682af535734ff96acf66ab2758651d9119bab6ea704c4c024e0cab9e14
3
+ size 411770
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77299558912,
3
+ "active_bytes_current_delta": 32778758656,
4
+ "active_bytes_peak": 82480875008,
5
+ "active_bytes_peak_delta": 37934908928,
6
+ "after": {
7
+ "active_bytes_current": 77299558912,
8
+ "active_bytes_peak": 82480875008,
9
+ "allocated_bytes_current": 77299558912,
10
+ "allocated_bytes_peak": 82480875008,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1368806912,
13
+ "inactive_split_bytes_peak": 9548864000,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 90267713536,
21
+ "reserved_bytes_peak": 90267713536,
22
+ "segment_count_current": 1718,
23
+ "segment_count_peak": 1718,
24
+ "timestamp_utc": "2026-08-26T19:11:43.968843Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77299558912,
28
+ "allocated_bytes_current_delta": 32778758656,
29
+ "allocated_bytes_peak": 82480875008,
30
+ "allocated_bytes_peak_delta": 37934908928,
31
+ "before": {
32
+ "active_bytes_current": 44520800256,
33
+ "active_bytes_peak": 44545966080,
34
+ "allocated_bytes_current": 44520800256,
35
+ "allocated_bytes_peak": 44545966080,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 421167104,
38
+ "inactive_split_bytes_peak": 1025146880,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T19:11:17.937342Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1368806912,
55
+ "inactive_split_bytes_current_delta": 947639808,
56
+ "inactive_split_bytes_peak": 9548864000,
57
+ "inactive_split_bytes_peak_delta": 8523717120,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 90267713536,
68
+ "reserved_bytes_current_delta": 42016440320,
69
+ "reserved_bytes_peak": 90267713536,
70
+ "reserved_bytes_peak_delta": 42016440320,
71
+ "segment_count_current": 1718,
72
+ "segment_count_current_delta": 1097,
73
+ "segment_count_peak": 1718,
74
+ "segment_count_peak_delta": 1097,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:44.009608Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T191143969307Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:815384b03107533a05df19ec1085b9480eefa54d8bd0488516d73ea922bb3ecd
3
+ size 2304358
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77299558912,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113344452608,
5
+ "active_bytes_peak_delta": 30863577600,
6
+ "after": {
7
+ "active_bytes_current": 77299558912,
8
+ "active_bytes_peak": 113344452608,
9
+ "allocated_bytes_current": 77299558912,
10
+ "allocated_bytes_peak": 113344452608,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1368806912,
13
+ "inactive_split_bytes_peak": 9548864000,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 115364331520,
21
+ "reserved_bytes_peak": 115364331520,
22
+ "segment_count_current": 1934,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:44.855177Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77299558912,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113344452608,
30
+ "allocated_bytes_peak_delta": 30863577600,
31
+ "before": {
32
+ "active_bytes_current": 77299558912,
33
+ "active_bytes_peak": 82480875008,
34
+ "allocated_bytes_current": 77299558912,
35
+ "allocated_bytes_peak": 82480875008,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1368806912,
38
+ "inactive_split_bytes_peak": 9548864000,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 90267713536,
46
+ "reserved_bytes_peak": 90267713536,
47
+ "segment_count_current": 1718,
48
+ "segment_count_peak": 1718,
49
+ "timestamp_utc": "2026-08-26T19:11:44.014261Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1368806912,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 9548864000,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 115364331520,
68
+ "reserved_bytes_current_delta": 25096617984,
69
+ "reserved_bytes_peak": 115364331520,
70
+ "reserved_bytes_peak_delta": 25096617984,
71
+ "segment_count_current": 1934,
72
+ "segment_count_current_delta": 216,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 216,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:44.892180Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T191144855609Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0e924ed7d72f88dd565acd94a3993cca8869ec8eaf1d717b05d561c9cf5939f
3
+ size 1833399
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77299558912,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113344452608,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77299558912,
8
+ "active_bytes_peak": 113344452608,
9
+ "allocated_bytes_current": 77299558912,
10
+ "allocated_bytes_peak": 113344452608,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1368806912,
13
+ "inactive_split_bytes_peak": 9548864000,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 115364331520,
21
+ "reserved_bytes_peak": 115364331520,
22
+ "segment_count_current": 1934,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:45.209140Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77299558912,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113344452608,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77299558912,
33
+ "active_bytes_peak": 113344452608,
34
+ "allocated_bytes_current": 77299558912,
35
+ "allocated_bytes_peak": 113344452608,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1368806912,
38
+ "inactive_split_bytes_peak": 9548864000,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 115364331520,
46
+ "reserved_bytes_peak": 115364331520,
47
+ "segment_count_current": 1934,
48
+ "segment_count_peak": 1934,
49
+ "timestamp_utc": "2026-08-26T19:11:44.896151Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1368806912,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 9548864000,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 115364331520,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 115364331520,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1934,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:45.240203Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T191145209564Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d8f6f12368938c04a7a33415f661c3e6e261df0a86be8fe035119b6f60a55ce
3
+ size 1828562
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77316602368,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 113352645632,
5
+ "active_bytes_peak_delta": 8193024,
6
+ "after": {
7
+ "active_bytes_current": 77316602368,
8
+ "active_bytes_peak": 113352645632,
9
+ "allocated_bytes_current": 77316602368,
10
+ "allocated_bytes_peak": 113352645632,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1477592576,
13
+ "inactive_split_bytes_peak": 10673852416,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 116394033152,
21
+ "reserved_bytes_peak": 116394033152,
22
+ "segment_count_current": 1701,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:47.859142Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77316602368,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 113352645632,
30
+ "allocated_bytes_peak_delta": 8193024,
31
+ "before": {
32
+ "active_bytes_current": 77299558912,
33
+ "active_bytes_peak": 113344452608,
34
+ "allocated_bytes_current": 77299558912,
35
+ "allocated_bytes_peak": 113344452608,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1368806912,
38
+ "inactive_split_bytes_peak": 9548864000,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 115364331520,
46
+ "reserved_bytes_peak": 115364331520,
47
+ "segment_count_current": 1934,
48
+ "segment_count_peak": 1934,
49
+ "timestamp_utc": "2026-08-26T19:11:45.243780Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1477592576,
55
+ "inactive_split_bytes_current_delta": 108785664,
56
+ "inactive_split_bytes_peak": 10673852416,
57
+ "inactive_split_bytes_peak_delta": 1124988416,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 116394033152,
68
+ "reserved_bytes_current_delta": 1029701632,
69
+ "reserved_bytes_peak": 116394033152,
70
+ "reserved_bytes_peak_delta": 1029701632,
71
+ "segment_count_current": 1701,
72
+ "segment_count_current_delta": -233,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:47.897687Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T191147859711Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:be9b4543ec95a2ba876ec647c3bf34f42c3c7d64e3aa58f3bc5c440b6f7afeb3
3
+ size 2193889
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77316602368,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113352645632,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77316602368,
8
+ "active_bytes_peak": 113352645632,
9
+ "allocated_bytes_current": 77316602368,
10
+ "allocated_bytes_peak": 113352645632,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1477592576,
13
+ "inactive_split_bytes_peak": 10673852416,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 116394033152,
21
+ "reserved_bytes_peak": 116394033152,
22
+ "segment_count_current": 1701,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:48.200546Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77316602368,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113352645632,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77316602368,
33
+ "active_bytes_peak": 113352645632,
34
+ "allocated_bytes_current": 77316602368,
35
+ "allocated_bytes_peak": 113352645632,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1477592576,
38
+ "inactive_split_bytes_peak": 10673852416,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 116394033152,
46
+ "reserved_bytes_peak": 116394033152,
47
+ "segment_count_current": 1701,
48
+ "segment_count_peak": 1934,
49
+ "timestamp_utc": "2026-08-26T19:11:47.902545Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1477592576,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 10673852416,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 116394033152,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 116394033152,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1701,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:48.209332Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T191148200943Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3a4d3b1d51744d2bb80d93fbaf542d0cd7ef040974de68a67e6b149c6e0dcdc
3
+ size 411608
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77316602368,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113352645632,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77316602368,
8
+ "active_bytes_peak": 113352645632,
9
+ "allocated_bytes_current": 77316602368,
10
+ "allocated_bytes_peak": 113352645632,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1477592576,
13
+ "inactive_split_bytes_peak": 10673852416,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 116394033152,
21
+ "reserved_bytes_peak": 116394033152,
22
+ "segment_count_current": 1701,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:48.509293Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77316602368,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113352645632,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77316602368,
33
+ "active_bytes_peak": 113352645632,
34
+ "allocated_bytes_current": 77316602368,
35
+ "allocated_bytes_peak": 113352645632,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1477592576,
38
+ "inactive_split_bytes_peak": 10673852416,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 116394033152,
46
+ "reserved_bytes_peak": 116394033152,
47
+ "segment_count_current": 1701,
48
+ "segment_count_peak": 1934,
49
+ "timestamp_utc": "2026-08-26T19:11:48.210429Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1477592576,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 10673852416,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 116394033152,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 116394033152,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1701,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:48.518004Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T191148509664Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:964457ab0632888b1b0e58ff3c1f51b88f659ffb6b3f8e3cbdb133b3f8d503ad
3
+ size 411608
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77299558912,
3
+ "active_bytes_current_delta": 32778758656,
4
+ "active_bytes_peak": 82480875008,
5
+ "active_bytes_peak_delta": 37934908928,
6
+ "after": {
7
+ "active_bytes_current": 77299558912,
8
+ "active_bytes_peak": 82480875008,
9
+ "allocated_bytes_current": 77299558912,
10
+ "allocated_bytes_peak": 82480875008,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 1368806912,
13
+ "inactive_split_bytes_peak": 9548864000,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 90267713536,
21
+ "reserved_bytes_peak": 90267713536,
22
+ "segment_count_current": 1718,
23
+ "segment_count_peak": 1718,
24
+ "timestamp_utc": "2026-08-26T19:11:43.968838Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77299558912,
28
+ "allocated_bytes_current_delta": 32778758656,
29
+ "allocated_bytes_peak": 82480875008,
30
+ "allocated_bytes_peak_delta": 37934908928,
31
+ "before": {
32
+ "active_bytes_current": 44520800256,
33
+ "active_bytes_peak": 44545966080,
34
+ "allocated_bytes_current": 44520800256,
35
+ "allocated_bytes_peak": 44545966080,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 421167104,
38
+ "inactive_split_bytes_peak": 1025146880,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T19:11:17.937038Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1368806912,
55
+ "inactive_split_bytes_current_delta": 947639808,
56
+ "inactive_split_bytes_peak": 9548864000,
57
+ "inactive_split_bytes_peak_delta": 8523717120,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 90267713536,
68
+ "reserved_bytes_current_delta": 42016440320,
69
+ "reserved_bytes_peak": 90267713536,
70
+ "reserved_bytes_peak_delta": 42016440320,
71
+ "segment_count_current": 1718,
72
+ "segment_count_current_delta": 1097,
73
+ "segment_count_peak": 1718,
74
+ "segment_count_peak_delta": 1097,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:44.008245Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T191143969303Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1f8d9f6081984ccd5c5fe51aab0a3e3e8467fdc958903e68857b52025e995c65
3
+ size 2304358
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77299558912,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113344452608,
5
+ "active_bytes_peak_delta": 30863577600,
6
+ "after": {
7
+ "active_bytes_current": 77299558912,
8
+ "active_bytes_peak": 113344452608,
9
+ "allocated_bytes_current": 77299558912,
10
+ "allocated_bytes_peak": 113344452608,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 1368806912,
13
+ "inactive_split_bytes_peak": 9548864000,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 115364331520,
21
+ "reserved_bytes_peak": 115364331520,
22
+ "segment_count_current": 1934,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:44.855203Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77299558912,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113344452608,
30
+ "allocated_bytes_peak_delta": 30863577600,
31
+ "before": {
32
+ "active_bytes_current": 77299558912,
33
+ "active_bytes_peak": 82480875008,
34
+ "allocated_bytes_current": 77299558912,
35
+ "allocated_bytes_peak": 82480875008,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 1368806912,
38
+ "inactive_split_bytes_peak": 9548864000,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 90267713536,
46
+ "reserved_bytes_peak": 90267713536,
47
+ "segment_count_current": 1718,
48
+ "segment_count_peak": 1718,
49
+ "timestamp_utc": "2026-08-26T19:11:44.012798Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1368806912,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 9548864000,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 115364331520,
68
+ "reserved_bytes_current_delta": 25096617984,
69
+ "reserved_bytes_peak": 115364331520,
70
+ "reserved_bytes_peak_delta": 25096617984,
71
+ "segment_count_current": 1934,
72
+ "segment_count_current_delta": 216,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 216,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:44.892124Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T191144855619Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e16105fc8a481ebf77593be97dde94d82e8367120bc404653621454fbc513eeb
3
+ size 1833396
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77299558912,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113344452608,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77299558912,
8
+ "active_bytes_peak": 113344452608,
9
+ "allocated_bytes_current": 77299558912,
10
+ "allocated_bytes_peak": 113344452608,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 1368806912,
13
+ "inactive_split_bytes_peak": 9548864000,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 115364331520,
21
+ "reserved_bytes_peak": 115364331520,
22
+ "segment_count_current": 1934,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:45.209139Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77299558912,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113344452608,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77299558912,
33
+ "active_bytes_peak": 113344452608,
34
+ "allocated_bytes_current": 77299558912,
35
+ "allocated_bytes_peak": 113344452608,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 1368806912,
38
+ "inactive_split_bytes_peak": 9548864000,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 115364331520,
46
+ "reserved_bytes_peak": 115364331520,
47
+ "segment_count_current": 1934,
48
+ "segment_count_peak": 1934,
49
+ "timestamp_utc": "2026-08-26T19:11:44.896027Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1368806912,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 9548864000,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 115364331520,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 115364331520,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1934,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:45.240436Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T191145209541Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04cad6d6b0d89aa3a18cdbe7ae40d41a1b82d05deda56a0d48972a25f5405133
3
+ size 1828450
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77316602368,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 113352645632,
5
+ "active_bytes_peak_delta": 8193024,
6
+ "after": {
7
+ "active_bytes_current": 77316602368,
8
+ "active_bytes_peak": 113352645632,
9
+ "allocated_bytes_current": 77316602368,
10
+ "allocated_bytes_peak": 113352645632,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 1477592576,
13
+ "inactive_split_bytes_peak": 10673852416,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 116394033152,
21
+ "reserved_bytes_peak": 116394033152,
22
+ "segment_count_current": 1701,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:47.859143Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77316602368,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 113352645632,
30
+ "allocated_bytes_peak_delta": 8193024,
31
+ "before": {
32
+ "active_bytes_current": 77299558912,
33
+ "active_bytes_peak": 113344452608,
34
+ "allocated_bytes_current": 77299558912,
35
+ "allocated_bytes_peak": 113344452608,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 1368806912,
38
+ "inactive_split_bytes_peak": 9548864000,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 115364331520,
46
+ "reserved_bytes_peak": 115364331520,
47
+ "segment_count_current": 1934,
48
+ "segment_count_peak": 1934,
49
+ "timestamp_utc": "2026-08-26T19:11:45.243981Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1477592576,
55
+ "inactive_split_bytes_current_delta": 108785664,
56
+ "inactive_split_bytes_peak": 10673852416,
57
+ "inactive_split_bytes_peak_delta": 1124988416,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 116394033152,
68
+ "reserved_bytes_current_delta": 1029701632,
69
+ "reserved_bytes_peak": 116394033152,
70
+ "reserved_bytes_peak_delta": 1029701632,
71
+ "segment_count_current": 1701,
72
+ "segment_count_current_delta": -233,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:47.897322Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T191147859681Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e52652eb4021b76676884a199a82ac80943346346a366f19bfbcad71fabb5b7
3
+ size 2193725
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 77316602368,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 113352645632,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 77316602368,
8
+ "active_bytes_peak": 113352645632,
9
+ "allocated_bytes_current": 77316602368,
10
+ "allocated_bytes_peak": 113352645632,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 1477592576,
13
+ "inactive_split_bytes_peak": 10673852416,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 116394033152,
21
+ "reserved_bytes_peak": 116394033152,
22
+ "segment_count_current": 1701,
23
+ "segment_count_peak": 1934,
24
+ "timestamp_utc": "2026-08-26T19:11:48.200547Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 77316602368,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 113352645632,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 77316602368,
33
+ "active_bytes_peak": 113352645632,
34
+ "allocated_bytes_current": 77316602368,
35
+ "allocated_bytes_peak": 113352645632,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 1477592576,
38
+ "inactive_split_bytes_peak": 10673852416,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 116394033152,
46
+ "reserved_bytes_peak": 116394033152,
47
+ "segment_count_current": 1701,
48
+ "segment_count_peak": 1934,
49
+ "timestamp_utc": "2026-08-26T19:11:47.901458Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1477592576,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 10673852416,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 116394033152,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 116394033152,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1701,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1934,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:11:48.209259Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
ultraep/tierbmem/tierbmem_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p00__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T191148200939Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe460f29325376f03ccea0720aacafe5fe7c016cd04f88b0a95e2395321f6b41
3
+ size 411768