Hugging Face's logo

dotlabs
/
void.1

Text Generation
Transformers
TensorBoard
Safetensors
English
qwen3
byte-level
pretraining
symbolic
text-generation-inference
Model card Files
xet
 Metrics Community
3
appvoid commited on
Commit
0dab379
·
verified ·
1 Parent(s): a0189ad

Update TensorBoard training metrics

Browse files
Files changed (3) hide show
  1. benchmark_cache/performance.json +222 -0
  2. benchmark_cache/shard_scheduler.json +230 -0
  3. runs/void-byte-v1/events.out.tfevents.1790390190.c8125a87-1345-4bfd-b42f-0e32ed8b7bb7-6w295.451.0.1790390190216476996-4d8dc439 +3 -0
benchmark_cache/performance.json ADDED
@@ -0,0 +1,222 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cache_key": {
3
+ "version": 1,
4
+ "kind": "gpu",
5
+ "gpu": {
6
+ "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
7
+ "total_memory": 101975851008,
8
+ "capability": [
9
+ 12,
10
+ 0
11
+ ]
12
+ },
13
+ "torch": "2.8.0+cu128",
14
+ "cuda": "12.8",
15
+ "python": "3.13.11",
16
+ "packages": {
17
+ "transformers": "4.57.1",
18
+ "numpy": "2.2.6",
19
+ "pyarrow": "21.0.0",
20
+ "huggingface_hub": "0.36.0"
21
+ },
22
+ "settings": {
23
+ "context": 2048,
24
+ "global_batch": 90,
25
+ "microbatch_candidates": [
26
+ 8,
27
+ 16,
28
+ 32,
29
+ 48,
30
+ 64,
31
+ 72,
32
+ 76,
33
+ 80,
34
+ 86,
35
+ 90
36
+ ],
37
+ "batch_selection": "max_batch",
38
+ "gradient_checkpointing": false,
39
+ "compile_model": true,
40
+ "autotune_microbatch": true,
41
+ "attention_backend": "flash",
42
+ "vram_fraction": 0.95
43
+ },
44
+ "source": "e6d135a4487a9fd1907e12903cda79e75fd61305fad149fda7ec5281279dc18b",
45
+ "requested_microbatch_ceiling": 90
46
+ },
47
+ "cache_reused": false,
48
+ "attention": "PyTorch FlashAttention (forced; forward/backward verified)",
49
+ "selected": {
50
+ "microbatch": 86,
51
+ "mode": "compiled",
52
+ "microbatch_seconds": 0.8818629400000191,
53
+ "peak_gb": 84.78368663787842,
54
+ "fits_budget": true,
55
+ "estimated_update_seconds": 0.9349941874999956,
56
+ "input_tokens_per_second": 199722.64624250587
57
+ },
58
+ "measurements": [
59
+ {
60
+ "microbatch": 8,
61
+ "mode": "eager",
62
+ "microbatch_seconds": 0.15782159200000478,
63
+ "peak_gb": 14.079077243804932,
64
+ "fits_budget": true,
65
+ "estimated_update_seconds": 1.793421506500053,
66
+ "input_tokens_per_second": 103813.42497165725
67
+ },
68
+ {
69
+ "microbatch": 16,
70
+ "mode": "eager",
71
+ "microbatch_seconds": 0.3450626039999918,
72
+ "peak_gb": 27.397329807281494,
73
+ "fits_budget": true,
74
+ "estimated_update_seconds": 1.9234990809999601,
75
+ "input_tokens_per_second": 94962.47817106481
76
+ },
77
+ {
78
+ "microbatch": 32,
79
+ "mode": "eager",
80
+ "microbatch_seconds": 0.7818640030000097,
81
+ "peak_gb": 54.14767265319824,
82
+ "fits_budget": true,
83
+ "estimated_update_seconds": 2.185234111500023,
84
+ "input_tokens_per_second": 83820.20370363462
85
+ },
86
+ {
87
+ "microbatch": 48,
88
+ "mode": "eager",
89
+ "microbatch_seconds": 1.1685763079999845,
90
+ "peak_gb": 80.94732475280762,
91
+ "fits_budget": true,
92
+ "estimated_update_seconds": 2.190914332499986,
93
+ "input_tokens_per_second": 84122.87612457851
94
+ },
95
+ {
96
+ "microbatch": 8,
97
+ "mode": "compiled",
98
+ "microbatch_seconds": 0.08803204299999834,
99
+ "peak_gb": 8.428553104400635,
100
+ "fits_budget": true,
101
+ "estimated_update_seconds": 1.0060347614999898,
102
+ "input_tokens_per_second": 186114.0494035826
103
+ },
104
+ {
105
+ "microbatch": 16,
106
+ "mode": "compiled",
107
+ "microbatch_seconds": 0.1704058820000114,
108
+ "peak_gb": 16.191233158111572,
109
+ "fits_budget": true,
110
+ "estimated_update_seconds": 0.9576335650000658,
111
+ "input_tokens_per_second": 192293.8317352086
112
+ },
113
+ {
114
+ "microbatch": 32,
115
+ "mode": "compiled",
116
+ "microbatch_seconds": 0.34082671699997036,
117
+ "peak_gb": 31.810625553131104,
118
+ "fits_budget": true,
119
+ "estimated_update_seconds": 0.9493586249999737,
120
+ "input_tokens_per_second": 192285.39527904938
121
+ },
122
+ {
123
+ "microbatch": 48,
124
+ "mode": "compiled",
125
+ "microbatch_seconds": 0.5061537779999981,
126
+ "peak_gb": 47.43971490859985,
127
+ "fits_budget": true,
128
+ "estimated_update_seconds": 0.9468006980000041,
129
+ "input_tokens_per_second": 194217.6553308279
130
+ },
131
+ {
132
+ "microbatch": 64,
133
+ "mode": "compiled",
134
+ "microbatch_seconds": 0.6658512800000267,
135
+ "peak_gb": 63.073320388793945,
136
+ "fits_budget": true,
137
+ "estimated_update_seconds": 0.9295456565000109,
138
+ "input_tokens_per_second": 196848.76178355434
139
+ },
140
+ {
141
+ "microbatch": 72,
142
+ "mode": "compiled",
143
+ "microbatch_seconds": 0.7430151979999664,
144
+ "peak_gb": 70.97307014465332,
145
+ "fits_budget": true,
146
+ "estimated_update_seconds": 0.9250642054999503,
147
+ "input_tokens_per_second": 198456.23669195347
148
+ },
149
+ {
150
+ "microbatch": 76,
151
+ "mode": "compiled",
152
+ "microbatch_seconds": 0.78068104700003,
153
+ "peak_gb": 74.84185981750488,
154
+ "fits_budget": true,
155
+ "estimated_update_seconds": 0.9241739715000392,
156
+ "input_tokens_per_second": 199374.63654089966
157
+ },
158
+ {
159
+ "microbatch": 80,
160
+ "mode": "compiled",
161
+ "microbatch_seconds": 0.8210239410000213,
162
+ "peak_gb": 78.72572612762451,
163
+ "fits_budget": true,
164
+ "estimated_update_seconds": 0.9229792625000073,
165
+ "input_tokens_per_second": 199555.69115370748
166
+ },
167
+ {
168
+ "microbatch": 86,
169
+ "mode": "compiled",
170
+ "microbatch_seconds": 0.8818629400000191,
171
+ "peak_gb": 84.78368663787842,
172
+ "fits_budget": true,
173
+ "estimated_update_seconds": 0.9349941874999956,
174
+ "input_tokens_per_second": 199722.64624250587
175
+ },
176
+ {
177
+ "microbatch": 90,
178
+ "mode": "compiled",
179
+ "microbatch_seconds": 0.9259720020000373,
180
+ "peak_gb": 88.6924238204956,
181
+ "fits_budget": false,
182
+ "estimated_update_seconds": 0.9259720020000373,
183
+ "input_tokens_per_second": 199055.69455867045
184
+ }
185
+ ],
186
+ "failures": [
187
+ {
188
+ "microbatch": 64,
189
+ "mode": "eager",
190
+ "reason": "oom"
191
+ },
192
+ {
193
+ "microbatch": 72,
194
+ "mode": "eager",
195
+ "reason": "oom"
196
+ },
197
+ {
198
+ "microbatch": 76,
199
+ "mode": "eager",
200
+ "reason": "oom"
201
+ },
202
+ {
203
+ "microbatch": 80,
204
+ "mode": "eager",
205
+ "reason": "oom"
206
+ },
207
+ {
208
+ "microbatch": 86,
209
+ "mode": "eager",
210
+ "reason": "oom"
211
+ },
212
+ {
213
+ "microbatch": 90,
214
+ "mode": "eager",
215
+ "reason": "oom"
216
+ }
217
+ ],
218
+ "selection_policy": "max_batch",
219
+ "budget_gb": 89.6964394569397,
220
+ "reserve_gb": 2.1791439056396484,
221
+ "note": "Forward/backward throughput; excludes optimizer, data, logging and checkpoint time."
222
+ }
benchmark_cache/shard_scheduler.json ADDED
@@ -0,0 +1,230 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": 1,
3
+ "startup_batch_seconds": [
4
+ 17.862166522000052,
5
+ 0.01599195200003578,
6
+ 0.012736249999989013,
7
+ 0.10922379700002693,
8
+ 0.015912060000005113,
9
+ 0.014246264000007614,
10
+ 0.01397435499995936,
11
+ 0.10559937200002878
12
+ ],
13
+ "source_benchmarks": [
14
+ {
15
+ "source": "climbmix",
16
+ "rows": 256,
17
+ "tokens": 721064,
18
+ "bytes": 302776,
19
+ "generation_seconds": 0.6739276440000594,
20
+ "write_seconds": 0.00442227799999273,
21
+ "read_seconds": 0.04565144100001817,
22
+ "tokens_per_second": 1062967.6168812835,
23
+ "note": "Bounded fresh sample; full-page latency is an extrapolation.",
24
+ "estimated_pages_for_buffer": 1,
25
+ "required_token_positions_per_second": 78853.96613762408,
26
+ "required_pages_per_second_upper_estimate": 0.00683486193125923
27
+ },
28
+ {
29
+ "source": "rewrite6",
30
+ "rows": 256,
31
+ "tokens": 156675,
32
+ "bytes": 68694,
33
+ "generation_seconds": 0.12810849499999222,
34
+ "write_seconds": 0.001732062999963091,
35
+ "read_seconds": 0.002349199000036606,
36
+ "tokens_per_second": 1206672.263377472,
37
+ "note": "Bounded fresh sample; full-page latency is an extrapolation.",
38
+ "estimated_pages_for_buffer": 1,
39
+ "required_token_positions_per_second": 9856.74576720301,
40
+ "required_pages_per_second_upper_estimate": 0.003932003258019391
41
+ },
42
+ {
43
+ "source": "openmath",
44
+ "rows": 256,
45
+ "tokens": 270174,
46
+ "bytes": 90063,
47
+ "generation_seconds": 0.6564187429999038,
48
+ "write_seconds": 0.002473517999987962,
49
+ "read_seconds": 0.0032515120000198294,
50
+ "tokens_per_second": 410042.7581134458,
51
+ "note": "Bounded fresh sample; full-page latency is an extrapolation.",
52
+ "estimated_pages_for_buffer": 1,
53
+ "required_token_positions_per_second": 19713.49153440602,
54
+ "required_pages_per_second_upper_estimate": 0.004560369320883491
55
+ },
56
+ {
57
+ "source": "ultra_qa",
58
+ "rows": 256,
59
+ "tokens": 1026361,
60
+ "bytes": 365167,
61
+ "generation_seconds": 0.46967248600003586,
62
+ "write_seconds": 0.004728889000034542,
63
+ "read_seconds": 0.0050225659999796335,
64
+ "tokens_per_second": 2163486.5624068812,
65
+ "note": "Bounded fresh sample; full-page latency is an extrapolation.",
66
+ "estimated_pages_for_buffer": 1,
67
+ "required_token_positions_per_second": 29570.23730160903,
68
+ "required_pages_per_second_upper_estimate": 0.0018006723086229548
69
+ },
70
+ {
71
+ "source": "ultra_style",
72
+ "rows": 256,
73
+ "tokens": 811870,
74
+ "bytes": 338806,
75
+ "generation_seconds": 0.5208254859999215,
76
+ "write_seconds": 0.004081093000081637,
77
+ "read_seconds": 0.004787273000033565,
78
+ "tokens_per_second": 1546694.2737633225,
79
+ "note": "Bounded fresh sample; full-page latency is an extrapolation.",
80
+ "estimated_pages_for_buffer": 1,
81
+ "required_token_positions_per_second": 29570.23730160903,
82
+ "required_pages_per_second_upper_estimate": 0.0022763987231337094
83
+ },
84
+ {
85
+ "source": "cortex",
86
+ "rows": 128,
87
+ "tokens": 80537,
88
+ "bytes": 22745,
89
+ "generation_seconds": 0.07537080299994159,
90
+ "write_seconds": 0.003962095000019872,
91
+ "read_seconds": 0.042780299000014566,
92
+ "tokens_per_second": 1015177.8395898147,
93
+ "note": "Bounded fresh sample; full-page latency is an extrapolation.",
94
+ "estimated_pages_for_buffer": 23,
95
+ "required_token_positions_per_second": 29570.23730160903,
96
+ "required_pages_per_second_upper_estimate": 0.36716338206798155
97
+ }
98
+ ],
99
+ "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
100
+ "layout": "Logical pages unchanged; multiple pages per immutable TAR shard.",
101
+ "candidate_shard_mib": [
102
+ 16,
103
+ 32,
104
+ 64,
105
+ 128,
106
+ 256
107
+ ],
108
+ "local_shard_size_benchmark": [
109
+ {
110
+ "target_mib": 16,
111
+ "actual_bytes": 16107520,
112
+ "pages": 6,
113
+ "packing_seconds": 0.03508346299997811,
114
+ "read_seconds": 0.0015898530000413302,
115
+ "io_bytes_per_second": 439216350.1111124,
116
+ "target_reached": true
117
+ },
118
+ {
119
+ "target_mib": 32,
120
+ "actual_bytes": 18657280,
121
+ "pages": 8,
122
+ "packing_seconds": 0.04401178199998412,
123
+ "read_seconds": 0.0016850460000341627,
124
+ "io_bytes_per_second": 408283918.5247723,
125
+ "target_reached": false
126
+ },
127
+ {
128
+ "target_mib": 64,
129
+ "actual_bytes": 18657280,
130
+ "pages": 8,
131
+ "packing_seconds": 0.04403130000002875,
132
+ "read_seconds": 0.0015284270000393008,
133
+ "io_bytes_per_second": 409512550.4148024,
134
+ "target_reached": false
135
+ },
136
+ {
137
+ "target_mib": 128,
138
+ "actual_bytes": 18657280,
139
+ "pages": 8,
140
+ "packing_seconds": 0.04190044699998907,
141
+ "read_seconds": 0.001583749000019452,
142
+ "io_bytes_per_second": 429058870.0316856,
143
+ "target_reached": false
144
+ },
145
+ {
146
+ "target_mib": 256,
147
+ "actual_bytes": 18657280,
148
+ "pages": 8,
149
+ "packing_seconds": 0.042332877000035296,
150
+ "read_seconds": 0.0015621619999137693,
151
+ "io_bytes_per_second": 425043021.37700915,
152
+ "target_reached": false
153
+ }
154
+ ],
155
+ "shard_benchmark_scope": "Local packing/read measurements; upload capacity measured only on real data commits. Targets larger than available pages are sample-limited.",
156
+ "estimated_fresh_generation_batch_seconds": 0.1434574083060457,
157
+ "selected": {
158
+ "prefetch_batches": 65,
159
+ "prefetch_capacity_limit": 512,
160
+ "batch_memory_estimate_bytes": 688946,
161
+ "producer_batch_seconds": 2.3582960475177965,
162
+ "producer_p95_seconds": 17.862166522000052,
163
+ "gpu_update_seconds": 0.9349941874999956,
164
+ "producer_batches_per_second": 0.4240349726458394,
165
+ "gpu_batches_per_second": 1.06952536536491,
166
+ "producer_headroom": 0.39647023472057946,
167
+ "sustainable": false,
168
+ "buffer_seconds": 60.774622187499716,
169
+ "buffer_drain_seconds": 100.69863274989008,
170
+ "shard_target_mib": 32,
171
+ "upload_pages_per_commit": 15,
172
+ "max_pending_pages": 1842,
173
+ "generation_bytes_per_second": 1035119.2626246791,
174
+ "upload_bytes_per_second": 6146691.738378885,
175
+ "upload_sustainable": true,
176
+ "logical_page_rows": 4096,
177
+ "logical_cortex_page_rows": 128,
178
+ "estimated_full_shards_per_hour": 111.0562487080349,
179
+ "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
180
+ },
181
+ "upload_measurements": [
182
+ {
183
+ "seconds": 0.8211703489999991,
184
+ "bytes": 5047481,
185
+ "pages": 1,
186
+ "shards": 1
187
+ }
188
+ ],
189
+ "cache_key": {
190
+ "version": 1,
191
+ "kind": "data",
192
+ "gpu": {
193
+ "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
194
+ "total_memory": 101975851008,
195
+ "capability": [
196
+ 12,
197
+ 0
198
+ ]
199
+ },
200
+ "torch": "2.8.0+cu128",
201
+ "cuda": "12.8",
202
+ "python": "3.13.11",
203
+ "packages": {
204
+ "transformers": "4.57.1",
205
+ "numpy": "2.2.6",
206
+ "pyarrow": "21.0.0",
207
+ "huggingface_hub": "0.36.0"
208
+ },
209
+ "settings": {
210
+ "context": 2048,
211
+ "global_batch": 90,
212
+ "page_rows": 4096,
213
+ "cortex_page_rows": 128,
214
+ "mix": {
215
+ "climbmix": 40,
216
+ "ultra_qa": 15,
217
+ "ultra_style": 15,
218
+ "cortex": 15,
219
+ "openmath": 10,
220
+ "rewrite6": 5
221
+ },
222
+ "cpu_threads": 4
223
+ },
224
+ "source": "f8722b66ff2f92bca174f36564336149e85c65c48b964d05ff3a4d5d85da308f",
225
+ "cpu": "unknown",
226
+ "cpu_count": 20,
227
+ "microbatch": 86
228
+ },
229
+ "cache_reused": false
230
+ }
runs/void-byte-v1/events.out.tfevents.1790390190.c8125a87-1345-4bfd-b42f-0e32ed8b7bb7-6w295.451.0.1790390190216476996-4d8dc439 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a10197bec45f649a59459d8677528d5ac83c3578cace0d8df7d92c9d62088448
3
+ size 161