appvoid commited on
Commit
bb20e19
·
verified ·
1 Parent(s): 43a071f

Qwen90M step 130000; complete resumable checkpoint

Browse files
Files changed (6) hide show
  1. complete.json +5 -5
  2. coverage.json +53 -47
  3. latest.json +1 -1
  4. model.safetensors +1 -1
  5. shard_scheduler.json +275 -143
  6. trainer.pt +2 -2
complete.json CHANGED
@@ -1,18 +1,18 @@
1
  {
2
- "step": 125000,
3
  "files": {
4
  "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
5
  "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
6
- "model.safetensors": "f0616fc9490755c1e55484204bd32a0a2d067fdbf3f53d5cd1eef115bc2c5201",
7
  "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
8
  "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
9
  "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
10
  "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
11
- "trainer.pt": "d623373e105ee371e8cd3e233d58364ce3b44c87e5d4ec425bc868f1ad80c99f",
12
  "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783",
13
  "performance.json": "10fa55d76d1a4ec4f9132ef494d950eba8aa9e6cecd5fa1ceb70ba6de35537e4",
14
- "shard_scheduler.json": "35169e3854bb2f8709178543fd2af9faeefbd503738dc898df29818c62aff861",
15
- "coverage.json": "81eba1ac2ae272a1f49252c853da80c2647c2cfb73db9e4286115d001cacf6ff",
16
  "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
17
  },
18
  "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"
 
1
  {
2
+ "step": 130000,
3
  "files": {
4
  "config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7",
5
  "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4",
6
+ "model.safetensors": "a73db074580069c3fdf4301fb6af1029a078b391c5165ea48b96c01748f5025d",
7
  "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03",
8
  "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300",
9
  "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263",
10
  "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6",
11
+ "trainer.pt": "1c525535393ae76fb7f0b50c0f3b7faad471ed2dab82116dabc5b696349d4172",
12
  "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783",
13
  "performance.json": "10fa55d76d1a4ec4f9132ef494d950eba8aa9e6cecd5fa1ceb70ba6de35537e4",
14
+ "shard_scheduler.json": "b20554f0390356a3ef9e930a512a0abd6d6eff832bd071e13704a207d35a2b2b",
15
+ "coverage.json": "2aa3dc9ef607943696d89d798827282dea265f4d49dfb1f7ff37579e8039b857",
16
  "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"
17
  },
18
  "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063"
coverage.json CHANGED
@@ -1,101 +1,107 @@
1
  {
2
- "step": 125000,
3
  "stats": {
4
- "climbmix/blocks": 4500000,
5
- "climbmix/targets": 7519797808,
6
- "climbmix/input_tokens": 7520274931,
7
- "ultra_qa/blocks": 1687500,
8
- "ultra_qa/targets": 2752448828,
9
- "ultra_qa/input_tokens": 2752454113,
10
- "cortex/blocks": 1687500,
11
- "cortex/targets": 64164981,
12
- "cortex/input_tokens": 2682291180,
13
- "ultra_style/blocks": 1687500,
14
- "ultra_style/targets": 2668606742,
15
- "ultra_style/input_tokens": 2668746996,
16
- "openmath/blocks": 1125000,
17
- "openmath/targets": 1738258233,
18
- "openmath/input_tokens": 1738837895,
19
- "rewrite6/blocks": 562500,
20
- "rewrite6/targets": 922143403,
21
- "rewrite6/input_tokens": 923154365
22
  },
23
  "sources": {
24
  "climbmix": {
25
- "page": 616,
26
- "row": 3708,
27
  "start": {
28
- "file": 29,
29
  "reader": {
30
  "format": "parquet-row-v1",
31
- "row_group": 61,
32
  "row": 1024
33
  }
34
  },
35
  "epoch": 0,
36
  "offset": 0,
37
- "documents": 2526844
38
  },
39
  "rewrite6": {
40
- "page": 0,
41
- "row": 2445,
42
- "start": null,
 
 
 
 
 
 
43
  "epoch": 7,
44
- "offset": 2048,
45
- "documents": 1440273
46
  },
47
  "openmath": {
48
- "page": 385,
49
- "row": 1190,
50
  "start": {
51
  "file": 3,
52
  "reader": {
53
  "format": "parquet-row-v1",
54
- "row_group": 267,
55
- "row": 10
56
  }
57
  },
58
  "epoch": 0,
59
  "offset": 0,
60
- "documents": 1578150
61
  },
62
  "ultra_qa": {
63
- "page": 166,
64
- "row": 2396,
65
  "start": {
66
  "file": 1,
67
  "reader": {
68
  "format": "parquet-row-v1",
69
  "row_group": 2,
70
- "row": 31116
71
  }
72
  },
73
  "epoch": 0,
74
- "offset": 2048,
75
- "documents": 682332
76
  },
77
  "ultra_style": {
78
- "page": 254,
79
- "row": 1950,
80
  "start": {
81
  "file": 1,
82
  "reader": {
83
  "format": "parquet-row-v1",
84
  "row_group": 4,
85
- "row": 14310
86
  }
87
  },
88
  "epoch": 0,
89
- "offset": 0,
90
- "documents": 1042334
91
  },
92
  "cortex": {
93
- "page": 32727,
94
- "row": 12,
95
  "start": null,
96
  "epoch": 0,
97
  "offset": 0,
98
- "documents": 4189068
99
  }
100
  }
101
  }
 
1
  {
2
+ "step": 130000,
3
  "stats": {
4
+ "climbmix/blocks": 4680000,
5
+ "climbmix/targets": 7820767801,
6
+ "climbmix/input_tokens": 7821263965,
7
+ "ultra_qa/blocks": 1755000,
8
+ "ultra_qa/targets": 2862601826,
9
+ "ultra_qa/input_tokens": 2862607324,
10
+ "cortex/blocks": 1755000,
11
+ "cortex/targets": 66725095,
12
+ "cortex/input_tokens": 2789568338,
13
+ "ultra_style/blocks": 1755000,
14
+ "ultra_style/targets": 2775831408,
15
+ "ultra_style/input_tokens": 2775976931,
16
+ "openmath/blocks": 1170000,
17
+ "openmath/targets": 1807748576,
18
+ "openmath/input_tokens": 1808351530,
19
+ "rewrite6/blocks": 585000,
20
+ "rewrite6/targets": 959000753,
21
+ "rewrite6/input_tokens": 960052201
22
  },
23
  "sources": {
24
  "climbmix": {
25
+ "page": 641,
26
+ "row": 2468,
27
  "start": {
28
+ "file": 30,
29
  "reader": {
30
  "format": "parquet-row-v1",
31
+ "row_group": 79,
32
  "row": 1024
33
  }
34
  },
35
  "epoch": 0,
36
  "offset": 0,
37
+ "documents": 2628004
38
  },
39
  "rewrite6": {
40
+ "page": 14,
41
+ "row": 2722,
42
+ "start": {
43
+ "file": 0,
44
+ "reader": {
45
+ "format": "jsonl-row-v1",
46
+ "row": 57344
47
+ }
48
+ },
49
  "epoch": 7,
50
+ "offset": 4096,
51
+ "documents": 1497894
52
  },
53
  "openmath": {
54
+ "page": 400,
55
+ "row": 3091,
56
  "start": {
57
  "file": 3,
58
  "reader": {
59
  "format": "parquet-row-v1",
60
+ "row_group": 328,
61
+ "row": 450
62
  }
63
  },
64
  "epoch": 0,
65
  "offset": 0,
66
+ "documents": 1641491
67
  },
68
  "ultra_qa": {
69
+ "page": 173,
70
+ "row": 982,
71
  "start": {
72
  "file": 1,
73
  "reader": {
74
  "format": "parquet-row-v1",
75
  "row_group": 2,
76
+ "row": 59788
77
  }
78
  },
79
  "epoch": 0,
80
+ "offset": 4096,
81
+ "documents": 709590
82
  },
83
  "ultra_style": {
84
+ "page": 264,
85
+ "row": 1770,
86
  "start": {
87
  "file": 1,
88
  "reader": {
89
  "format": "parquet-row-v1",
90
  "row_group": 4,
91
+ "row": 55270
92
  }
93
  },
94
  "epoch": 0,
95
+ "offset": 2048,
96
+ "documents": 1083114
97
  },
98
  "cortex": {
99
+ "page": 34036,
100
+ "row": 55,
101
  "start": null,
102
  "epoch": 0,
103
  "offset": 0,
104
+ "documents": 4356663
105
  }
106
  }
107
  }
latest.json CHANGED
@@ -1 +1 @@
1
- {"step": 125000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "f0616fc9490755c1e55484204bd32a0a2d067fdbf3f53d5cd1eef115bc2c5201", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "d623373e105ee371e8cd3e233d58364ce3b44c87e5d4ec425bc868f1ad80c99f", "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783", "performance.json": "10fa55d76d1a4ec4f9132ef494d950eba8aa9e6cecd5fa1ceb70ba6de35537e4", "shard_scheduler.json": "35169e3854bb2f8709178543fd2af9faeefbd503738dc898df29818c62aff861", "coverage.json": "81eba1ac2ae272a1f49252c853da80c2647c2cfb73db9e4286115d001cacf6ff", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000125000"}
 
1
+ {"step": 130000, "files": {"config.json": "cbc72137da3dae65085a2a8c39ad7634d16bf92d785ca00d283ee82fe32b37e7", "generation_config.json": "54ed0d19f6226888c91157964ede8a892bd5ae175bfad85336152b3d9b6cd7c4", "model.safetensors": "a73db074580069c3fdf4301fb6af1029a078b391c5165ea48b96c01748f5025d", "tokenization_byte.py": "279b9990f77b1eaedc7c1184fe13d60273a713ead856200f1fb9ae323c7b1e03", "tokenizer_config.json": "0ecbd4fd39e35125800d806af981ba0cacd8a7ba5f02c9e729c4ecc54af6d300", "special_tokens_map.json": "20d4b94d0e657c3f44d9123522c106e7315b6c1a1b6d259478f17bd2918b1263", "byte_vocab.json": "57a1cd2e70dd5b7cc6fd2eaa12ebfcd173c1620fa5d45c2c6282511c29b6a5f6", "trainer.pt": "1c525535393ae76fb7f0b50c0f3b7faad471ed2dab82116dabc5b696349d4172", "training_config.json": "95ba9e002790d11959d1191b36ed994bfa10a6da51c22c772e143bec517a9783", "performance.json": "10fa55d76d1a4ec4f9132ef494d950eba8aa9e6cecd5fa1ceb70ba6de35537e4", "shard_scheduler.json": "b20554f0390356a3ef9e930a512a0abd6d6eff832bd071e13704a207d35a2b2b", "coverage.json": "2aa3dc9ef607943696d89d798827282dea265f4d49dfb1f7ff37579e8039b857", "README.md": "9ce0f9efacbc46177a7e818b6aed274d84e7145ff944b5c7d6ecd5d33b9d4419"}, "signature": "9fee47918905b0750bee87876aa424f021ccc4507ee8db22a11d01fbc6109063", "folder": "step-000000130000"}
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f0616fc9490755c1e55484204bd32a0a2d067fdbf3f53d5cd1eef115bc2c5201
3
  size 360622744
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a73db074580069c3fdf4301fb6af1029a078b391c5165ea48b96c01748f5025d
3
  size 360622744
shard_scheduler.json CHANGED
@@ -22,8 +22,8 @@
22
  "tokens_per_second": 1062967.6168812835,
23
  "note": "Bounded fresh sample; full-page latency is an extrapolation.",
24
  "estimated_pages_for_buffer": 1,
25
- "required_token_positions_per_second": 76000.48524185155,
26
- "required_pages_per_second_upper_estimate": 0.006587529439294878
27
  },
28
  {
29
  "source": "rewrite6",
@@ -36,8 +36,8 @@
36
  "tokens_per_second": 1206672.263377472,
37
  "note": "Bounded fresh sample; full-page latency is an extrapolation.",
38
  "estimated_pages_for_buffer": 1,
39
- "required_token_positions_per_second": 9500.060655231444,
40
- "required_pages_per_second_upper_estimate": 0.003789716233936271
41
  },
42
  {
43
  "source": "openmath",
@@ -50,8 +50,8 @@
50
  "tokens_per_second": 410042.7581134458,
51
  "note": "Bounded fresh sample; full-page latency is an extrapolation.",
52
  "estimated_pages_for_buffer": 1,
53
- "required_token_positions_per_second": 19000.121310462888,
54
- "required_pages_per_second_upper_estimate": 0.0043953436744613855
55
  },
56
  {
57
  "source": "ultra_qa",
@@ -64,8 +64,8 @@
64
  "tokens_per_second": 2163486.5624068812,
65
  "note": "Bounded fresh sample; full-page latency is an extrapolation.",
66
  "estimated_pages_for_buffer": 1,
67
- "required_token_positions_per_second": 28500.181965694333,
68
- "required_pages_per_second_upper_estimate": 0.001735511552812213
69
  },
70
  {
71
  "source": "ultra_style",
@@ -78,8 +78,8 @@
78
  "tokens_per_second": 1546694.2737633225,
79
  "note": "Bounded fresh sample; full-page latency is an extrapolation.",
80
  "estimated_pages_for_buffer": 1,
81
- "required_token_positions_per_second": 28500.181965694333,
82
- "required_pages_per_second_upper_estimate": 0.002194022901272243
83
  },
84
  {
85
  "source": "cortex",
@@ -92,8 +92,8 @@
92
  "tokens_per_second": 1015177.8395898147,
93
  "note": "Bounded fresh sample; full-page latency is an extrapolation.",
94
  "estimated_pages_for_buffer": 22,
95
- "required_token_positions_per_second": 28500.181965694333,
96
- "required_pages_per_second_upper_estimate": 0.3538768760407556
97
  }
98
  ],
99
  "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
@@ -158,147 +158,27 @@
158
  "prefetch_batches": 62,
159
  "prefetch_capacity_limit": 512,
160
  "batch_memory_estimate_bytes": 688946,
161
- "producer_batch_seconds": 0.08882844120317657,
162
- "producer_p95_seconds": 0.2087490030007757,
163
- "gpu_update_seconds": 0.9700990693069923,
164
- "producer_batches_per_second": 11.257655616321221,
165
- "gpu_batches_per_second": 1.030822553736051,
166
- "producer_headroom": 10.92104123597185,
167
  "sustainable": true,
168
- "buffer_seconds": 60.14614229703352,
169
  "buffer_drain_seconds": null,
170
  "shard_target_mib": 16,
171
- "upload_pages_per_commit": 83,
172
  "max_pending_pages": 4096,
173
- "generation_bytes_per_second": 538524.4832146277,
174
- "upload_bytes_per_second": 6058163.920097129,
175
  "upload_sustainable": true,
176
  "logical_page_rows": 4096,
177
  "logical_cortex_page_rows": 128,
178
- "estimated_full_shards_per_hour": 115.5548178894913,
179
  "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
180
  },
181
  "upload_measurements": [
182
- {
183
- "seconds": 1.272373552999852,
184
- "bytes": 4555121,
185
- "pages": 1,
186
- "shards": 1
187
- },
188
- {
189
- "seconds": 0.8187937080001575,
190
- "bytes": 1193417,
191
- "pages": 39,
192
- "shards": 1
193
- },
194
- {
195
- "seconds": 1.121016316999885,
196
- "bytes": 13167246,
197
- "pages": 32,
198
- "shards": 4
199
- },
200
- {
201
- "seconds": 0.7197847669995099,
202
- "bytes": 852888,
203
- "pages": 25,
204
- "shards": 1
205
- },
206
- {
207
- "seconds": 0.9669863799999803,
208
- "bytes": 6140852,
209
- "pages": 35,
210
- "shards": 2
211
- },
212
- {
213
- "seconds": 1.2269071499995334,
214
- "bytes": 11690462,
215
- "pages": 40,
216
- "shards": 4
217
- },
218
- {
219
- "seconds": 0.9768262770003275,
220
- "bytes": 1050575,
221
- "pages": 32,
222
- "shards": 1
223
- },
224
- {
225
- "seconds": 0.9254432580000866,
226
- "bytes": 6140767,
227
- "pages": 27,
228
- "shards": 2
229
- },
230
- {
231
- "seconds": 1.0060759970001527,
232
- "bytes": 2526019,
233
- "pages": 35,
234
- "shards": 2
235
- },
236
- {
237
- "seconds": 1.1933680550000645,
238
- "bytes": 16188168,
239
- "pages": 34,
240
- "shards": 4
241
- },
242
- {
243
- "seconds": 1.1706134349997228,
244
- "bytes": 7624358,
245
- "pages": 37,
246
- "shards": 3
247
- },
248
- {
249
- "seconds": 0.8122863409998899,
250
- "bytes": 1264426,
251
- "pages": 34,
252
- "shards": 1
253
- },
254
- {
255
- "seconds": 1.096932616000231,
256
- "bytes": 5773373,
257
- "pages": 27,
258
- "shards": 2
259
- },
260
- {
261
- "seconds": 1.2743502019993684,
262
- "bytes": 11373993,
263
- "pages": 42,
264
- "shards": 4
265
- },
266
- {
267
- "seconds": 0.9631920529991476,
268
- "bytes": 874408,
269
- "pages": 26,
270
- "shards": 1
271
- },
272
- {
273
- "seconds": 1.358759662999546,
274
- "bytes": 11963324,
275
- "pages": 41,
276
- "shards": 3
277
- },
278
- {
279
- "seconds": 0.99124130600012,
280
- "bytes": 2283760,
281
- "pages": 27,
282
- "shards": 2
283
- },
284
- {
285
- "seconds": 1.149097012999846,
286
- "bytes": 11248733,
287
- "pages": 40,
288
- "shards": 3
289
- },
290
- {
291
- "seconds": 1.4425641930001802,
292
- "bytes": 7147715,
293
- "pages": 28,
294
- "shards": 3
295
- },
296
- {
297
- "seconds": 1.1025827090006715,
298
- "bytes": 1108574,
299
- "pages": 33,
300
- "shards": 1
301
- },
302
  {
303
  "seconds": 1.2109743800001525,
304
  "bytes": 6387518,
@@ -430,6 +310,258 @@
430
  "bytes": 10884064,
431
  "pages": 37,
432
  "shards": 3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
433
  }
434
  ],
435
  "cache_key": {
 
22
  "tokens_per_second": 1062967.6168812835,
23
  "note": "Bounded fresh sample; full-page latency is an extrapolation.",
24
  "estimated_pages_for_buffer": 1,
25
+ "required_token_positions_per_second": 75488.56392437725,
26
+ "required_pages_per_second_upper_estimate": 0.0065431573969489225
27
  },
28
  {
29
  "source": "rewrite6",
 
36
  "tokens_per_second": 1206672.263377472,
37
  "note": "Bounded fresh sample; full-page latency is an extrapolation.",
38
  "estimated_pages_for_buffer": 1,
39
+ "required_token_positions_per_second": 9436.070490547156,
40
+ "required_pages_per_second_upper_estimate": 0.003764189600505487
41
  },
42
  {
43
  "source": "openmath",
 
50
  "tokens_per_second": 410042.7581134458,
51
  "note": "Bounded fresh sample; full-page latency is an extrapolation.",
52
  "estimated_pages_for_buffer": 1,
53
+ "required_token_positions_per_second": 18872.14098109431,
54
+ "required_pages_per_second_upper_estimate": 0.0043657376776388345
55
  },
56
  {
57
  "source": "ultra_qa",
 
64
  "tokens_per_second": 2163486.5624068812,
65
  "note": "Bounded fresh sample; full-page latency is an extrapolation.",
66
  "estimated_pages_for_buffer": 1,
67
+ "required_token_positions_per_second": 28308.211471641465,
68
+ "required_pages_per_second_upper_estimate": 0.001723821556915736
69
  },
70
  {
71
  "source": "ultra_style",
 
78
  "tokens_per_second": 1546694.2737633225,
79
  "note": "Bounded fresh sample; full-page latency is an extrapolation.",
80
  "estimated_pages_for_buffer": 1,
81
+ "required_token_positions_per_second": 28308.211471641465,
82
+ "required_pages_per_second_upper_estimate": 0.002179244481231714
83
  },
84
  {
85
  "source": "cortex",
 
92
  "tokens_per_second": 1015177.8395898147,
93
  "note": "Bounded fresh sample; full-page latency is an extrapolation.",
94
  "estimated_pages_for_buffer": 22,
95
+ "required_token_positions_per_second": 28308.211471641465,
96
+ "required_pages_per_second_upper_estimate": 0.3514932449885328
97
  }
98
  ],
99
  "gpu_measurement": "Selected forward/backward estimate; replaced by live successful update timings.",
 
158
  "prefetch_batches": 62,
159
  "prefetch_capacity_limit": 512,
160
  "batch_memory_estimate_bytes": 688946,
161
+ "producer_batch_seconds": 0.08863170175000334,
162
+ "producer_p95_seconds": 0.2126851870016253,
163
+ "gpu_update_seconds": 0.9766777398740697,
164
+ "producer_batches_per_second": 11.282644699981317,
165
+ "gpu_batches_per_second": 1.023879176491662,
166
+ "producer_headroom": 11.019507925379903,
167
  "sustainable": true,
168
+ "buffer_seconds": 60.55401987219232,
169
  "buffer_drain_seconds": null,
170
  "shard_target_mib": 16,
171
+ "upload_pages_per_commit": 79,
172
  "max_pending_pages": 4096,
173
+ "generation_bytes_per_second": 282894.61719676596,
174
+ "upload_bytes_per_second": 6006243.388537014,
175
  "upload_sustainable": true,
176
  "logical_page_rows": 4096,
177
  "logical_cortex_page_rows": 128,
178
+ "estimated_full_shards_per_hour": 60.70259940078005,
179
  "selection_policy": "Smallest shard within 90% of measured local peak, enlarged to amortize 30 seconds of measured page production; bounded to 256 MiB."
180
  },
181
  "upload_measurements": [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
182
  {
183
  "seconds": 1.2109743800001525,
184
  "bytes": 6387518,
 
310
  "bytes": 10884064,
311
  "pages": 37,
312
  "shards": 3
313
+ },
314
+ {
315
+ "seconds": 1.0351326020008855,
316
+ "bytes": 2533356,
317
+ "pages": 35,
318
+ "shards": 2
319
+ },
320
+ {
321
+ "seconds": 1.0790466939997714,
322
+ "bytes": 5521631,
323
+ "pages": 10,
324
+ "shards": 2
325
+ },
326
+ {
327
+ "seconds": 0.8282065210005385,
328
+ "bytes": 1197297,
329
+ "pages": 34,
330
+ "shards": 1
331
+ },
332
+ {
333
+ "seconds": 0.9917393740015541,
334
+ "bytes": 7505816,
335
+ "pages": 31,
336
+ "shards": 3
337
+ },
338
+ {
339
+ "seconds": 1.108150611000383,
340
+ "bytes": 11838640,
341
+ "pages": 37,
342
+ "shards": 3
343
+ },
344
+ {
345
+ "seconds": 1.0752307399998244,
346
+ "bytes": 5923485,
347
+ "pages": 35,
348
+ "shards": 2
349
+ },
350
+ {
351
+ "seconds": 1.4506824640011473,
352
+ "bytes": 7595330,
353
+ "pages": 34,
354
+ "shards": 3
355
+ },
356
+ {
357
+ "seconds": 0.9329213019991585,
358
+ "bytes": 994205,
359
+ "pages": 30,
360
+ "shards": 1
361
+ },
362
+ {
363
+ "seconds": 1.4256255729997065,
364
+ "bytes": 11482907,
365
+ "pages": 35,
366
+ "shards": 4
367
+ },
368
+ {
369
+ "seconds": 0.8028865010001027,
370
+ "bytes": 1091530,
371
+ "pages": 33,
372
+ "shards": 1
373
+ },
374
+ {
375
+ "seconds": 1.2239964609998424,
376
+ "bytes": 11636309,
377
+ "pages": 39,
378
+ "shards": 4
379
+ },
380
+ {
381
+ "seconds": 1.0268402850015264,
382
+ "bytes": 7548268,
383
+ "pages": 28,
384
+ "shards": 3
385
+ },
386
+ {
387
+ "seconds": 1.2450350950002758,
388
+ "bytes": 5949961,
389
+ "pages": 40,
390
+ "shards": 2
391
+ },
392
+ {
393
+ "seconds": 0.8509849889996985,
394
+ "bytes": 5837001,
395
+ "pages": 27,
396
+ "shards": 2
397
+ },
398
+ {
399
+ "seconds": 0.9184139579992916,
400
+ "bytes": 2547467,
401
+ "pages": 36,
402
+ "shards": 2
403
+ },
404
+ {
405
+ "seconds": 1.207064377000279,
406
+ "bytes": 6456430,
407
+ "pages": 36,
408
+ "shards": 2
409
+ },
410
+ {
411
+ "seconds": 1.751341240000329,
412
+ "bytes": 17299657,
413
+ "pages": 29,
414
+ "shards": 5
415
+ },
416
+ {
417
+ "seconds": 0.8660049680001976,
418
+ "bytes": 1103360,
419
+ "pages": 31,
420
+ "shards": 1
421
+ },
422
+ {
423
+ "seconds": 0.9722155079998629,
424
+ "bytes": 6032233,
425
+ "pages": 35,
426
+ "shards": 2
427
+ },
428
+ {
429
+ "seconds": 1.1764326539996546,
430
+ "bytes": 2559639,
431
+ "pages": 36,
432
+ "shards": 2
433
+ },
434
+ {
435
+ "seconds": 1.289128628999606,
436
+ "bytes": 10630071,
437
+ "pages": 35,
438
+ "shards": 3
439
+ },
440
+ {
441
+ "seconds": 1.0585477320000791,
442
+ "bytes": 5855284,
443
+ "pages": 31,
444
+ "shards": 2
445
+ },
446
+ {
447
+ "seconds": 1.2534552170000097,
448
+ "bytes": 8355752,
449
+ "pages": 36,
450
+ "shards": 3
451
+ },
452
+ {
453
+ "seconds": 1.102831222999157,
454
+ "bytes": 5969835,
455
+ "pages": 36,
456
+ "shards": 2
457
+ },
458
+ {
459
+ "seconds": 0.994495390999873,
460
+ "bytes": 2312576,
461
+ "pages": 26,
462
+ "shards": 2
463
+ },
464
+ {
465
+ "seconds": 1.5429994639998768,
466
+ "bytes": 10370731,
467
+ "pages": 36,
468
+ "shards": 3
469
+ },
470
+ {
471
+ "seconds": 1.2985985390005226,
472
+ "bytes": 6343213,
473
+ "pages": 36,
474
+ "shards": 2
475
+ },
476
+ {
477
+ "seconds": 1.028006202999677,
478
+ "bytes": 2590442,
479
+ "pages": 36,
480
+ "shards": 2
481
+ },
482
+ {
483
+ "seconds": 1.1724209370004246,
484
+ "bytes": 11575534,
485
+ "pages": 25,
486
+ "shards": 3
487
+ },
488
+ {
489
+ "seconds": 1.0102253660006681,
490
+ "bytes": 11368590,
491
+ "pages": 44,
492
+ "shards": 4
493
+ },
494
+ {
495
+ "seconds": 1.024016710000069,
496
+ "bytes": 910901,
497
+ "pages": 26,
498
+ "shards": 1
499
+ },
500
+ {
501
+ "seconds": 0.949438960000407,
502
+ "bytes": 6133978,
503
+ "pages": 35,
504
+ "shards": 2
505
+ },
506
+ {
507
+ "seconds": 1.1755309580003086,
508
+ "bytes": 7041796,
509
+ "pages": 38,
510
+ "shards": 3
511
+ },
512
+ {
513
+ "seconds": 0.8336417800001072,
514
+ "bytes": 5838305,
515
+ "pages": 35,
516
+ "shards": 2
517
+ },
518
+ {
519
+ "seconds": 0.9415913409993664,
520
+ "bytes": 652793,
521
+ "pages": 19,
522
+ "shards": 1
523
+ },
524
+ {
525
+ "seconds": 1.3964870420004445,
526
+ "bytes": 13572779,
527
+ "pages": 43,
528
+ "shards": 4
529
+ },
530
+ {
531
+ "seconds": 1.3172718099995109,
532
+ "bytes": 5670029,
533
+ "pages": 25,
534
+ "shards": 2
535
+ },
536
+ {
537
+ "seconds": 1.2858697970004869,
538
+ "bytes": 11849581,
539
+ "pages": 46,
540
+ "shards": 4
541
+ },
542
+ {
543
+ "seconds": 1.148199516999739,
544
+ "bytes": 1138117,
545
+ "pages": 34,
546
+ "shards": 1
547
+ },
548
+ {
549
+ "seconds": 1.0136643189998722,
550
+ "bytes": 5709641,
551
+ "pages": 27,
552
+ "shards": 2
553
+ },
554
+ {
555
+ "seconds": 1.273396311000397,
556
+ "bytes": 6852766,
557
+ "pages": 33,
558
+ "shards": 3
559
+ },
560
+ {
561
+ "seconds": 1.0978697010014002,
562
+ "bytes": 12068565,
563
+ "pages": 36,
564
+ "shards": 4
565
  }
566
  ],
567
  "cache_key": {
trainer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d623373e105ee371e8cd3e233d58364ce3b44c87e5d4ec425bc868f1ad80c99f
3
- size 721783615
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c525535393ae76fb7f0b50c0f3b7faad471ed2dab82116dabc5b696349d4172
3
+ size 721763903