data-ladder tokenized: refinedweb_smollm2/meta.json
Browse files
data-ladder/refinedweb_smollm2/meta.json
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "refinedweb",
|
| 3 |
+
"tokenizer": "HuggingFaceTB/SmolLM2-135M",
|
| 4 |
+
"tiktoken_encoding": "HuggingFaceTB/SmolLM2-135M",
|
| 5 |
+
"eot_id": 0,
|
| 6 |
+
"dtype": "uint16",
|
| 7 |
+
"seed": 1234,
|
| 8 |
+
"shuffle_buffer": 0,
|
| 9 |
+
"val_tokens": 5001712,
|
| 10 |
+
"val_bytes": 20404052,
|
| 11 |
+
"val_docs": 6958,
|
| 12 |
+
"train_tokens": 9100001669,
|
| 13 |
+
"train_bytes": 38177323636,
|
| 14 |
+
"train_docs": 14581094
|
| 15 |
+
}
|