data-ladder tokenized: c4_smollm2/meta.json
Browse files
data-ladder/c4_smollm2/meta.json
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": "c4",
|
| 3 |
+
"tokenizer": "HuggingFaceTB/SmolLM2-135M",
|
| 4 |
+
"tiktoken_encoding": "HuggingFaceTB/SmolLM2-135M",
|
| 5 |
+
"eot_id": 0,
|
| 6 |
+
"dtype": "uint16",
|
| 7 |
+
"seed": 1234,
|
| 8 |
+
"shuffle_buffer": 0,
|
| 9 |
+
"val_tokens": 5000535,
|
| 10 |
+
"val_bytes": 22155480,
|
| 11 |
+
"val_docs": 10248,
|
| 12 |
+
"train_tokens": 9100000657,
|
| 13 |
+
"train_bytes": 40187264015,
|
| 14 |
+
"train_docs": 18540746
|
| 15 |
+
}
|