baileymachihirota commited on
Commit
a6934b5
·
verified ·
1 Parent(s): b3864bb

data-ladder tokenized: refinedweb_neox/meta.json

Browse files
data-ladder/refinedweb_neox/meta.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": "refinedweb",
3
+ "tokenizer": "EleutherAI/gpt-neox-20b",
4
+ "tiktoken_encoding": "EleutherAI/gpt-neox-20b",
5
+ "eot_id": 0,
6
+ "dtype": "uint16",
7
+ "seed": 1234,
8
+ "shuffle_buffer": 0,
9
+ "val_tokens": 5000624,
10
+ "val_bytes": 21211670,
11
+ "val_docs": 7215,
12
+ "train_tokens": 9100011930,
13
+ "train_bytes": 39552307772,
14
+ "train_docs": 15115142
15
+ }