AdminHugging commited on
Commit
f890162
·
verified ·
1 Parent(s): c1c07d6

Upload tokenizer/meta.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer/meta.json +13 -0
tokenizer/meta.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 24576,
3
+ "special_tokens": [
4
+ "<|pad|>",
5
+ "<|bos|>",
6
+ "<|eos|>"
7
+ ],
8
+ "sample_chars": 40062469,
9
+ "chars_per_token": {
10
+ "en": 3.18,
11
+ "py": 3.11
12
+ }
13
+ }