Hugging Face's logo

usrnotfound101
/
anlp-a2-tokenizers

Model card Files
xet
 Community
anlp-a2-tokenizers
File size: 828 Bytes
fc83f97
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
{
  "mt_en": {
    "mean_tokens": 18.8248,
    "p95_tokens": 50.0,
    "max_tokens": 112,
    "tokens_per_char": 0.2717304496837542,
    "tokens_per_whitespace_word": 1.5909059147501896,
    "unk_tokens": 0,
    "byte_fallback_tokens": 71
  },
  "mt_vi": {
    "mean_tokens": 22.5454,
    "p95_tokens": 60.0,
    "max_tokens": 120,
    "tokens_per_char": 0.30452669132286336,
    "tokens_per_whitespace_word": 1.3601911156958297,
    "unk_tokens": 0,
    "byte_fallback_tokens": 105
  },
  "mt_ja": {
    "mean_tokens": 17.9854,
    "p95_tokens": 47.0,
    "max_tokens": 110,
    "tokens_per_char": 0.48894536698039154,
    "tokens_per_whitespace_word": 10.451642071573078,
    "unk_tokens": 0,
    "byte_fallback_tokens": 1100
  },
  "lm_tokens_per_split": {
    "train": 40141738,
    "val": 2217353,
    "test": 2248878
  }
}