Hugging Face's logo

kuramot6f
/
classical-ja-tokenizer

Transformers
Japanese
Literary Chinese
tokenizer
sentencepiece
classical-japanese
literary-chinese
Model card Files
xet
 Community
classical-ja-tokenizer
File size: 939 Bytes
5fe47a9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
{
  "by_source": {
    "kanripo": {
      "documents": 79505,
      "characters": 706134606,
      "candidate_chunks": 34835916,
      "sampled_chunks": 1885590,
      "sampled_characters": 36343674
    },
    "wikisource": {
      "documents": 987,
      "characters": 14689164,
      "candidate_chunks": 211036,
      "sampled_chunks": 11604,
      "sampled_characters": 807834
    },
    "minhon": {
      "documents": 10657,
      "characters": 3379619,
      "candidate_chunks": 273448,
      "sampled_chunks": 14813,
      "sampled_characters": 169486
    },
    "ndl": {
      "documents": 30213,
      "characters": 604518087,
      "candidate_chunks": 1612013,
      "sampled_chunks": 87993,
      "sampled_characters": 32921773
    }
  },
  "candidate_chunks": 36932413,
  "sampled_chunks": 2000000,
  "characters": 1328721476,
  "special_characters": {
    "literal_u2581": 0,
    "newlines": 36797349
  },
  "seed": 20261020
}