classical-ja-tokenizer / sampling.json
kuramot6f's picture
Initial release (history squashed)
5fe47a9
Raw History Blame Contribute Delete
939 Bytes
{
"by_source": {
"kanripo": {
"documents": 79505,
"characters": 706134606,
"candidate_chunks": 34835916,
"sampled_chunks": 1885590,
"sampled_characters": 36343674
},
"wikisource": {
"documents": 987,
"characters": 14689164,
"candidate_chunks": 211036,
"sampled_chunks": 11604,
"sampled_characters": 807834
},
"minhon": {
"documents": 10657,
"characters": 3379619,
"candidate_chunks": 273448,
"sampled_chunks": 14813,
"sampled_characters": 169486
},
"ndl": {
"documents": 30213,
"characters": 604518087,
"candidate_chunks": 1612013,
"sampled_chunks": 87993,
"sampled_characters": 32921773
}
},
"candidate_chunks": 36932413,
"sampled_chunks": 2000000,
"characters": 1328721476,
"special_characters": {
"literal_u2581": 0,
"newlines": 36797349
},
"seed": 20261020
}