classical-ja-tokenizer / config.json
kuramot6f's picture
Initial release (history squashed)
5fe47a9
Raw History Blame Contribute Delete
934 Bytes
{
"trainer": {
"model_type": "unigram",
"vocab_size": 32000,
"character_coverage": 0.9995,
"byte_fallback": true,
"normalization_rule_name": "identity",
"add_dummy_prefix": false,
"remove_extra_whitespaces": false,
"unk_id": 0,
"bos_id": 1,
"eos_id": 2,
"pad_id": 3,
"hard_vocab_limit": true,
"input_sentence_size": 0,
"shuffle_input_sentence": false,
"max_sentence_length": 8192,
"num_threads": 16,
"seed_sentencepiece_size": 1000000
},
"sampling": {
"method": "uniform reservoir over nonblank line chunks; no source reweighting",
"seed": 20261020,
"max_sentences": 2000000,
"chunk_chars": 1024,
"evaluation_per_source": 128,
"evaluation": "uniform document reservoir, up to 2048 characters at document midpoint; overlaps training; not held-out",
"literal_u2581": "training boundary; encode wrapper forces UTF-8 byte pieces"
}
}