| # Formal baseline: learn the vocabulary from a random sample of all train rows. | |
| # LM training still uses the full corpus. Only a manual command starts training. | |
| [paths] | |
| corpus_dir = "outputs/corpus-fast-v1" | |
| output_dir = "artifacts/tokenizers/ja-unigram-16k-v1" | |
| [training] | |
| model_type = "unigram" | |
| vocab_size = 16384 | |
| character_coverage = 0.9995 | |
| byte_fallback = true | |
| normalization_rule_name = "identity" | |
| remove_extra_whitespaces = false | |
| num_threads = 12 | |
| input_sentence_size = 2000000 | |
| seed = 42 | |
| max_sentence_length = 8192 | |
| [measurement] | |
| # Independent processes; each native encoder uses one thread. | |
| workers = 8 | |
| batch_size = 4096 | |
| [monitoring] | |
| tensorboard = true | |