# Formal baseline: learn the vocabulary from a random sample of all train rows.
# LM training still uses the full corpus. Only a manual command starts training.
[paths]
corpus_dir = "outputs/corpus-fast-v1"
output_dir = "artifacts/tokenizers/ja-unigram-16k-v1"

[training]
model_type = "unigram"
vocab_size = 16384
character_coverage = 0.9995
byte_fallback = true
normalization_rule_name = "identity"
remove_extra_whitespaces = false
num_threads = 12
input_sentence_size = 2000000
seed = 42
max_sentence_length = 8192

[measurement]
# Independent processes; each native encoder uses one thread.
workers = 8
batch_size = 4096

[monitoring]
tensorboard = true