vimeml-tiny-zh-v1 / source /configs /tokenizer.toml
Voltline's picture
Release vimeml-tiny-zh-v1 (step 844730): FP32 bundle, INT8 Core ML packages, source snapshot
81cffe7 verified
Raw History Blame Contribute Delete
658 Bytes
# Formal baseline: learn the vocabulary from a random sample of all train rows.
# LM training still uses the full corpus. Only a manual command starts training.
[paths]
corpus_dir = "outputs/corpus-fast-v1"
output_dir = "artifacts/tokenizers/ja-unigram-16k-v1"
[training]
model_type = "unigram"
vocab_size = 16384
character_coverage = 0.9995
byte_fallback = true
normalization_rule_name = "identity"
remove_extra_whitespaces = false
num_threads = 12
input_sentence_size = 2000000
seed = 42
max_sentence_length = 8192
[measurement]
# Independent processes; each native encoder uses one thread.
workers = 8
batch_size = 4096
[monitoring]
tensorboard = true