vimeml-tiny-ja-v4.2 / source /configs /corpus-parallel.toml
Voltline's picture
Release VimeML V4.2 epoch20 FP32 and Core ML INT8 (GPL-2.0)
c83c617 verified
Raw History Blame Contribute Delete
1.24 kB
# Local parallel runner: one command starts all workers and merges automatically.
[inputs]
fineweb_paths = [
"datasets/fineweb-2-edu-japanese/train-00010-of-00283.parquet",
"datasets/fineweb-2-edu-japanese/train-00030-of-00283.parquet",
"datasets/fineweb-2-edu-japanese/train-00050-of-00283.parquet",
"datasets/fineweb-2-edu-japanese/train-00070-of-00283.parquet",
"datasets/fineweb-2-edu-japanese/train-00090-of-00283.parquet",
"datasets/fineweb-2-edu-japanese/train-00110-of-00283.parquet",
"datasets/fineweb-2-edu-japanese/train-00130-of-00283.parquet",
"datasets/fineweb-2-edu-japanese/train-00150-of-00283.parquet",
"datasets/fineweb-2-edu-japanese/train-00170-of-00283.parquet",
]
tatoeba_path = "datasets/Tatoeba/jpn_sentences.tsv"
calibration_ids = "annotations/calibration-documents-v1.json"
[build]
quality_mode = "rules_with_approved_annotations"
output_dir = "outputs/corpus-parallel-v1"
seed = 42
batch_size = 1024
max_join_chars = 2048
fineweb_documents_per_shard = 0
tatoeba_documents = 0
commit_every = 5000
progress_every = 5000
[review]
annotations = "annotations/approved-v1.jsonl"
require_all_annotations = true
[split]
train = 0.98
validation = 0.01
test = 0.01