| { | |
| "vocab_size": 4192, | |
| "special_tokens": [ | |
| "<pad>", | |
| "<unk>", | |
| "<bos>", | |
| "<eos>" | |
| ], | |
| "pad_id": 0, | |
| "unk_id": 1, | |
| "bos_id": 2, | |
| "eos_id": 3, | |
| "train_ratio": 0.8, | |
| "seed": 42, | |
| "languages": [ | |
| "ml" | |
| ], | |
| "normalizer": "NFC", | |
| "pre_tokenizer": "ByteLevel", | |
| "recommended_seq_len": 768, | |
| "fertility_per_lang": { | |
| "ml": { | |
| "mean_fertility": 0.6979, | |
| "mean_tokens": 391.4, | |
| "p50_tokens": 394, | |
| "p90_tokens": 576, | |
| "p95_tokens": 688, | |
| "p99_tokens": 1086, | |
| "max_tokens": 1316 | |
| } | |
| }, | |
| "token_overlap": { | |
| "pairwise": {}, | |
| "family_overlap": { | |
| "indo_aryan_vs_indo_aryan": { | |
| "shared_tokens": 0, | |
| "jaccard_similarity": 0 | |
| }, | |
| "dravidian_vs_indo_aryan": { | |
| "shared_tokens": 0, | |
| "jaccard_similarity": 0.0 | |
| }, | |
| "dravidian_vs_dravidian": { | |
| "shared_tokens": 3628, | |
| "jaccard_similarity": 1.0 | |
| } | |
| }, | |
| "per_lang_unique_tokens": { | |
| "ml": 3628 | |
| } | |
| }, | |
| "vocab_utilization": { | |
| "vocab_size": 4192, | |
| "used_tokens": 3628, | |
| "unused_tokens": 564, | |
| "utilization_rate": 0.8655, | |
| "shared_all_langs": 3628, | |
| "unique_per_lang": { | |
| "ml": 3628 | |
| } | |
| }, | |
| "oov_rates": { | |
| "ml": { | |
| "total_tokens": 880083, | |
| "unk_tokens": 0, | |
| "oov_rate": 0.0 | |
| } | |
| } | |
| } |