| { |
| "what": "Out-of-distribution, perturbation, code-mixed and other-language token metrics (numbers only) from sota/robustness/robustness.json; report: ROBUSTNESS.md. Input normalized with hp.normalize 1.0.1 for Hindko/Urdu/Punjabi/Saraiki/Pashto; English and code as found.", |
| "generated_utc_source": "2026-09-27T08:44:17Z", |
| "tokenizers": [ |
| { |
| "key": "released", |
| "display": "Hindko SP-32k (released)", |
| "family": "this study: SentencePiece Unigram 32,768 (R2-A4-SPnat-D2-32k)" |
| }, |
| { |
| "key": "mingram48k", |
| "display": "MinGram-48k (runner-up)", |
| "family": "this study: MinGram 49,152 (R2-A10-MinGram-P1r3-D2-48k), pre-registered pick" |
| }, |
| { |
| "key": "gpt-4o", |
| "display": "GPT-4o (o200k)", |
| "family": "OpenAI o200k_base, byte-level BPE 200k" |
| }, |
| { |
| "key": "gemma-3", |
| "display": "Gemma 3", |
| "family": "Google Gemma 3 SentencePiece BPE 262k (= Gemma 4 encodings)" |
| }, |
| { |
| "key": "llama-3", |
| "display": "Llama 3", |
| "family": "Meta Llama 3 tiktoken BPE 128k" |
| }, |
| { |
| "key": "llama-4", |
| "display": "Llama 4", |
| "family": "Meta Llama 4 BPE 202k" |
| }, |
| { |
| "key": "qwen-3.5", |
| "display": "Qwen 3.5", |
| "family": "Alibaba Qwen3.5 byte-level BPE 248k" |
| }, |
| { |
| "key": "deepseek-v3", |
| "display": "DeepSeek-V3", |
| "family": "DeepSeek-V3 byte-level BPE 128k (= V4 encodings)" |
| }, |
| { |
| "key": "bloom", |
| "display": "BLOOM", |
| "family": "BigScience BLOOM byte-level BPE 250k" |
| }, |
| { |
| "key": "roberta-urdu", |
| "display": "RoBERTa-Urdu", |
| "family": "UrduHack RoBERTa-Urdu byte-level BPE 52k (best external on Hindko test)" |
| }, |
| { |
| "key": "mt5", |
| "display": "mT5", |
| "family": "Google mT5 SentencePiece Unigram 250k (normalizes: not lossless)" |
| } |
| ], |
| "test_strict": { |
| "released": { |
| "docs": 491, |
| "bytes": 1451026, |
| "words": 171769, |
| "tokens": 199390, |
| "pieces": 194960, |
| "lossless_docs": 491, |
| "bytes_per_token": 7.277325843823662, |
| "fertility": 1.1350127205723965 |
| }, |
| "mingram48k": { |
| "docs": 491, |
| "bytes": 1451026, |
| "words": 171769, |
| "tokens": 196793, |
| "pieces": 193155, |
| "lossless_docs": 491, |
| "bytes_per_token": 7.373361857383139, |
| "fertility": 1.1245044216360345 |
| }, |
| "gpt-4o": { |
| "docs": 491, |
| "bytes": 1451026, |
| "words": 171769, |
| "tokens": 334524, |
| "pieces": 331606, |
| "lossless_docs": 491, |
| "bytes_per_token": 4.337584149418278, |
| "fertility": 1.9305346133469952 |
| }, |
| "gemma-3": { |
| "docs": 491, |
| "bytes": 1451026, |
| "words": 171769, |
| "tokens": 314731, |
| "pieces": 310416, |
| "lossless_docs": 491, |
| "bytes_per_token": 4.61036885467272, |
| "fertility": 1.8071712590746876 |
| }, |
| "llama-3": { |
| "docs": 491, |
| "bytes": 1451026, |
| "words": 171769, |
| "tokens": 549377, |
| "pieces": 535730, |
| "lossless_docs": 491, |
| "bytes_per_token": 2.64122087382617, |
| "fertility": 3.1188980549458867 |
| }, |
| "llama-4": { |
| "docs": 491, |
| "bytes": 1451026, |
| "words": 171769, |
| "tokens": 389341, |
| "pieces": 386574, |
| "lossless_docs": 491, |
| "bytes_per_token": 3.7268769536216326, |
| "fertility": 2.250545791149742 |
| }, |
| "qwen-3.5": { |
| "docs": 491, |
| "bytes": 1451026, |
| "words": 171769, |
| "tokens": 403264, |
| "pieces": 398965, |
| "lossless_docs": 491, |
| "bytes_per_token": 3.5982036581495, |
| "fertility": 2.3226833712718826 |
| }, |
| "deepseek-v3": { |
| "docs": 491, |
| "bytes": 1451026, |
| "words": 171769, |
| "tokens": 430288, |
| "pieces": 425219, |
| "lossless_docs": 491, |
| "bytes_per_token": 3.372220466292344, |
| "fertility": 2.4755281802886433 |
| }, |
| "bloom": { |
| "docs": 491, |
| "bytes": 1451026, |
| "words": 171769, |
| "tokens": 283310, |
| "pieces": 280524, |
| "lossless_docs": 491, |
| "bytes_per_token": 5.121690021531185, |
| "fertility": 1.6331468425618128 |
| }, |
| "roberta-urdu": { |
| "docs": 491, |
| "bytes": 1451026, |
| "words": 171769, |
| "tokens": 254167, |
| "pieces": 250530, |
| "lossless_docs": 491, |
| "bytes_per_token": 5.708947266954404, |
| "fertility": 1.4585286052780186 |
| }, |
| "mt5": { |
| "docs": 491, |
| "bytes": 1451026, |
| "words": 171769, |
| "tokens": 370870, |
| "pieces": 321976, |
| "lossless_docs": 175, |
| "bytes_per_token": 3.9124922479575055, |
| "fertility": 1.874470946445517 |
| } |
| }, |
| "sets": { |
| "ooc_gotquestions_southern_hindko": { |
| "released": { |
| "norm": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 11347, |
| "pieces": 10925, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": 2, |
| "bytes_per_token": 6.18207455715167, |
| "chars_per_token": 3.467965100907729, |
| "fertility": 1.3992059426229508 |
| }, |
| "raw": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 11347, |
| "pieces": 10925, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": 2, |
| "bytes_per_token": 6.18207455715167, |
| "chars_per_token": 3.467965100907729, |
| "fertility": 1.3992059426229508 |
| } |
| }, |
| "mingram48k": { |
| "norm": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 10720, |
| "pieces": 10441, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": 2, |
| "bytes_per_token": 6.54365671641791, |
| "chars_per_token": 3.6708022388059702, |
| "fertility": 1.337218237704918 |
| }, |
| "raw": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 10720, |
| "pieces": 10441, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": 2, |
| "bytes_per_token": 6.54365671641791, |
| "chars_per_token": 3.6708022388059702, |
| "fertility": 1.337218237704918 |
| } |
| }, |
| "gpt-4o": { |
| "norm": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 17517, |
| "pieces": 17372, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.004566992064851, |
| "chars_per_token": 2.2464463092995377, |
| "fertility": 2.2248975409836067 |
| }, |
| "raw": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 17517, |
| "pieces": 17372, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.004566992064851, |
| "chars_per_token": 2.2464463092995377, |
| "fertility": 2.2248975409836067 |
| } |
| }, |
| "gemma-3": { |
| "norm": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 16907, |
| "pieces": 16757, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.149050689063701, |
| "chars_per_token": 2.3274974862482996, |
| "fertility": 2.1461321721311477 |
| }, |
| "raw": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 16907, |
| "pieces": 16757, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.149050689063701, |
| "chars_per_token": 2.3274974862482996, |
| "fertility": 2.1461321721311477 |
| } |
| }, |
| "llama-3": { |
| "norm": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 28752, |
| "pieces": 27604, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.4397607122982747, |
| "chars_per_token": 1.368635225375626, |
| "fertility": 3.5353483606557377 |
| }, |
| "raw": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 28752, |
| "pieces": 27604, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.4397607122982747, |
| "chars_per_token": 1.368635225375626, |
| "fertility": 3.5353483606557377 |
| } |
| }, |
| "llama-4": { |
| "norm": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 19650, |
| "pieces": 19435, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.5698727735368956, |
| "chars_per_token": 2.0025954198473284, |
| "fertility": 2.4891137295081966 |
| }, |
| "raw": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 19650, |
| "pieces": 19435, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.5698727735368956, |
| "chars_per_token": 2.0025954198473284, |
| "fertility": 2.4891137295081966 |
| } |
| }, |
| "qwen-3.5": { |
| "norm": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 20350, |
| "pieces": 20198, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.447076167076167, |
| "chars_per_token": 1.9337100737100736, |
| "fertility": 2.5868340163934427 |
| }, |
| "raw": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 20350, |
| "pieces": 20198, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.447076167076167, |
| "chars_per_token": 1.9337100737100736, |
| "fertility": 2.5868340163934427 |
| } |
| }, |
| "deepseek-v3": { |
| "norm": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 21499, |
| "pieces": 21207, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.262849434857435, |
| "chars_per_token": 1.8303642029861853, |
| "fertility": 2.716060450819672 |
| }, |
| "raw": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 21499, |
| "pieces": 21207, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.262849434857435, |
| "chars_per_token": 1.8303642029861853, |
| "fertility": 2.716060450819672 |
| } |
| }, |
| "bloom": { |
| "norm": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 15202, |
| "pieces": 15055, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.614392843046968, |
| "chars_per_token": 2.588540981449809, |
| "fertility": 1.9281506147540983 |
| }, |
| "raw": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 15202, |
| "pieces": 15055, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.614392843046968, |
| "chars_per_token": 2.588540981449809, |
| "fertility": 1.9281506147540983 |
| } |
| }, |
| "roberta-urdu": { |
| "norm": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 14801, |
| "pieces": 14801, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.7394094993581515, |
| "chars_per_token": 2.6586717113708533, |
| "fertility": 1.8956198770491803 |
| }, |
| "raw": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 14801, |
| "pieces": 14801, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.7394094993581515, |
| "chars_per_token": 2.6586717113708533, |
| "fertility": 1.8956198770491803 |
| } |
| }, |
| "mt5": { |
| "norm": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 19128, |
| "pieces": 16797, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.6672940192388124, |
| "chars_per_token": 2.057245922208281, |
| "fertility": 2.1512551229508197 |
| }, |
| "raw": { |
| "docs": 9, |
| "bytes": 70148, |
| "chars": 39351, |
| "words": 7808, |
| "tokens": 19128, |
| "pieces": 16797, |
| "lossless_docs": 9, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.6672940192388124, |
| "chars_per_token": 2.057245922208281, |
| "fertility": 2.1512551229508197 |
| } |
| } |
| }, |
| "ooc_translatewiki_hno": { |
| "released": { |
| "norm": { |
| "docs": 1, |
| "bytes": 13784, |
| "chars": 8172, |
| "words": 1581, |
| "tokens": 3807, |
| "pieces": 2997, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": 409, |
| "bytes_per_token": 3.6206987128972945, |
| "chars_per_token": 2.1465721040189125, |
| "fertility": 1.8956356736242885 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 13802, |
| "chars": 8179, |
| "words": 1581, |
| "tokens": 3829, |
| "pieces": 3019, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": 427, |
| "bytes_per_token": 3.604596500391747, |
| "chars_per_token": 2.136066858187516, |
| "fertility": 1.90955091714105 |
| } |
| }, |
| "mingram48k": { |
| "norm": { |
| "docs": 1, |
| "bytes": 13784, |
| "chars": 8172, |
| "words": 1581, |
| "tokens": 3741, |
| "pieces": 2993, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": 409, |
| "bytes_per_token": 3.684576316492916, |
| "chars_per_token": 2.1844426623897353, |
| "fertility": 1.8931056293485136 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 13802, |
| "chars": 8179, |
| "words": 1581, |
| "tokens": 3761, |
| "pieces": 3013, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": 427, |
| "bytes_per_token": 3.669768678542941, |
| "chars_per_token": 2.1746875830896037, |
| "fertility": 1.9057558507273877 |
| } |
| }, |
| "gpt-4o": { |
| "norm": { |
| "docs": 1, |
| "bytes": 13784, |
| "chars": 8172, |
| "words": 1581, |
| "tokens": 4283, |
| "pieces": 3715, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.2183049264534205, |
| "chars_per_token": 1.9080084053233715, |
| "fertility": 2.3497786211258695 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 13802, |
| "chars": 8179, |
| "words": 1581, |
| "tokens": 4294, |
| "pieces": 3725, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.2142524452724732, |
| "chars_per_token": 1.9047508150908243, |
| "fertility": 2.3561037318153066 |
| } |
| }, |
| "gemma-3": { |
| "norm": { |
| "docs": 1, |
| "bytes": 13784, |
| "chars": 8172, |
| "words": 1581, |
| "tokens": 4293, |
| "pieces": 3590, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.2108082925692987, |
| "chars_per_token": 1.9035639412997905, |
| "fertility": 2.270714737507906 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 13802, |
| "chars": 8179, |
| "words": 1581, |
| "tokens": 4304, |
| "pieces": 3600, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": 6, |
| "bytes_per_token": 3.2067843866171004, |
| "chars_per_token": 1.900325278810409, |
| "fertility": 2.2770398481973433 |
| } |
| }, |
| "llama-3": { |
| "norm": { |
| "docs": 1, |
| "bytes": 13784, |
| "chars": 8172, |
| "words": 1581, |
| "tokens": 5685, |
| "pieces": 5076, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.424626209322779, |
| "chars_per_token": 1.437467018469657, |
| "fertility": 3.210626185958254 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 13802, |
| "chars": 8179, |
| "words": 1581, |
| "tokens": 5696, |
| "pieces": 5086, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.4231039325842696, |
| "chars_per_token": 1.4359199438202248, |
| "fertility": 3.216951296647691 |
| } |
| }, |
| "llama-4": { |
| "norm": { |
| "docs": 1, |
| "bytes": 13784, |
| "chars": 8172, |
| "words": 1581, |
| "tokens": 4638, |
| "pieces": 4070, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.971970677015955, |
| "chars_per_token": 1.761966364812419, |
| "fertility": 2.5743200506008854 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 13802, |
| "chars": 8179, |
| "words": 1581, |
| "tokens": 4650, |
| "pieces": 4081, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.9681720430107528, |
| "chars_per_token": 1.7589247311827958, |
| "fertility": 2.581277672359266 |
| } |
| }, |
| "qwen-3.5": { |
| "norm": { |
| "docs": 1, |
| "bytes": 13784, |
| "chars": 8172, |
| "words": 1581, |
| "tokens": 4877, |
| "pieces": 4174, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.8263276604469962, |
| "chars_per_token": 1.6756202583555464, |
| "fertility": 2.640101201771031 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 13802, |
| "chars": 8179, |
| "words": 1581, |
| "tokens": 4891, |
| "pieces": 4188, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.8219178082191783, |
| "chars_per_token": 1.6722551625434472, |
| "fertility": 2.648956356736243 |
| } |
| }, |
| "deepseek-v3": { |
| "norm": { |
| "docs": 1, |
| "bytes": 13784, |
| "chars": 8172, |
| "words": 1581, |
| "tokens": 4955, |
| "pieces": 4364, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.7818365287588294, |
| "chars_per_token": 1.6492431886982846, |
| "fertility": 2.760278304870335 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 13802, |
| "chars": 8179, |
| "words": 1581, |
| "tokens": 4970, |
| "pieces": 4376, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.7770623742454728, |
| "chars_per_token": 1.6456740442655935, |
| "fertility": 2.76786843769766 |
| } |
| }, |
| "bloom": { |
| "norm": { |
| "docs": 1, |
| "bytes": 13784, |
| "chars": 8172, |
| "words": 1581, |
| "tokens": 3871, |
| "pieces": 3210, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.5608369930250583, |
| "chars_per_token": 2.111082407646603, |
| "fertility": 2.030360531309298 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 13802, |
| "chars": 8179, |
| "words": 1581, |
| "tokens": 3884, |
| "pieces": 3222, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.553553038105046, |
| "chars_per_token": 2.105818743563337, |
| "fertility": 2.0379506641366225 |
| } |
| }, |
| "roberta-urdu": { |
| "norm": { |
| "docs": 1, |
| "bytes": 13784, |
| "chars": 8172, |
| "words": 1581, |
| "tokens": 3857, |
| "pieces": 3106, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.573761991184859, |
| "chars_per_token": 2.118745138708841, |
| "fertility": 1.9645793801391525 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 13802, |
| "chars": 8179, |
| "words": 1581, |
| "tokens": 3875, |
| "pieces": 3126, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.5618064516129033, |
| "chars_per_token": 2.110709677419355, |
| "fertility": 1.9772296015180266 |
| } |
| }, |
| "mt5": { |
| "norm": { |
| "docs": 1, |
| "bytes": 13784, |
| "chars": 8172, |
| "words": 1581, |
| "tokens": 3700, |
| "pieces": 3296, |
| "lossless_docs": 0, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.7254054054054055, |
| "chars_per_token": 2.2086486486486487, |
| "fertility": 2.084756483238457 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 13802, |
| "chars": 8179, |
| "words": 1581, |
| "tokens": 3711, |
| "pieces": 3305, |
| "lossless_docs": 0, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.719213150094314, |
| "chars_per_token": 2.2039881433575856, |
| "fertility": 2.09044908285895 |
| } |
| } |
| }, |
| "ooc_chup_di_kahani_pdf": { |
| "released": { |
| "norm": { |
| "docs": 1, |
| "bytes": 110089, |
| "chars": 58242, |
| "words": 4676, |
| "tokens": 24976, |
| "pieces": 23411, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": 269, |
| "bytes_per_token": 4.407791479820628, |
| "chars_per_token": 2.3319186418962206, |
| "fertility": 5.006629597946963 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 124331, |
| "chars": 63023, |
| "words": 4722, |
| "tokens": 40901, |
| "pieces": 37938, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": 14546, |
| "bytes_per_token": 3.0398034277890518, |
| "chars_per_token": 1.5408669714676904, |
| "fertility": 8.03430749682338 |
| } |
| }, |
| "mingram48k": { |
| "norm": { |
| "docs": 1, |
| "bytes": 110089, |
| "chars": 58242, |
| "words": 4676, |
| "tokens": 23255, |
| "pieces": 21768, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": 269, |
| "bytes_per_token": 4.733992689744141, |
| "chars_per_token": 2.5044936572780045, |
| "fertility": 4.6552609067579125 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 124331, |
| "chars": 63023, |
| "words": 4722, |
| "tokens": 38034, |
| "pieces": 36019, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": 14546, |
| "bytes_per_token": 3.2689435767997055, |
| "chars_per_token": 1.657017405479308, |
| "fertility": 7.627911901736552 |
| } |
| }, |
| "gpt-4o": { |
| "norm": { |
| "docs": 1, |
| "bytes": 110089, |
| "chars": 58242, |
| "words": 4676, |
| "tokens": 32414, |
| "pieces": 31803, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.39634108718455, |
| "chars_per_token": 1.796816190534954, |
| "fertility": 6.8013259195893925 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 124331, |
| "chars": 63023, |
| "words": 4722, |
| "tokens": 36657, |
| "pieces": 36478, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.3917396404506643, |
| "chars_per_token": 1.719262350983441, |
| "fertility": 7.725116476069462 |
| } |
| }, |
| "gemma-3": { |
| "norm": { |
| "docs": 1, |
| "bytes": 110089, |
| "chars": 58242, |
| "words": 4676, |
| "tokens": 31429, |
| "pieces": 30278, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": 2, |
| "bytes_per_token": 3.5027840529447327, |
| "chars_per_token": 1.8531292755098794, |
| "fertility": 6.475192472198461 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 124331, |
| "chars": 63023, |
| "words": 4722, |
| "tokens": 36087, |
| "pieces": 34870, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": 85, |
| "bytes_per_token": 3.445312716490703, |
| "chars_per_token": 1.7464183778091833, |
| "fertility": 7.384582803896654 |
| } |
| }, |
| "llama-3": { |
| "norm": { |
| "docs": 1, |
| "bytes": 110089, |
| "chars": 58242, |
| "words": 4676, |
| "tokens": 45647, |
| "pieces": 44455, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.4117466646219903, |
| "chars_per_token": 1.2759217473218394, |
| "fertility": 9.507057313943541 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 124331, |
| "chars": 63023, |
| "words": 4722, |
| "tokens": 49749, |
| "pieces": 49222, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.499165812378138, |
| "chars_per_token": 1.2668194335564533, |
| "fertility": 10.423972892842016 |
| } |
| }, |
| "llama-4": { |
| "norm": { |
| "docs": 1, |
| "bytes": 110089, |
| "chars": 58242, |
| "words": 4676, |
| "tokens": 34946, |
| "pieces": 34214, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.15026040176272, |
| "chars_per_token": 1.6666285125622389, |
| "fertility": 7.316937553464499 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 124331, |
| "chars": 63023, |
| "words": 4722, |
| "tokens": 41227, |
| "pieces": 40038, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.0157663667014334, |
| "chars_per_token": 1.5286826594222234, |
| "fertility": 8.479034307496823 |
| } |
| }, |
| "qwen-3.5": { |
| "norm": { |
| "docs": 1, |
| "bytes": 110089, |
| "chars": 58242, |
| "words": 4676, |
| "tokens": 36412, |
| "pieces": 35282, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.023426342963858, |
| "chars_per_token": 1.5995276282544215, |
| "fertility": 7.545337895637297 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 124331, |
| "chars": 63023, |
| "words": 4722, |
| "tokens": 46006, |
| "pieces": 44811, |
| "lossless_docs": 0, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.7024953266965177, |
| "chars_per_token": 1.3698865365387123, |
| "fertility": 9.489834815756035 |
| } |
| }, |
| "deepseek-v3": { |
| "norm": { |
| "docs": 1, |
| "bytes": 110089, |
| "chars": 58242, |
| "words": 4676, |
| "tokens": 37930, |
| "pieces": 36976, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.902425520696019, |
| "chars_per_token": 1.5355127867123648, |
| "fertility": 7.907613344739093 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 124331, |
| "chars": 63023, |
| "words": 4722, |
| "tokens": 45963, |
| "pieces": 44101, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.7050236059439112, |
| "chars_per_token": 1.3711681134825837, |
| "fertility": 9.339474798814061 |
| } |
| }, |
| "bloom": { |
| "norm": { |
| "docs": 1, |
| "bytes": 110089, |
| "chars": 58242, |
| "words": 4676, |
| "tokens": 28859, |
| "pieces": 28268, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.814719844762466, |
| "chars_per_token": 2.0181572473058664, |
| "fertility": 6.045337895637297 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 124331, |
| "chars": 63023, |
| "words": 4722, |
| "tokens": 35036, |
| "pieces": 34050, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.548664231076607, |
| "chars_per_token": 1.7988069414316703, |
| "fertility": 7.2109275730622615 |
| } |
| }, |
| "roberta-urdu": { |
| "norm": { |
| "docs": 1, |
| "bytes": 110089, |
| "chars": 58242, |
| "words": 4676, |
| "tokens": 26192, |
| "pieces": 25213, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.2031536346976175, |
| "chars_per_token": 2.2236560781918144, |
| "fertility": 5.392001710863986 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 124331, |
| "chars": 63023, |
| "words": 4722, |
| "tokens": 35721, |
| "pieces": 34738, |
| "lossless_docs": 1, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.4806136446348086, |
| "chars_per_token": 1.7643123092858544, |
| "fertility": 7.3566285472257515 |
| } |
| }, |
| "mt5": { |
| "norm": { |
| "docs": 1, |
| "bytes": 110089, |
| "chars": 58242, |
| "words": 4676, |
| "tokens": 28379, |
| "pieces": 26891, |
| "lossless_docs": 0, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.8792416928010147, |
| "chars_per_token": 2.052292187885408, |
| "fertility": 5.750855431993156 |
| }, |
| "raw": { |
| "docs": 1, |
| "bytes": 124331, |
| "chars": 63023, |
| "words": 4722, |
| "tokens": 32280, |
| "pieces": 31329, |
| "lossless_docs": 0, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.851641883519207, |
| "chars_per_token": 1.9523853779429987, |
| "fertility": 6.634688691232529 |
| } |
| } |
| }, |
| "ooc_finepdfs_hindko_org": { |
| "released": { |
| "norm": { |
| "docs": 10, |
| "bytes": 215859, |
| "chars": 124205, |
| "words": 26814, |
| "tokens": 43404, |
| "pieces": 39612, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": 534, |
| "bytes_per_token": 4.973251313243019, |
| "chars_per_token": 2.8616026172702975, |
| "fertility": 1.4772879838890132 |
| }, |
| "raw": { |
| "docs": 10, |
| "bytes": 216683, |
| "chars": 125029, |
| "words": 26814, |
| "tokens": 44264, |
| "pieces": 39648, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": 534, |
| "bytes_per_token": 4.8952421832640525, |
| "chars_per_token": 2.824620459063799, |
| "fertility": 1.478630566122175 |
| } |
| }, |
| "mingram48k": { |
| "norm": { |
| "docs": 10, |
| "bytes": 215859, |
| "chars": 124205, |
| "words": 26814, |
| "tokens": 41692, |
| "pieces": 38468, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": 534, |
| "bytes_per_token": 5.1774680993955675, |
| "chars_per_token": 2.9791087019092393, |
| "fertility": 1.4346237040352054 |
| }, |
| "raw": { |
| "docs": 10, |
| "bytes": 216683, |
| "chars": 125029, |
| "words": 26814, |
| "tokens": 42562, |
| "pieces": 38514, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": 534, |
| "bytes_per_token": 5.090996663690616, |
| "chars_per_token": 2.9375734223015835, |
| "fertility": 1.4363392257775789 |
| } |
| }, |
| "gpt-4o": { |
| "norm": { |
| "docs": 10, |
| "bytes": 215859, |
| "chars": 124205, |
| "words": 26814, |
| "tokens": 49748, |
| "pieces": 48064, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.33904880598215, |
| "chars_per_token": 2.4966832837501003, |
| "fertility": 1.7924964570746624 |
| }, |
| "raw": { |
| "docs": 10, |
| "bytes": 216683, |
| "chars": 125029, |
| "words": 26814, |
| "tokens": 49934, |
| "pieces": 48046, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.339387992149637, |
| "chars_per_token": 2.503885128369448, |
| "fertility": 1.7918251659580815 |
| } |
| }, |
| "gemma-3": { |
| "norm": { |
| "docs": 10, |
| "bytes": 215859, |
| "chars": 124205, |
| "words": 26814, |
| "tokens": 47622, |
| "pieces": 45395, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.532757969005922, |
| "chars_per_token": 2.6081432951157026, |
| "fertility": 1.692958902066085 |
| }, |
| "raw": { |
| "docs": 10, |
| "bytes": 216683, |
| "chars": 125029, |
| "words": 26814, |
| "tokens": 47831, |
| "pieces": 45399, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.530179172503188, |
| "chars_per_token": 2.6139742008320965, |
| "fertility": 1.6931080778697696 |
| } |
| }, |
| "llama-3": { |
| "norm": { |
| "docs": 10, |
| "bytes": 215859, |
| "chars": 124205, |
| "words": 26814, |
| "tokens": 73577, |
| "pieces": 70978, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.9337836552183427, |
| "chars_per_token": 1.688095464615301, |
| "fertility": 2.647050048482136 |
| }, |
| "raw": { |
| "docs": 10, |
| "bytes": 216683, |
| "chars": 125029, |
| "words": 26814, |
| "tokens": 73769, |
| "pieces": 70966, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.937317843538614, |
| "chars_per_token": 1.6948718296303324, |
| "fertility": 2.6466025210710824 |
| } |
| }, |
| "llama-4": { |
| "norm": { |
| "docs": 10, |
| "bytes": 215859, |
| "chars": 124205, |
| "words": 26814, |
| "tokens": 55089, |
| "pieces": 53181, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.9183684583129117, |
| "chars_per_token": 2.25462433516673, |
| "fertility": 1.9833296039382413 |
| }, |
| "raw": { |
| "docs": 10, |
| "bytes": 216683, |
| "chars": 125029, |
| "words": 26814, |
| "tokens": 55284, |
| "pieces": 53172, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.9194522827581215, |
| "chars_per_token": 2.261576586354099, |
| "fertility": 1.9829939583799507 |
| } |
| }, |
| "qwen-3.5": { |
| "norm": { |
| "docs": 10, |
| "bytes": 215859, |
| "chars": 124205, |
| "words": 26814, |
| "tokens": 57078, |
| "pieces": 54832, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.781824871228845, |
| "chars_per_token": 2.1760573250639474, |
| "fertility": 2.0449019169090774 |
| }, |
| "raw": { |
| "docs": 10, |
| "bytes": 216683, |
| "chars": 125029, |
| "words": 26814, |
| "tokens": 57261, |
| "pieces": 54811, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.784128813677721, |
| "chars_per_token": 2.183493127957947, |
| "fertility": 2.044118743939733 |
| } |
| }, |
| "deepseek-v3": { |
| "norm": { |
| "docs": 10, |
| "bytes": 215859, |
| "chars": 124205, |
| "words": 26814, |
| "tokens": 61241, |
| "pieces": 59351, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.52474649336229, |
| "chars_per_token": 2.028134746330073, |
| "fertility": 2.213433281121802 |
| }, |
| "raw": { |
| "docs": 10, |
| "bytes": 216683, |
| "chars": 125029, |
| "words": 26814, |
| "tokens": 61444, |
| "pieces": 59350, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.5265119458368597, |
| "chars_per_token": 2.03484473667079, |
| "fertility": 2.213395987170881 |
| } |
| }, |
| "bloom": { |
| "norm": { |
| "docs": 10, |
| "bytes": 215859, |
| "chars": 124205, |
| "words": 26814, |
| "tokens": 43933, |
| "pieces": 42310, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.913368083217627, |
| "chars_per_token": 2.8271458812282337, |
| "fertility": 1.5779070634743044 |
| }, |
| "raw": { |
| "docs": 10, |
| "bytes": 216683, |
| "chars": 125029, |
| "words": 26814, |
| "tokens": 44008, |
| "pieces": 42308, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.923718414833667, |
| "chars_per_token": 2.8410516269769133, |
| "fertility": 1.5778324755724622 |
| } |
| }, |
| "roberta-urdu": { |
| "norm": { |
| "docs": 10, |
| "bytes": 215859, |
| "chars": 124205, |
| "words": 26814, |
| "tokens": 44158, |
| "pieces": 41010, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.888332804927759, |
| "chars_per_token": 2.8127406132524118, |
| "fertility": 1.5294249272767957 |
| }, |
| "raw": { |
| "docs": 10, |
| "bytes": 216683, |
| "chars": 125029, |
| "words": 26814, |
| "tokens": 45043, |
| "pieces": 41071, |
| "lossless_docs": 10, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.810581000377417, |
| "chars_per_token": 2.7757698199498257, |
| "fertility": 1.5316998582829866 |
| } |
| }, |
| "mt5": { |
| "norm": { |
| "docs": 10, |
| "bytes": 215859, |
| "chars": 124205, |
| "words": 26814, |
| "tokens": 52283, |
| "pieces": 45840, |
| "lossless_docs": 0, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.128665149283706, |
| "chars_per_token": 2.3756287894726777, |
| "fertility": 1.7095547102260014 |
| }, |
| "raw": { |
| "docs": 10, |
| "bytes": 216683, |
| "chars": 125029, |
| "words": 26814, |
| "tokens": 52292, |
| "pieces": 45849, |
| "lossless_docs": 0, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.143712231316454, |
| "chars_per_token": 2.390977587393865, |
| "fertility": 1.7098903557842917 |
| } |
| } |
| }, |
| "lang_urdu": { |
| "released": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1319154, |
| "chars": 743417, |
| "words": 157387, |
| "tokens": 200048, |
| "pieces": 193860, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": 291, |
| "bytes_per_token": 6.594187395025194, |
| "chars_per_token": 3.7161931136527233, |
| "fertility": 1.2317408680513637 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1319152, |
| "chars": 743409, |
| "words": 157387, |
| "tokens": 200200, |
| "pieces": 194010, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": 362, |
| "bytes_per_token": 6.589170829170829, |
| "chars_per_token": 3.7133316683316684, |
| "fertility": 1.2326939327898747 |
| } |
| }, |
| "mingram48k": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1319154, |
| "chars": 743417, |
| "words": 157387, |
| "tokens": 190894, |
| "pieces": 186763, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": 291, |
| "bytes_per_token": 6.910400536423356, |
| "chars_per_token": 3.894396890420862, |
| "fertility": 1.186648198389956 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1319152, |
| "chars": 743409, |
| "words": 157387, |
| "tokens": 191031, |
| "pieces": 186899, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": 362, |
| "bytes_per_token": 6.905434196544016, |
| "chars_per_token": 3.891562102485984, |
| "fertility": 1.187512310419539 |
| } |
| }, |
| "gpt-4o": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1319154, |
| "chars": 743417, |
| "words": 157387, |
| "tokens": 246720, |
| "pieces": 244395, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 5.346765564202335, |
| "chars_per_token": 3.0132011997405965, |
| "fertility": 1.5528283784556538 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1319152, |
| "chars": 743409, |
| "words": 157387, |
| "tokens": 246774, |
| "pieces": 244448, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 5.345587460591473, |
| "chars_per_token": 3.012509421576017, |
| "fertility": 1.5531651279965943 |
| } |
| }, |
| "gemma-3": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1319154, |
| "chars": 743417, |
| "words": 157387, |
| "tokens": 232500, |
| "pieces": 228236, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 5.67378064516129, |
| "chars_per_token": 3.1974924731182797, |
| "fertility": 1.4501578910583466 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1319152, |
| "chars": 743409, |
| "words": 157387, |
| "tokens": 232543, |
| "pieces": 228279, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 5.672722894260416, |
| "chars_per_token": 3.1968668160297233, |
| "fertility": 1.450431102950053 |
| } |
| }, |
| "llama-3": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1319154, |
| "chars": 743417, |
| "words": 157387, |
| "tokens": 472029, |
| "pieces": 456099, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.7946460916596227, |
| "chars_per_token": 1.574939251613778, |
| "fertility": 2.897945827800263 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1319152, |
| "chars": 743409, |
| "words": 157387, |
| "tokens": 472061, |
| "pieces": 456130, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.7944524118704996, |
| "chars_per_token": 1.5748155429065311, |
| "fertility": 2.898142794512889 |
| } |
| }, |
| "llama-4": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1319154, |
| "chars": 743417, |
| "words": 157387, |
| "tokens": 290321, |
| "pieces": 287546, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.543777405010316, |
| "chars_per_token": 2.5606724969947057, |
| "fertility": 1.8269996886655187 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1319152, |
| "chars": 743409, |
| "words": 157387, |
| "tokens": 290385, |
| "pieces": 287609, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.542769082425056, |
| "chars_per_token": 2.560080582674725, |
| "fertility": 1.8273999758556934 |
| } |
| }, |
| "qwen-3.5": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1319154, |
| "chars": 743417, |
| "words": 157387, |
| "tokens": 308012, |
| "pieces": 303690, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.282800670103762, |
| "chars_per_token": 2.4135975221744608, |
| "fertility": 1.929574869588975 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1319152, |
| "chars": 743409, |
| "words": 157387, |
| "tokens": 308053, |
| "pieces": 303731, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.282224162725245, |
| "chars_per_token": 2.413250317315527, |
| "fertility": 1.9298353739508345 |
| } |
| }, |
| "deepseek-v3": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1319154, |
| "chars": 743417, |
| "words": 157387, |
| "tokens": 367308, |
| "pieces": 362100, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.591411022901761, |
| "chars_per_token": 2.023960817624446, |
| "fertility": 2.3006982787650823 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1319152, |
| "chars": 743409, |
| "words": 157387, |
| "tokens": 367354, |
| "pieces": 362145, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.5909558627373053, |
| "chars_per_token": 2.023685600265684, |
| "fertility": 2.3009841981866357 |
| } |
| }, |
| "bloom": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1319154, |
| "chars": 743417, |
| "words": 157387, |
| "tokens": 203261, |
| "pieces": 202297, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 6.489951343346732, |
| "chars_per_token": 3.6574502732939425, |
| "fertility": 1.2853475827101348 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1319152, |
| "chars": 743409, |
| "words": 157387, |
| "tokens": 203301, |
| "pieces": 202337, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 6.488664590926754, |
| "chars_per_token": 3.6566913099296117, |
| "fertility": 1.285601733307071 |
| } |
| }, |
| "roberta-urdu": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1319154, |
| "chars": 743417, |
| "words": 157387, |
| "tokens": 189634, |
| "pieces": 186909, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 6.956315850533132, |
| "chars_per_token": 3.92027273590179, |
| "fertility": 1.1875758480687733 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1319152, |
| "chars": 743409, |
| "words": 157387, |
| "tokens": 189797, |
| "pieces": 187071, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 6.95033114327413, |
| "chars_per_token": 3.916863807120239, |
| "fertility": 1.1886051579863648 |
| } |
| }, |
| "mt5": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1319154, |
| "chars": 743417, |
| "words": 157387, |
| "tokens": 290448, |
| "pieces": 258322, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.541790613121798, |
| "chars_per_token": 2.5595528287335427, |
| "fertility": 1.6413172625439203 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1319152, |
| "chars": 743409, |
| "words": 157387, |
| "tokens": 290467, |
| "pieces": 258350, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.5414866404789525, |
| "chars_per_token": 2.5593578616503767, |
| "fertility": 1.6414951679617757 |
| } |
| } |
| }, |
| "lang_punjabi_shahmukhi": { |
| "released": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1222473, |
| "chars": 686612, |
| "words": 144991, |
| "tokens": 187125, |
| "pieces": 181169, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": 1097, |
| "bytes_per_token": 6.532921843687375, |
| "chars_per_token": 3.6692692050768203, |
| "fertility": 1.2495189356580754 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1222527, |
| "chars": 686634, |
| "words": 144992, |
| "tokens": 189127, |
| "pieces": 183122, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": 1452, |
| "bytes_per_token": 6.464053255220037, |
| "chars_per_token": 3.6305445547172006, |
| "fertility": 1.26298002648422 |
| } |
| }, |
| "mingram48k": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1222473, |
| "chars": 686612, |
| "words": 144991, |
| "tokens": 179714, |
| "pieces": 175076, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": 1097, |
| "bytes_per_token": 6.802324804967894, |
| "chars_per_token": 3.8205815907497467, |
| "fertility": 1.2074956376602686 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1222527, |
| "chars": 686634, |
| "words": 144992, |
| "tokens": 181676, |
| "pieces": 176989, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": 1452, |
| "bytes_per_token": 6.72916070367027, |
| "chars_per_token": 3.77944252405381, |
| "fertility": 1.2206811410284706 |
| } |
| }, |
| "gpt-4o": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1222473, |
| "chars": 686612, |
| "words": 144991, |
| "tokens": 279061, |
| "pieces": 276372, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.380665875919602, |
| "chars_per_token": 2.460436965394663, |
| "fertility": 1.9061321047513293 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1222527, |
| "chars": 686634, |
| "words": 144992, |
| "tokens": 279471, |
| "pieces": 276780, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.3744324098028065, |
| "chars_per_token": 2.456906083278766, |
| "fertility": 1.908932906643125 |
| } |
| }, |
| "gemma-3": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1222473, |
| "chars": 686612, |
| "words": 144991, |
| "tokens": 263472, |
| "pieces": 259711, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": 176, |
| "bytes_per_token": 4.639859263982511, |
| "chars_per_token": 2.606015060423878, |
| "fertility": 1.791221524094599 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1222527, |
| "chars": 686634, |
| "words": 144992, |
| "tokens": 263818, |
| "pieces": 260038, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": 176, |
| "bytes_per_token": 4.633978727759288, |
| "chars_per_token": 2.602680635892926, |
| "fertility": 1.7934644670050761 |
| } |
| }, |
| "llama-3": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1222473, |
| "chars": 686612, |
| "words": 144991, |
| "tokens": 456793, |
| "pieces": 445781, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.676207822799386, |
| "chars_per_token": 1.5031141020111956, |
| "fertility": 3.0745425578139334 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1222527, |
| "chars": 686634, |
| "words": 144992, |
| "tokens": 456938, |
| "pieces": 445924, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.6754767605233094, |
| "chars_per_token": 1.5026852658347523, |
| "fertility": 3.075507614213198 |
| } |
| }, |
| "llama-4": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1222473, |
| "chars": 686612, |
| "words": 144991, |
| "tokens": 323424, |
| "pieces": 319617, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.7797844315820717, |
| "chars_per_token": 2.122946967448303, |
| "fertility": 2.2043919967446257 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1222527, |
| "chars": 686634, |
| "words": 144992, |
| "tokens": 323488, |
| "pieces": 319679, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.7792035562370163, |
| "chars_per_token": 2.1225949648827775, |
| "fertility": 2.204804403001545 |
| } |
| }, |
| "qwen-3.5": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1222473, |
| "chars": 686612, |
| "words": 144991, |
| "tokens": 339309, |
| "pieces": 335285, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.602831047805982, |
| "chars_per_token": 2.023559646222175, |
| "fertility": 2.3124538764475036 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1222527, |
| "chars": 686634, |
| "words": 144992, |
| "tokens": 339372, |
| "pieces": 335347, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.6023213464870407, |
| "chars_per_token": 2.0232488242989994, |
| "fertility": 2.3128655374089604 |
| } |
| }, |
| "deepseek-v3": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1222473, |
| "chars": 686612, |
| "words": 144991, |
| "tokens": 360272, |
| "pieces": 354556, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.3931945863125637, |
| "chars_per_token": 1.9058156059865878, |
| "fertility": 2.445365574414964 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1222527, |
| "chars": 686634, |
| "words": 144992, |
| "tokens": 360569, |
| "pieces": 354852, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.3905493816717467, |
| "chars_per_token": 1.9043068039681725, |
| "fertility": 2.447390200838667 |
| } |
| }, |
| "bloom": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1222473, |
| "chars": 686612, |
| "words": 144991, |
| "tokens": 239560, |
| "pieces": 237706, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 5.102992987143096, |
| "chars_per_token": 2.8661379195191183, |
| "fertility": 1.6394534833196543 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1222527, |
| "chars": 686634, |
| "words": 144992, |
| "tokens": 239881, |
| "pieces": 238026, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 5.096389459773804, |
| "chars_per_token": 2.8623942704924525, |
| "fertility": 1.6416491944383138 |
| } |
| }, |
| "roberta-urdu": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1222473, |
| "chars": 686612, |
| "words": 144991, |
| "tokens": 209633, |
| "pieces": 206837, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 5.83149122514108, |
| "chars_per_token": 3.275304937676797, |
| "fertility": 1.4265506134863544 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1222527, |
| "chars": 686634, |
| "words": 144992, |
| "tokens": 212146, |
| "pieces": 209342, |
| "lossless_docs": 300, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 5.7626681624918685, |
| "chars_per_token": 3.2366106360713847, |
| "fertility": 1.4438175899359964 |
| } |
| }, |
| "mt5": { |
| "norm": { |
| "docs": 300, |
| "bytes": 1222473, |
| "chars": 686612, |
| "words": 144991, |
| "tokens": 306233, |
| "pieces": 262916, |
| "lossless_docs": 107, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.991970166507202, |
| "chars_per_token": 2.2421228280426995, |
| "fertility": 1.81332634439379 |
| }, |
| "raw": { |
| "docs": 300, |
| "bytes": 1222527, |
| "chars": 686634, |
| "words": 144992, |
| "tokens": 306516, |
| "pieces": 263143, |
| "lossless_docs": 99, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.9884606350076344, |
| "chars_per_token": 2.240124495948009, |
| "fertility": 1.8148794416243654 |
| } |
| } |
| }, |
| "lang_saraiki": { |
| "released": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1228900, |
| "chars": 691571, |
| "words": 141656, |
| "tokens": 248196, |
| "pieces": 230702, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": 22285, |
| "bytes_per_token": 4.951328788538091, |
| "chars_per_token": 2.7863905945301295, |
| "fertility": 1.6286073304342916 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1230655, |
| "chars": 692388, |
| "words": 141704, |
| "tokens": 250879, |
| "pieces": 233221, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": 24751, |
| "bytes_per_token": 4.905372709553211, |
| "chars_per_token": 2.7598483731201098, |
| "fertility": 1.645832157172698 |
| } |
| }, |
| "mingram48k": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1228900, |
| "chars": 691571, |
| "words": 141656, |
| "tokens": 238582, |
| "pieces": 222159, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": 22285, |
| "bytes_per_token": 5.150849603071481, |
| "chars_per_token": 2.8986721546470395, |
| "fertility": 1.56829926017959 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1230655, |
| "chars": 692388, |
| "words": 141704, |
| "tokens": 241217, |
| "pieces": 224669, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": 24751, |
| "bytes_per_token": 5.10185849256064, |
| "chars_per_token": 2.8703947068407283, |
| "fertility": 1.5854810026534183 |
| } |
| }, |
| "gpt-4o": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1228900, |
| "chars": 691571, |
| "words": 141656, |
| "tokens": 329554, |
| "pieces": 319646, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.7289791657816322, |
| "chars_per_token": 2.098505859434266, |
| "fertility": 2.256494606652736 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1230655, |
| "chars": 692388, |
| "words": 141704, |
| "tokens": 330503, |
| "pieces": 320579, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.7235819342033203, |
| "chars_per_token": 2.0949522394652997, |
| "fertility": 2.2623144018517474 |
| } |
| }, |
| "gemma-3": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1228900, |
| "chars": 691571, |
| "words": 141656, |
| "tokens": 312825, |
| "pieces": 298889, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": 10884, |
| "bytes_per_token": 3.9283944697514586, |
| "chars_per_token": 2.210728042835451, |
| "fertility": 2.1099635737279043 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1230655, |
| "chars": 692388, |
| "words": 141704, |
| "tokens": 313565, |
| "pieces": 299550, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": 10938, |
| "bytes_per_token": 3.9247205523575652, |
| "chars_per_token": 2.208116339514933, |
| "fertility": 2.1139135098515216 |
| } |
| }, |
| "llama-3": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1228900, |
| "chars": 691571, |
| "words": 141656, |
| "tokens": 513369, |
| "pieces": 490034, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.3937947168605818, |
| "chars_per_token": 1.347122634985751, |
| "fertility": 3.4593239961597106 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1230655, |
| "chars": 692388, |
| "words": 141704, |
| "tokens": 514103, |
| "pieces": 490783, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.3937907384317927, |
| "chars_per_token": 1.3467884840197393, |
| "fertility": 3.4634378704906 |
| } |
| }, |
| "llama-4": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1228900, |
| "chars": 691571, |
| "words": 141656, |
| "tokens": 372151, |
| "pieces": 361653, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.3021542330935563, |
| "chars_per_token": 1.8583075149603252, |
| "fertility": 2.55303693454566 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1230655, |
| "chars": 692388, |
| "words": 141704, |
| "tokens": 373089, |
| "pieces": 362597, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.298556108596072, |
| "chars_per_token": 1.8558252856557014, |
| "fertility": 2.5588339072997233 |
| } |
| }, |
| "qwen-3.5": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1228900, |
| "chars": 691571, |
| "words": 141656, |
| "tokens": 390013, |
| "pieces": 374460, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.1509206103386296, |
| "chars_per_token": 1.7731998676967178, |
| "fertility": 2.643446094764782 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1230655, |
| "chars": 692388, |
| "words": 141704, |
| "tokens": 390879, |
| "pieces": 375313, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.148429565159551, |
| "chars_per_token": 1.7713614699178006, |
| "fertility": 2.648570259131711 |
| } |
| }, |
| "deepseek-v3": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1228900, |
| "chars": 691571, |
| "words": 141656, |
| "tokens": 410173, |
| "pieces": 396824, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.996052885002182, |
| "chars_per_token": 1.6860471069524323, |
| "fertility": 2.8013215112667305 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1230655, |
| "chars": 692388, |
| "words": 141704, |
| "tokens": 411102, |
| "pieces": 397760, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.993551478708447, |
| "chars_per_token": 1.6842243530802574, |
| "fertility": 2.8069779258171965 |
| } |
| }, |
| "bloom": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1228900, |
| "chars": 691571, |
| "words": 141656, |
| "tokens": 300381, |
| "pieces": 288726, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.091137588595816, |
| "chars_per_token": 2.3023127295002013, |
| "fertility": 2.0382193482803412 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1230655, |
| "chars": 692388, |
| "words": 141704, |
| "tokens": 301032, |
| "pieces": 289371, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.088120199845863, |
| "chars_per_token": 2.3000478354460654, |
| "fertility": 2.0420806752102973 |
| } |
| }, |
| "roberta-urdu": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1228900, |
| "chars": 691571, |
| "words": 141656, |
| "tokens": 278238, |
| "pieces": 266914, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.416722374370144, |
| "chars_per_token": 2.485537561368325, |
| "fertility": 1.8842406957700344 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1230655, |
| "chars": 692388, |
| "words": 141704, |
| "tokens": 280681, |
| "pieces": 269353, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.384532618880509, |
| "chars_per_token": 2.466814640107453, |
| "fertility": 1.900814373623892 |
| } |
| }, |
| "mt5": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1228900, |
| "chars": 691571, |
| "words": 141656, |
| "tokens": 334429, |
| "pieces": 291873, |
| "lossless_docs": 20, |
| "unk": 0, |
| "byte_fallback_tokens": 70, |
| "bytes_per_token": 3.6746215190668274, |
| "chars_per_token": 2.067915760893942, |
| "fertility": 2.060435138645733 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1230655, |
| "chars": 692388, |
| "words": 141704, |
| "tokens": 335069, |
| "pieces": 292438, |
| "lossless_docs": 20, |
| "unk": 0, |
| "byte_fallback_tokens": 70, |
| "bytes_per_token": 3.672840519415404, |
| "chars_per_token": 2.0664042331579466, |
| "fertility": 2.0637243832213628 |
| } |
| } |
| }, |
| "lang_pashto": { |
| "released": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1179034, |
| "chars": 666198, |
| "words": 142583, |
| "tokens": 366736, |
| "pieces": 347141, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": 69564, |
| "bytes_per_token": 3.2149393569216005, |
| "chars_per_token": 1.8165601413550891, |
| "fertility": 2.4346591108336897 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1179686, |
| "chars": 666540, |
| "words": 142617, |
| "tokens": 372102, |
| "pieces": 352202, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": 71615, |
| "bytes_per_token": 3.1703296407974157, |
| "chars_per_token": 1.7912830352967735, |
| "fertility": 2.4695653393354227 |
| } |
| }, |
| "mingram48k": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1179034, |
| "chars": 666198, |
| "words": 142583, |
| "tokens": 353204, |
| "pieces": 337572, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": 69564, |
| "bytes_per_token": 3.338110553674364, |
| "chars_per_token": 1.886156442169398, |
| "fertility": 2.3675473233134383 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1179686, |
| "chars": 666540, |
| "words": 142617, |
| "tokens": 359159, |
| "pieces": 343310, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": 71615, |
| "bytes_per_token": 3.2845786963434023, |
| "chars_per_token": 1.855835437786607, |
| "fertility": 2.407216530988592 |
| } |
| }, |
| "gpt-4o": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1179034, |
| "chars": 666198, |
| "words": 142583, |
| "tokens": 242808, |
| "pieces": 240389, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.855828473526408, |
| "chars_per_token": 2.743723435801127, |
| "fertility": 1.6859583540814824 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1179686, |
| "chars": 666540, |
| "words": 142617, |
| "tokens": 243376, |
| "pieces": 240947, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.847174741963053, |
| "chars_per_token": 2.7387252646111366, |
| "fertility": 1.6894689973846035 |
| } |
| }, |
| "gemma-3": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1179034, |
| "chars": 666198, |
| "words": 142583, |
| "tokens": 265861, |
| "pieces": 261846, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.434776067193007, |
| "chars_per_token": 2.5058131880945305, |
| "fertility": 1.8364461401429344 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1179686, |
| "chars": 666540, |
| "words": 142617, |
| "tokens": 267420, |
| "pieces": 263379, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": 3, |
| "bytes_per_token": 4.411360406850647, |
| "chars_per_token": 2.4924837334529952, |
| "fertility": 1.846757399188035 |
| } |
| }, |
| "llama-3": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1179034, |
| "chars": 666198, |
| "words": 142583, |
| "tokens": 358045, |
| "pieces": 350445, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.292977139745004, |
| "chars_per_token": 1.8606543870183916, |
| "fertility": 2.4578315787997167 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1179686, |
| "chars": 666540, |
| "words": 142617, |
| "tokens": 359803, |
| "pieces": 352197, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.2786997329093976, |
| "chars_per_token": 1.8525137366836852, |
| "fertility": 2.469530280401355 |
| } |
| }, |
| "llama-4": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1179034, |
| "chars": 666198, |
| "words": 142583, |
| "tokens": 286608, |
| "pieces": 284160, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.113751186289287, |
| "chars_per_token": 2.324422207335455, |
| "fertility": 1.9929444604195452 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1179686, |
| "chars": 666540, |
| "words": 142617, |
| "tokens": 288101, |
| "pieces": 285647, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.094695957320523, |
| "chars_per_token": 2.3135636460824505, |
| "fertility": 2.0028958679540305 |
| } |
| }, |
| "qwen-3.5": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1179034, |
| "chars": 666198, |
| "words": 142583, |
| "tokens": 323796, |
| "pieces": 312852, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.6412864890239534, |
| "chars_per_token": 2.0574621057702998, |
| "fertility": 2.1941746210978867 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1179686, |
| "chars": 666540, |
| "words": 142617, |
| "tokens": 325601, |
| "pieces": 314653, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.623103123147656, |
| "chars_per_token": 2.0471067349301753, |
| "fertility": 2.2062797562702903 |
| } |
| }, |
| "deepseek-v3": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1179034, |
| "chars": 666198, |
| "words": 142583, |
| "tokens": 346895, |
| "pieces": 344463, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.3988209688810733, |
| "chars_per_token": 1.9204600815808819, |
| "fertility": 2.4158770680936716 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1179686, |
| "chars": 666540, |
| "words": 142617, |
| "tokens": 349348, |
| "pieces": 346910, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.3768219654899987, |
| "chars_per_token": 1.9079542461957704, |
| "fertility": 2.432458963517673 |
| } |
| }, |
| "bloom": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1179034, |
| "chars": 666198, |
| "words": 142583, |
| "tokens": 338733, |
| "pieces": 331788, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.4807178515231763, |
| "chars_per_token": 1.9667348619709328, |
| "fertility": 2.3269814774552366 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1179686, |
| "chars": 666540, |
| "words": 142617, |
| "tokens": 340377, |
| "pieces": 333432, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.46582172120913, |
| "chars_per_token": 1.9582404216501115, |
| "fertility": 2.337954100843518 |
| } |
| }, |
| "roberta-urdu": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1179034, |
| "chars": 666198, |
| "words": 142583, |
| "tokens": 393253, |
| "pieces": 389905, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.9981564031297916, |
| "chars_per_token": 1.694069721019293, |
| "fertility": 2.7345826641324704 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1179686, |
| "chars": 666540, |
| "words": 142617, |
| "tokens": 403274, |
| "pieces": 399920, |
| "lossless_docs": 200, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.925271651532209, |
| "chars_per_token": 1.6528216547558237, |
| "fertility": 2.8041537825083966 |
| } |
| }, |
| "mt5": { |
| "norm": { |
| "docs": 200, |
| "bytes": 1179034, |
| "chars": 666198, |
| "words": 142583, |
| "tokens": 281818, |
| "pieces": 250305, |
| "lossless_docs": 5, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.183671731401117, |
| "chars_per_token": 2.3639299122128468, |
| "fertility": 1.7555038118148727 |
| }, |
| "raw": { |
| "docs": 200, |
| "bytes": 1179686, |
| "chars": 666540, |
| "words": 142617, |
| "tokens": 282937, |
| "pieces": 251920, |
| "lossless_docs": 5, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.1694299437684, |
| "chars_per_token": 2.355789451361964, |
| "fertility": 1.7664093340906064 |
| } |
| } |
| }, |
| "lang_english": { |
| "released": { |
| "norm": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 10165, |
| "pieces": 9290, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": 23, |
| "bytes_per_token": 1.8235120511559273, |
| "chars_per_token": 1.8221347761928184, |
| "fertility": 3.140635564570656 |
| }, |
| "raw": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 10165, |
| "pieces": 9290, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": 23, |
| "bytes_per_token": 1.8235120511559273, |
| "chars_per_token": 1.8221347761928184, |
| "fertility": 3.140635564570656 |
| } |
| }, |
| "mingram48k": { |
| "norm": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 9069, |
| "pieces": 8753, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": 23, |
| "bytes_per_token": 2.04388576469291, |
| "chars_per_token": 2.0423420443268276, |
| "fertility": 2.9590939824205544 |
| }, |
| "raw": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 9069, |
| "pieces": 8753, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": 23, |
| "bytes_per_token": 2.04388576469291, |
| "chars_per_token": 2.0423420443268276, |
| "fertility": 2.9590939824205544 |
| } |
| }, |
| "gpt-4o": { |
| "norm": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 3930, |
| "pieces": 3870, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.7165394402035625, |
| "chars_per_token": 4.7129770992366415, |
| "fertility": 1.308316430020284 |
| }, |
| "raw": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 3930, |
| "pieces": 3870, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.7165394402035625, |
| "chars_per_token": 4.7129770992366415, |
| "fertility": 1.308316430020284 |
| } |
| }, |
| "gemma-3": { |
| "norm": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 3994, |
| "pieces": 3934, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.640961442163245, |
| "chars_per_token": 4.637456184276415, |
| "fertility": 1.329952670723462 |
| }, |
| "raw": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 3994, |
| "pieces": 3934, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.640961442163245, |
| "chars_per_token": 4.637456184276415, |
| "fertility": 1.329952670723462 |
| } |
| }, |
| "llama-3": { |
| "norm": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 3951, |
| "pieces": 3891, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.691470513793976, |
| "chars_per_token": 4.687927107061504, |
| "fertility": 1.315415821501014 |
| }, |
| "raw": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 3951, |
| "pieces": 3891, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.691470513793976, |
| "chars_per_token": 4.687927107061504, |
| "fertility": 1.315415821501014 |
| } |
| }, |
| "llama-4": { |
| "norm": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 3895, |
| "pieces": 3835, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.758921694480103, |
| "chars_per_token": 4.755327342747112, |
| "fertility": 1.2964841108857337 |
| }, |
| "raw": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 3895, |
| "pieces": 3835, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.758921694480103, |
| "chars_per_token": 4.755327342747112, |
| "fertility": 1.2964841108857337 |
| } |
| }, |
| "qwen-3.5": { |
| "norm": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 3993, |
| "pieces": 3933, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.642123716503882, |
| "chars_per_token": 4.638617580766341, |
| "fertility": 1.3296146044624746 |
| }, |
| "raw": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 3993, |
| "pieces": 3933, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.642123716503882, |
| "chars_per_token": 4.638617580766341, |
| "fertility": 1.3296146044624746 |
| } |
| }, |
| "deepseek-v3": { |
| "norm": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 4000, |
| "pieces": 3940, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.634, |
| "chars_per_token": 4.6305, |
| "fertility": 1.3319810682893847 |
| }, |
| "raw": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 4000, |
| "pieces": 3940, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.634, |
| "chars_per_token": 4.6305, |
| "fertility": 1.3319810682893847 |
| } |
| }, |
| "bloom": { |
| "norm": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 3947, |
| "pieces": 3946, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.696224980998227, |
| "chars_per_token": 4.692677983278439, |
| "fertility": 1.3340094658553077 |
| }, |
| "raw": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 3947, |
| "pieces": 3946, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.696224980998227, |
| "chars_per_token": 4.692677983278439, |
| "fertility": 1.3340094658553077 |
| } |
| }, |
| "roberta-urdu": { |
| "norm": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 9251, |
| "pieces": 9015, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.0036752783482865, |
| "chars_per_token": 2.002161928440169, |
| "fertility": 3.0476673427991887 |
| }, |
| "raw": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 9251, |
| "pieces": 9015, |
| "lossless_docs": 50, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.0036752783482865, |
| "chars_per_token": 2.002161928440169, |
| "fertility": 3.0476673427991887 |
| } |
| }, |
| "mt5": { |
| "norm": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 4825, |
| "pieces": 4243, |
| "lossless_docs": 47, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.841658031088083, |
| "chars_per_token": 3.8387564766839377, |
| "fertility": 1.4344151453684921 |
| }, |
| "raw": { |
| "docs": 50, |
| "bytes": 18536, |
| "chars": 18522, |
| "words": 2958, |
| "tokens": 4825, |
| "pieces": 4243, |
| "lossless_docs": 47, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.841658031088083, |
| "chars_per_token": 3.8387564766839377, |
| "fertility": 1.4344151453684921 |
| } |
| } |
| }, |
| "lang_python_code": { |
| "released": { |
| "norm": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 474471, |
| "pieces": 300247, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": 2665, |
| "bytes_per_token": 1.366190979006093, |
| "chars_per_token": 1.3660413386698027, |
| "fertility": 3.9848567295313684 |
| }, |
| "raw": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 474471, |
| "pieces": 300247, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": 2665, |
| "bytes_per_token": 1.366190979006093, |
| "chars_per_token": 1.3660413386698027, |
| "fertility": 3.9848567295313684 |
| } |
| }, |
| "mingram48k": { |
| "norm": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 434794, |
| "pieces": 291139, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": 2665, |
| "bytes_per_token": 1.4908623394067075, |
| "chars_per_token": 1.4906990436850553, |
| "fertility": 3.8639760043531926 |
| }, |
| "raw": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 434794, |
| "pieces": 291139, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": 2665, |
| "bytes_per_token": 1.4908623394067075, |
| "chars_per_token": 1.4906990436850553, |
| "fertility": 3.8639760043531926 |
| } |
| }, |
| "gpt-4o": { |
| "norm": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 156515, |
| "pieces": 135262, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.141571095422164, |
| "chars_per_token": 4.141117464779733, |
| "fertility": 1.7951875987099686 |
| }, |
| "raw": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 156515, |
| "pieces": 135262, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.141571095422164, |
| "chars_per_token": 4.141117464779733, |
| "fertility": 1.7951875987099686 |
| } |
| }, |
| "gemma-3": { |
| "norm": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 179094, |
| "pieces": 147496, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.619428903257507, |
| "chars_per_token": 3.6190324633991087, |
| "fertility": 1.9575563725164904 |
| }, |
| "raw": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 179094, |
| "pieces": 147496, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.619428903257507, |
| "chars_per_token": 3.6190324633991087, |
| "fertility": 1.9575563725164904 |
| } |
| }, |
| "llama-3": { |
| "norm": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 155707, |
| "pieces": 134474, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.163062675409583, |
| "chars_per_token": 4.16260669077177, |
| "fertility": 1.7847293190173463 |
| }, |
| "raw": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 155707, |
| "pieces": 134474, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.163062675409583, |
| "chars_per_token": 4.16260669077177, |
| "fertility": 1.7847293190173463 |
| } |
| }, |
| "llama-4": { |
| "norm": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 156388, |
| "pieces": 135132, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.144934393943269, |
| "chars_per_token": 4.1444803949152105, |
| "fertility": 1.7934622479992568 |
| }, |
| "raw": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 156388, |
| "pieces": 135132, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.144934393943269, |
| "chars_per_token": 4.1444803949152105, |
| "fertility": 1.7934622479992568 |
| } |
| }, |
| "qwen-3.5": { |
| "norm": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 168115, |
| "pieces": 136461, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.8558010885405825, |
| "chars_per_token": 3.8553787585878716, |
| "fertility": 1.8111006410341486 |
| }, |
| "raw": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 168115, |
| "pieces": 136461, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.8558010885405825, |
| "chars_per_token": 3.8553787585878716, |
| "fertility": 1.8111006410341486 |
| } |
| }, |
| "deepseek-v3": { |
| "norm": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 162859, |
| "pieces": 141758, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.9802405762039554, |
| "chars_per_token": 3.9798046162631477, |
| "fertility": 1.8814020465313814 |
| }, |
| "raw": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 162859, |
| "pieces": 141758, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.9802405762039554, |
| "chars_per_token": 3.9798046162631477, |
| "fertility": 1.8814020465313814 |
| } |
| }, |
| "bloom": { |
| "norm": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 156304, |
| "pieces": 145309, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.147161940833248, |
| "chars_per_token": 4.146707697819633, |
| "fertility": 1.9285306647909009 |
| }, |
| "raw": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 156304, |
| "pieces": 145309, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.147161940833248, |
| "chars_per_token": 4.146707697819633, |
| "fertility": 1.9285306647909009 |
| } |
| }, |
| "roberta-urdu": { |
| "norm": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 431279, |
| "pieces": 295278, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 1.5030131307112102, |
| "chars_per_token": 1.502848504100594, |
| "fertility": 3.9189085165965465 |
| }, |
| "raw": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 431279, |
| "pieces": 295278, |
| "lossless_docs": 30, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 1.5030131307112102, |
| "chars_per_token": 1.502848504100594, |
| "fertility": 3.9189085165965465 |
| } |
| }, |
| "mt5": { |
| "norm": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 179259, |
| "pieces": 158708, |
| "lossless_docs": 0, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.6160973786532336, |
| "chars_per_token": 3.6157013037002326, |
| "fertility": 2.106361235351109 |
| }, |
| "raw": { |
| "docs": 30, |
| "bytes": 648218, |
| "chars": 648147, |
| "words": 75347, |
| "tokens": 179259, |
| "pieces": 158708, |
| "lossless_docs": 0, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.6160973786532336, |
| "chars_per_token": 3.6157013037002326, |
| "fertility": 2.106361235351109 |
| } |
| } |
| }, |
| "mixed_natural_test": { |
| "released": { |
| "norm": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1078, |
| "pieces": 993, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 6.102968460111318, |
| "chars_per_token": 3.388682745825603, |
| "fertility": 1.228960396039604 |
| }, |
| "raw": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1078, |
| "pieces": 993, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 6.102968460111318, |
| "chars_per_token": 3.388682745825603, |
| "fertility": 1.228960396039604 |
| } |
| }, |
| "mingram48k": { |
| "norm": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1027, |
| "pieces": 945, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 6.4060370009737095, |
| "chars_per_token": 3.5569620253164556, |
| "fertility": 1.1695544554455446 |
| }, |
| "raw": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1027, |
| "pieces": 945, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 6.4060370009737095, |
| "chars_per_token": 3.5569620253164556, |
| "fertility": 1.1695544554455446 |
| } |
| }, |
| "gpt-4o": { |
| "norm": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1546, |
| "pieces": 1465, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.255498059508409, |
| "chars_per_token": 2.3628719275549805, |
| "fertility": 1.813118811881188 |
| }, |
| "raw": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1546, |
| "pieces": 1465, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.255498059508409, |
| "chars_per_token": 2.3628719275549805, |
| "fertility": 1.813118811881188 |
| } |
| }, |
| "gemma-3": { |
| "norm": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1462, |
| "pieces": 1381, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.5, |
| "chars_per_token": 2.4986320109439126, |
| "fertility": 1.7091584158415842 |
| }, |
| "raw": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1462, |
| "pieces": 1381, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.5, |
| "chars_per_token": 2.4986320109439126, |
| "fertility": 1.7091584158415842 |
| } |
| }, |
| "llama-3": { |
| "norm": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 2520, |
| "pieces": 2386, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.6107142857142858, |
| "chars_per_token": 1.4496031746031746, |
| "fertility": 2.952970297029703 |
| }, |
| "raw": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 2520, |
| "pieces": 2386, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.6107142857142858, |
| "chars_per_token": 1.4496031746031746, |
| "fertility": 2.952970297029703 |
| } |
| }, |
| "llama-4": { |
| "norm": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1736, |
| "pieces": 1655, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.789746543778802, |
| "chars_per_token": 2.10426267281106, |
| "fertility": 2.0482673267326734 |
| }, |
| "raw": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1736, |
| "pieces": 1655, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.789746543778802, |
| "chars_per_token": 2.10426267281106, |
| "fertility": 2.0482673267326734 |
| } |
| }, |
| "qwen-3.5": { |
| "norm": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1813, |
| "pieces": 1732, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.6287920573634858, |
| "chars_per_token": 2.014892443463872, |
| "fertility": 2.1435643564356437 |
| }, |
| "raw": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1813, |
| "pieces": 1732, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.6287920573634858, |
| "chars_per_token": 2.014892443463872, |
| "fertility": 2.1435643564356437 |
| } |
| }, |
| "deepseek-v3": { |
| "norm": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1978, |
| "pieces": 1892, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.3260869565217392, |
| "chars_per_token": 1.846814964610718, |
| "fertility": 2.3415841584158414 |
| }, |
| "raw": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1978, |
| "pieces": 1892, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.3260869565217392, |
| "chars_per_token": 1.846814964610718, |
| "fertility": 2.3415841584158414 |
| } |
| }, |
| "bloom": { |
| "norm": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1373, |
| "pieces": 1292, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.79169701383831, |
| "chars_per_token": 2.6605972323379463, |
| "fertility": 1.599009900990099 |
| }, |
| "raw": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1373, |
| "pieces": 1292, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.79169701383831, |
| "chars_per_token": 2.6605972323379463, |
| "fertility": 1.599009900990099 |
| } |
| }, |
| "roberta-urdu": { |
| "norm": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1226, |
| "pieces": 1144, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 5.366231647634584, |
| "chars_per_token": 2.9796084828711256, |
| "fertility": 1.4158415841584158 |
| }, |
| "raw": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1226, |
| "pieces": 1144, |
| "lossless_docs": 8, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 5.366231647634584, |
| "chars_per_token": 2.9796084828711256, |
| "fertility": 1.4158415841584158 |
| } |
| }, |
| "mt5": { |
| "norm": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1598, |
| "pieces": 1422, |
| "lossless_docs": 2, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.117021276595745, |
| "chars_per_token": 2.285982478097622, |
| "fertility": 1.75990099009901 |
| }, |
| "raw": { |
| "docs": 8, |
| "bytes": 6579, |
| "chars": 3653, |
| "words": 808, |
| "tokens": 1598, |
| "pieces": 1422, |
| "lossless_docs": 2, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.117021276595745, |
| "chars_per_token": 2.285982478097622, |
| "fertility": 1.75990099009901 |
| } |
| } |
| } |
| }, |
| "perturbations": { |
| "harakat": { |
| "released": { |
| "edits": 8396, |
| "docs_affected": 461, |
| "tokens": 198488, |
| "rel_token_change": -0.004523797582627012, |
| "words_compared": 171765, |
| "words_affected": 8214, |
| "words_vanished": 4, |
| "extra_tokens_per_affected_word": -0.10981251521792063, |
| "seg_change_rate": 0.006538002503420371, |
| "seg_change_rate_affected": 0.13671779887996105, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1434238, |
| "perturbed_bytes_per_token": 7.2258171778646565 |
| }, |
| "mingram48k": { |
| "edits": 8396, |
| "docs_affected": 461, |
| "tokens": 196149, |
| "rel_token_change": -0.003272474122555172, |
| "words_compared": 171765, |
| "words_affected": 8214, |
| "words_vanished": 4, |
| "extra_tokens_per_affected_word": -0.0784027270513757, |
| "seg_change_rate": 0.004581841469449539, |
| "seg_change_rate_affected": 0.09581202824446068, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1434238, |
| "perturbed_bytes_per_token": 7.311982217599886 |
| }, |
| "gpt-4o": { |
| "edits": 8396, |
| "docs_affected": 461, |
| "tokens": 326068, |
| "rel_token_change": -0.025277708026927814, |
| "words_compared": 171765, |
| "words_affected": 8214, |
| "words_vanished": 4, |
| "extra_tokens_per_affected_word": -1.0294618943267593, |
| "seg_change_rate": 0.025348586731872035, |
| "seg_change_rate_affected": 0.5300706111516922, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1434238, |
| "perturbed_bytes_per_token": 4.398585571107867 |
| }, |
| "gemma-3": { |
| "edits": 8396, |
| "docs_affected": 461, |
| "tokens": 305872, |
| "rel_token_change": -0.0281478468914724, |
| "words_compared": 171765, |
| "words_affected": 8214, |
| "words_vanished": 4, |
| "extra_tokens_per_affected_word": -1.0785244704163623, |
| "seg_change_rate": 0.027415364014787644, |
| "seg_change_rate_affected": 0.5732895057219382, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1434238, |
| "perturbed_bytes_per_token": 4.689013705079248 |
| }, |
| "llama-3": { |
| "edits": 8396, |
| "docs_affected": 461, |
| "tokens": 540104, |
| "rel_token_change": -0.016879119438928093, |
| "words_compared": 171765, |
| "words_affected": 8214, |
| "words_vanished": 4, |
| "extra_tokens_per_affected_word": -1.1289262235208182, |
| "seg_change_rate": 0.02388728786423311, |
| "seg_change_rate_affected": 0.4995130265400536, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1434238, |
| "perturbed_bytes_per_token": 2.655484869580673 |
| }, |
| "llama-4": { |
| "edits": 8396, |
| "docs_affected": 461, |
| "tokens": 379056, |
| "rel_token_change": -0.026416431868208076, |
| "words_compared": 171765, |
| "words_affected": 8214, |
| "words_vanished": 4, |
| "extra_tokens_per_affected_word": -1.2521305088872656, |
| "seg_change_rate": 0.026419817774284635, |
| "seg_change_rate_affected": 0.5524713903092281, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1434238, |
| "perturbed_bytes_per_token": 3.7837100586720696 |
| }, |
| "qwen-3.5": { |
| "edits": 8396, |
| "docs_affected": 461, |
| "tokens": 395191, |
| "rel_token_change": -0.020019143786700524, |
| "words_compared": 171765, |
| "words_affected": 8214, |
| "words_vanished": 4, |
| "extra_tokens_per_affected_word": -0.9828341855368883, |
| "seg_change_rate": 0.021052018746543243, |
| "seg_change_rate_affected": 0.44022400779157533, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1434238, |
| "perturbed_bytes_per_token": 3.629227386251205 |
| }, |
| "deepseek-v3": { |
| "edits": 8396, |
| "docs_affected": 461, |
| "tokens": 420623, |
| "rel_token_change": -0.022461700070650355, |
| "words_compared": 171765, |
| "words_affected": 8214, |
| "words_vanished": 4, |
| "extra_tokens_per_affected_word": -1.1766496225955685, |
| "seg_change_rate": 0.022949960702122085, |
| "seg_change_rate_affected": 0.4799123447772096, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1434238, |
| "perturbed_bytes_per_token": 3.409794519082409 |
| }, |
| "bloom": { |
| "edits": 8396, |
| "docs_affected": 461, |
| "tokens": 276898, |
| "rel_token_change": -0.022632452084289295, |
| "words_compared": 171765, |
| "words_affected": 8214, |
| "words_vanished": 4, |
| "extra_tokens_per_affected_word": -0.780618456294132, |
| "seg_change_rate": 0.026309201525339852, |
| "seg_change_rate_affected": 0.5501582663744826, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1434238, |
| "perturbed_bytes_per_token": 5.179661824931924 |
| }, |
| "roberta-urdu": { |
| "edits": 8396, |
| "docs_affected": 461, |
| "tokens": 232280, |
| "rel_token_change": -0.08611267395059154, |
| "words_compared": 171765, |
| "words_affected": 8214, |
| "words_vanished": 4, |
| "extra_tokens_per_affected_word": -2.6645970294618944, |
| "seg_change_rate": 0.039169795942130235, |
| "seg_change_rate_affected": 0.8190893596299001, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1434238, |
| "perturbed_bytes_per_token": 6.174608231444808 |
| }, |
| "mt5": { |
| "edits": 8396, |
| "docs_affected": 461, |
| "tokens": 358690, |
| "rel_token_change": -0.032841696551352224, |
| "words_compared": 171765, |
| "words_affected": 8214, |
| "words_vanished": 4, |
| "extra_tokens_per_affected_word": -1.4828341855368883, |
| "seg_change_rate": 0.03309754606584578, |
| "seg_change_rate_affected": 0.6921110299488678, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1434238, |
| "perturbed_bytes_per_token": 3.998544704340796 |
| } |
| }, |
| "digits": { |
| "released": { |
| "edits": 1315, |
| "docs_affected": 175, |
| "tokens": 198501, |
| "rel_token_change": -0.0044585987261146496, |
| "words_compared": 171769, |
| "words_affected": 525, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -1.6933333333333334, |
| "seg_change_rate": 0.002416035489523721, |
| "seg_change_rate_affected": 0.7904761904761904, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1452151, |
| "perturbed_bytes_per_token": 7.315585311912787 |
| }, |
| "mingram48k": { |
| "edits": 1315, |
| "docs_affected": 175, |
| "tokens": 196874, |
| "rel_token_change": 0.00041160000609777786, |
| "words_compared": 171769, |
| "words_affected": 525, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.15428571428571428, |
| "seg_change_rate": 0.0004075240584738806, |
| "seg_change_rate_affected": 0.13333333333333333, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1452151, |
| "perturbed_bytes_per_token": 7.376042544978006 |
| }, |
| "gpt-4o": { |
| "edits": 1315, |
| "docs_affected": 175, |
| "tokens": 334889, |
| "rel_token_change": 0.0010911025815786013, |
| "words_compared": 171769, |
| "words_affected": 525, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.6952380952380952, |
| "seg_change_rate": 0.0020550856091611407, |
| "seg_change_rate_affected": 0.6723809523809524, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1452151, |
| "perturbed_bytes_per_token": 4.336215880485773 |
| }, |
| "gemma-3": { |
| "edits": 1315, |
| "docs_affected": 175, |
| "tokens": 314181, |
| "rel_token_change": -0.001747524076115794, |
| "words_compared": 171769, |
| "words_affected": 525, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -1.0476190476190477, |
| "seg_change_rate": 0.0010886714133516524, |
| "seg_change_rate_affected": 0.35619047619047617, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1452151, |
| "perturbed_bytes_per_token": 4.622020427715234 |
| }, |
| "llama-3": { |
| "edits": 1315, |
| "docs_affected": 175, |
| "tokens": 549560, |
| "rel_token_change": 0.00033310458938033444, |
| "words_compared": 171769, |
| "words_affected": 525, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.3485714285714286, |
| "seg_change_rate": 0.001094493185615565, |
| "seg_change_rate_affected": 0.3580952380952381, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1452151, |
| "perturbed_bytes_per_token": 2.6423884562195212 |
| }, |
| "llama-4": { |
| "edits": 1315, |
| "docs_affected": 175, |
| "tokens": 389625, |
| "rel_token_change": 0.0007294376908673888, |
| "words_compared": 171769, |
| "words_affected": 525, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.540952380952381, |
| "seg_change_rate": 0.0017174228178542111, |
| "seg_change_rate_affected": 0.5619047619047619, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1452151, |
| "perturbed_bytes_per_token": 3.7270478023740776 |
| }, |
| "qwen-3.5": { |
| "edits": 1315, |
| "docs_affected": 175, |
| "tokens": 404389, |
| "rel_token_change": 0.0027897357562291698, |
| "words_compared": 171769, |
| "words_affected": 525, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 2.142857142857143, |
| "seg_change_rate": 0.0030564304385541047, |
| "seg_change_rate_affected": 1.0, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1452151, |
| "perturbed_bytes_per_token": 3.5909755210947876 |
| }, |
| "deepseek-v3": { |
| "edits": 1315, |
| "docs_affected": 175, |
| "tokens": 430635, |
| "rel_token_change": 0.0008064366191945859, |
| "words_compared": 171769, |
| "words_affected": 525, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.660952380952381, |
| "seg_change_rate": 0.0019735807974663646, |
| "seg_change_rate_affected": 0.6457142857142857, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1452151, |
| "perturbed_bytes_per_token": 3.3721155967350542 |
| }, |
| "bloom": { |
| "edits": 1315, |
| "docs_affected": 175, |
| "tokens": 284402, |
| "rel_token_change": 0.0038544350711235045, |
| "words_compared": 171769, |
| "words_affected": 525, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 2.08, |
| "seg_change_rate": 0.0028934208151645525, |
| "seg_change_rate_affected": 0.9466666666666667, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1452151, |
| "perturbed_bytes_per_token": 5.105980267368022 |
| }, |
| "roberta-urdu": { |
| "edits": 1315, |
| "docs_affected": 175, |
| "tokens": 254348, |
| "rel_token_change": 0.0007121302135997199, |
| "words_compared": 171769, |
| "words_affected": 525, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.34476190476190477, |
| "seg_change_rate": 0.0012516810367412048, |
| "seg_change_rate_affected": 0.4095238095238095, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1452151, |
| "perturbed_bytes_per_token": 5.7093077201314735 |
| }, |
| "mt5": { |
| "edits": 1315, |
| "docs_affected": 175, |
| "tokens": 371219, |
| "rel_token_change": 0.0009410305497883356, |
| "words_compared": 171769, |
| "words_affected": 525, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.6647619047619048, |
| "seg_change_rate": 0.0013390076206998935, |
| "seg_change_rate_affected": 0.4380952380952381, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1452151, |
| "perturbed_bytes_per_token": 3.911844490718417 |
| } |
| }, |
| "punct_space": { |
| "released": { |
| "edits": 9695, |
| "docs_affected": 427, |
| "tokens": 214501, |
| "rel_token_change": 0.07578614775063945, |
| "words_compared": 171769, |
| "words_affected": 9451, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 1.5988784255634325, |
| "seg_change_rate": 0.03210125226321397, |
| "seg_change_rate_affected": 0.583430324833351, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1460721, |
| "perturbed_bytes_per_token": 6.809856364305994 |
| }, |
| "mingram48k": { |
| "edits": 9695, |
| "docs_affected": 427, |
| "tokens": 206849, |
| "rel_token_change": 0.05109937853480561, |
| "words_compared": 171769, |
| "words_affected": 9451, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 1.064014390011639, |
| "seg_change_rate": 0.001996867886522015, |
| "seg_change_rate_affected": 0.03629245582478045, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1460721, |
| "perturbed_bytes_per_token": 7.06177453117975 |
| }, |
| "gpt-4o": { |
| "edits": 9695, |
| "docs_affected": 427, |
| "tokens": 335655, |
| "rel_token_change": 0.0033809233418230084, |
| "words_compared": 171769, |
| "words_affected": 9451, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.11966987620357634, |
| "seg_change_rate": 0.0010886714133516524, |
| "seg_change_rate_affected": 0.019786266003597505, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1460721, |
| "perturbed_bytes_per_token": 4.3518523483934395 |
| }, |
| "gemma-3": { |
| "edits": 9695, |
| "docs_affected": 427, |
| "tokens": 316422, |
| "rel_token_change": 0.00537284220493056, |
| "words_compared": 171769, |
| "words_affected": 9451, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.1789228653052587, |
| "seg_change_rate": 0.015194825608811835, |
| "seg_change_rate_affected": 0.2761612527774839, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1460721, |
| "perturbed_bytes_per_token": 4.616369911068131 |
| }, |
| "llama-3": { |
| "edits": 9695, |
| "docs_affected": 427, |
| "tokens": 556185, |
| "rel_token_change": 0.012392218822411568, |
| "words_compared": 171769, |
| "words_affected": 9451, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.7203470532218813, |
| "seg_change_rate": 1.164354452782516e-05, |
| "seg_change_rate_affected": 0.00021161781822029416, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1460721, |
| "perturbed_bytes_per_token": 2.6263221769734892 |
| }, |
| "llama-4": { |
| "edits": 9695, |
| "docs_affected": 427, |
| "tokens": 396146, |
| "rel_token_change": 0.017478251712509087, |
| "words_compared": 171769, |
| "words_affected": 9451, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.7200296264945508, |
| "seg_change_rate": 1.164354452782516e-05, |
| "seg_change_rate_affected": 0.00021161781822029416, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1460721, |
| "perturbed_bytes_per_token": 3.687329923815967 |
| }, |
| "qwen-3.5": { |
| "edits": 9695, |
| "docs_affected": 427, |
| "tokens": 403266, |
| "rel_token_change": 4.959530233296302e-06, |
| "words_compared": 171769, |
| "words_affected": 9451, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.00021161781822029416, |
| "seg_change_rate": 1.164354452782516e-05, |
| "seg_change_rate_affected": 0.00021161781822029416, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1460721, |
| "perturbed_bytes_per_token": 3.622227016411004 |
| }, |
| "deepseek-v3": { |
| "edits": 9695, |
| "docs_affected": 427, |
| "tokens": 437093, |
| "rel_token_change": 0.015814989030602758, |
| "words_compared": 171769, |
| "words_affected": 9451, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.7200296264945508, |
| "seg_change_rate": 1.164354452782516e-05, |
| "seg_change_rate_affected": 0.00021161781822029416, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1460721, |
| "perturbed_bytes_per_token": 3.3418997787656175 |
| }, |
| "bloom": { |
| "edits": 9695, |
| "docs_affected": 427, |
| "tokens": 283492, |
| "rel_token_change": 0.0006424058451872507, |
| "words_compared": 171769, |
| "words_affected": 9451, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.01925722145804677, |
| "seg_change_rate": 0.0010595625520320895, |
| "seg_change_rate_affected": 0.018728176912496033, |
| "seg_changed_unaffected_words": 5, |
| "perturbed_bytes": 1460721, |
| "perturbed_bytes_per_token": 5.152600426114317 |
| }, |
| "roberta-urdu": { |
| "edits": 9695, |
| "docs_affected": 427, |
| "tokens": 254193, |
| "rel_token_change": 0.00010229494780990452, |
| "words_compared": 171769, |
| "words_affected": 9451, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.002751031636863824, |
| "seg_change_rate": 0.0001513660788617271, |
| "seg_change_rate_affected": 0.002751031636863824, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1460721, |
| "perturbed_bytes_per_token": 5.746503640934251 |
| }, |
| "mt5": { |
| "edits": 9695, |
| "docs_affected": 427, |
| "tokens": 377456, |
| "rel_token_change": 0.017758244128670424, |
| "words_compared": 171769, |
| "words_affected": 9451, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.6968574753994287, |
| "seg_change_rate": 0.027851358510557783, |
| "seg_change_rate_affected": 0.5061898211829436, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1460721, |
| "perturbed_bytes_per_token": 3.8699106650841424 |
| } |
| }, |
| "zwnj": { |
| "released": { |
| "edits": 230, |
| "docs_affected": 127, |
| "tokens": 200358, |
| "rel_token_change": 0.004854807161843623, |
| "words_compared": 171769, |
| "words_affected": 230, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 4.208695652173913, |
| "seg_change_rate": 0.0012167504031577294, |
| "seg_change_rate_affected": 0.908695652173913, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451716, |
| "perturbed_bytes_per_token": 7.245610357460146 |
| }, |
| "mingram48k": { |
| "edits": 230, |
| "docs_affected": 127, |
| "tokens": 197705, |
| "rel_token_change": 0.004634311179767573, |
| "words_compared": 171769, |
| "words_affected": 230, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 3.965217391304348, |
| "seg_change_rate": 0.0011759979973103413, |
| "seg_change_rate_affected": 0.8782608695652174, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451716, |
| "perturbed_bytes_per_token": 7.342839078424926 |
| }, |
| "gpt-4o": { |
| "edits": 230, |
| "docs_affected": 127, |
| "tokens": 334703, |
| "rel_token_change": 0.0005350886632947113, |
| "words_compared": 171769, |
| "words_affected": 230, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.7782608695652173, |
| "seg_change_rate": 0.00045409823658518124, |
| "seg_change_rate_affected": 0.3391304347826087, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451716, |
| "perturbed_bytes_per_token": 4.337325927762823 |
| }, |
| "gemma-3": { |
| "edits": 230, |
| "docs_affected": 127, |
| "tokens": 314964, |
| "rel_token_change": 0.000740314744972691, |
| "words_compared": 171769, |
| "words_affected": 230, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 1.0130434782608695, |
| "seg_change_rate": 0.00041916760300170577, |
| "seg_change_rate_affected": 0.3130434782608696, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451716, |
| "perturbed_bytes_per_token": 4.609148982105891 |
| }, |
| "llama-3": { |
| "edits": 230, |
| "docs_affected": 127, |
| "tokens": 549595, |
| "rel_token_change": 0.00039681311740389567, |
| "words_compared": 171769, |
| "words_affected": 230, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.9478260869565217, |
| "seg_change_rate": 0.0004948506424325693, |
| "seg_change_rate_affected": 0.3695652173913043, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451716, |
| "perturbed_bytes_per_token": 2.6414286883978204 |
| }, |
| "llama-4": { |
| "edits": 230, |
| "docs_affected": 127, |
| "tokens": 389510, |
| "rel_token_change": 0.00043406679491756584, |
| "words_compared": 171769, |
| "words_affected": 230, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.7347826086956522, |
| "seg_change_rate": 0.0005763554541273454, |
| "seg_change_rate_affected": 0.43043478260869567, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451716, |
| "perturbed_bytes_per_token": 3.7270313984236605 |
| }, |
| "qwen-3.5": { |
| "edits": 230, |
| "docs_affected": 127, |
| "tokens": 403498, |
| "rel_token_change": 0.0005802650372956674, |
| "words_compared": 171769, |
| "words_affected": 230, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 1.017391304347826, |
| "seg_change_rate": 0.00031437570225127936, |
| "seg_change_rate_affected": 0.23478260869565218, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451716, |
| "perturbed_bytes_per_token": 3.5978270028599892 |
| }, |
| "deepseek-v3": { |
| "edits": 230, |
| "docs_affected": 127, |
| "tokens": 430750, |
| "rel_token_change": 0.0010736994756999963, |
| "words_compared": 171769, |
| "words_affected": 230, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 2.008695652173913, |
| "seg_change_rate": 0.00039588051394605546, |
| "seg_change_rate_affected": 0.2956521739130435, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451716, |
| "perturbed_bytes_per_token": 3.3702054556006966 |
| }, |
| "bloom": { |
| "edits": 230, |
| "docs_affected": 127, |
| "tokens": 283562, |
| "rel_token_change": 0.0008894850164131164, |
| "words_compared": 171769, |
| "words_affected": 230, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 1.0956521739130434, |
| "seg_change_rate": 0.00039588051394605546, |
| "seg_change_rate_affected": 0.2956521739130435, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451716, |
| "perturbed_bytes_per_token": 5.119571733871252 |
| }, |
| "roberta-urdu": { |
| "edits": 230, |
| "docs_affected": 127, |
| "tokens": 254571, |
| "rel_token_change": 0.0015895061121231316, |
| "words_compared": 171769, |
| "words_affected": 230, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 1.7565217391304349, |
| "seg_change_rate": 0.0010420972352403518, |
| "seg_change_rate_affected": 0.7782608695652173, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451716, |
| "perturbed_bytes_per_token": 5.702597703587604 |
| }, |
| "mt5": { |
| "edits": 230, |
| "docs_affected": 127, |
| "tokens": 370909, |
| "rel_token_change": 0.00010515814166689136, |
| "words_compared": 171769, |
| "words_affected": 230, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.16956521739130434, |
| "seg_change_rate": 0.000512315959224307, |
| "seg_change_rate_affected": 0.3826086956521739, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451716, |
| "perturbed_bytes_per_token": 3.913941155377734 |
| } |
| }, |
| "arabic_keyboard": { |
| "released": { |
| "edits": 117777, |
| "docs_affected": 491, |
| "tokens": 383125, |
| "rel_token_change": 0.9214855308691509, |
| "words_compared": 171769, |
| "words_affected": 88504, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 2.076007864051342, |
| "seg_change_rate": 0.4873114473508025, |
| "seg_change_rate_affected": 0.9457764620808099, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451026, |
| "perturbed_bytes_per_token": 3.7873435562805873 |
| }, |
| "mingram48k": { |
| "edits": 117777, |
| "docs_affected": 491, |
| "tokens": 362178, |
| "rel_token_change": 0.8404008272651975, |
| "words_compared": 171769, |
| "words_affected": 88504, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 1.8686726023682545, |
| "seg_change_rate": 0.4871135070938295, |
| "seg_change_rate_affected": 0.9453922986531682, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451026, |
| "perturbed_bytes_per_token": 4.006389123580118 |
| }, |
| "gpt-4o": { |
| "edits": 117777, |
| "docs_affected": 491, |
| "tokens": 355200, |
| "rel_token_change": 0.06180722459375112, |
| "words_compared": 171769, |
| "words_affected": 88504, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.23361655970351622, |
| "seg_change_rate": 0.24658698601028126, |
| "seg_change_rate_affected": 0.47857723944680464, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451026, |
| "perturbed_bytes_per_token": 4.08509572072072 |
| }, |
| "gemma-3": { |
| "edits": 117777, |
| "docs_affected": 491, |
| "tokens": 345517, |
| "rel_token_change": 0.09781686583145607, |
| "words_compared": 171769, |
| "words_affected": 88504, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.34784868480520653, |
| "seg_change_rate": 0.25699049304589305, |
| "seg_change_rate_affected": 0.49876841724667814, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451026, |
| "perturbed_bytes_per_token": 4.199579181342742 |
| }, |
| "llama-3": { |
| "edits": 117777, |
| "docs_affected": 491, |
| "tokens": 498541, |
| "rel_token_change": -0.09253390658873596, |
| "words_compared": 171769, |
| "words_affected": 88504, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -0.5743921178703787, |
| "seg_change_rate": 0.26312081923979297, |
| "seg_change_rate_affected": 0.5106661845792281, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451026, |
| "perturbed_bytes_per_token": 2.910544970223111 |
| }, |
| "llama-4": { |
| "edits": 117777, |
| "docs_affected": 491, |
| "tokens": 403849, |
| "rel_token_change": 0.037262964856000266, |
| "words_compared": 171769, |
| "words_affected": 88504, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.16392479435957696, |
| "seg_change_rate": 0.24229633985177768, |
| "seg_change_rate_affected": 0.47024993220645395, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451026, |
| "perturbed_bytes_per_token": 3.592991439869852 |
| }, |
| "qwen-3.5": { |
| "edits": 117777, |
| "docs_affected": 491, |
| "tokens": 398567, |
| "rel_token_change": -0.011647456752896366, |
| "words_compared": 171769, |
| "words_affected": 88504, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -0.05307104763626503, |
| "seg_change_rate": 0.23542664858036083, |
| "seg_change_rate_affected": 0.4569172014824189, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451026, |
| "perturbed_bytes_per_token": 3.640607476283787 |
| }, |
| "deepseek-v3": { |
| "edits": 117777, |
| "docs_affected": 491, |
| "tokens": 425675, |
| "rel_token_change": -0.010720726583125721, |
| "words_compared": 171769, |
| "words_affected": 88504, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -0.052121937991503206, |
| "seg_change_rate": 0.26968195658122246, |
| "seg_change_rate_affected": 0.5234000723131158, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451026, |
| "perturbed_bytes_per_token": 3.408764902801433 |
| }, |
| "bloom": { |
| "edits": 117777, |
| "docs_affected": 491, |
| "tokens": 322135, |
| "rel_token_change": 0.13704069746920336, |
| "words_compared": 171769, |
| "words_affected": 88504, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.43868073759378107, |
| "seg_change_rate": 0.276836914693571, |
| "seg_change_rate_affected": 0.5372864503299286, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451026, |
| "perturbed_bytes_per_token": 4.504403433343164 |
| }, |
| "roberta-urdu": { |
| "edits": 117777, |
| "docs_affected": 491, |
| "tokens": 512389, |
| "rel_token_change": 1.0159540774372755, |
| "words_compared": 171769, |
| "words_affected": 88504, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 2.917630841543885, |
| "seg_change_rate": 0.515250132445319, |
| "seg_change_rate_affected": 1.0, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451026, |
| "perturbed_bytes_per_token": 2.8318835884454976 |
| }, |
| "mt5": { |
| "edits": 117777, |
| "docs_affected": 491, |
| "tokens": 380856, |
| "rel_token_change": 0.026925876991937877, |
| "words_compared": 171769, |
| "words_affected": 88504, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.1128310584832324, |
| "seg_change_rate": 0.2298144601179491, |
| "seg_change_rate_affected": 0.4460250384163428, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1451026, |
| "perturbed_bytes_per_token": 3.8099071565105973 |
| } |
| }, |
| "nr_nnu": { |
| "released": { |
| "edits": 7353, |
| "docs_affected": 383, |
| "tokens": 204902, |
| "rel_token_change": 0.027644315161241786, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 1.6116959064327485, |
| "seg_change_rate": 0.019572798351274096, |
| "seg_change_rate_affected": 0.9830409356725146, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 7.04334755151243 |
| }, |
| "mingram48k": { |
| "edits": 7353, |
| "docs_affected": 383, |
| "tokens": 202550, |
| "rel_token_change": 0.029254089322282805, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 1.6833333333333333, |
| "seg_change_rate": 0.019619372529385396, |
| "seg_change_rate_affected": 0.9853801169590644, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 7.125134534682794 |
| }, |
| "gpt-4o": { |
| "edits": 7353, |
| "docs_affected": 383, |
| "tokens": 337063, |
| "rel_token_change": 0.007589888916789229, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.7423976608187135, |
| "seg_change_rate": 0.019910461142581024, |
| "seg_change_rate_affected": 1.0, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 4.281680279354305 |
| }, |
| "gemma-3": { |
| "edits": 7353, |
| "docs_affected": 383, |
| "tokens": 314252, |
| "rel_token_change": -0.0015219346044717552, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -0.14005847953216374, |
| "seg_change_rate": 0.0171684064062782, |
| "seg_change_rate_affected": 0.862280701754386, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 4.592479920573298 |
| }, |
| "llama-3": { |
| "edits": 7353, |
| "docs_affected": 383, |
| "tokens": 548053, |
| "rel_token_change": -0.002410002602948431, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -0.3871345029239766, |
| "seg_change_rate": 0.019910461142581024, |
| "seg_change_rate_affected": 1.0, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 2.6333146611732805 |
| }, |
| "llama-4": { |
| "edits": 7353, |
| "docs_affected": 383, |
| "tokens": 390801, |
| "rel_token_change": 0.0037499261572760127, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.4269005847953216, |
| "seg_change_rate": 0.019910461142581024, |
| "seg_change_rate_affected": 1.0, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 3.6929178789204737 |
| }, |
| "qwen-3.5": { |
| "edits": 7353, |
| "docs_affected": 383, |
| "tokens": 404399, |
| "rel_token_change": 0.0028145334073956516, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.33187134502923976, |
| "seg_change_rate": 0.019910461142581024, |
| "seg_change_rate_affected": 1.0, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 3.5687427515893955 |
| }, |
| "deepseek-v3": { |
| "edits": 7353, |
| "docs_affected": 383, |
| "tokens": 432441, |
| "rel_token_change": 0.005003625478749117, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.62953216374269, |
| "seg_change_rate": 0.019910461142581024, |
| "seg_change_rate_affected": 1.0, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 3.3373246292557828 |
| }, |
| "bloom": { |
| "edits": 7353, |
| "docs_affected": 383, |
| "tokens": 286463, |
| "rel_token_change": 0.011129151812502206, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.9219298245614035, |
| "seg_change_rate": 0.019910461142581024, |
| "seg_change_rate_affected": 1.0, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 5.037983963024893 |
| }, |
| "roberta-urdu": { |
| "edits": 7353, |
| "docs_affected": 383, |
| "tokens": 258244, |
| "rel_token_change": 0.01604063470080695, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 1.1921052631578948, |
| "seg_change_rate": 0.019910461142581024, |
| "seg_change_rate_affected": 1.0, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 5.588497699849754 |
| }, |
| "mt5": { |
| "edits": 7353, |
| "docs_affected": 383, |
| "tokens": 369531, |
| "rel_token_change": -0.00361042953056327, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -0.3915204678362573, |
| "seg_change_rate": 0.013978075205654104, |
| "seg_change_rate_affected": 0.702046783625731, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 3.9054801897540394 |
| } |
| }, |
| "nr_n": { |
| "released": { |
| "edits": 3915, |
| "docs_affected": 383, |
| "tokens": 200197, |
| "rel_token_change": 0.004047344400421285, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.23596491228070177, |
| "seg_change_rate": 0.00739947254743289, |
| "seg_change_rate_affected": 0.3716374269005848, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 7.2088792539348745 |
| }, |
| "mingram48k": { |
| "edits": 3915, |
| "docs_affected": 383, |
| "tokens": 197385, |
| "rel_token_change": 0.003008237081603512, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": 0.17309941520467836, |
| "seg_change_rate": 0.004756387939616578, |
| "seg_change_rate_affected": 0.2388888888888889, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 7.311578894039568 |
| }, |
| "gpt-4o": { |
| "edits": 3915, |
| "docs_affected": 383, |
| "tokens": 330488, |
| "rel_token_change": -0.012064904162332149, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -1.1801169590643275, |
| "seg_change_rate": 0.008336777881922814, |
| "seg_change_rate_affected": 0.41871345029239765, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 4.366863547239234 |
| }, |
| "gemma-3": { |
| "edits": 3915, |
| "docs_affected": 383, |
| "tokens": 310620, |
| "rel_token_change": -0.013061948139840053, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -1.202046783625731, |
| "seg_change_rate": 0.007521729764975053, |
| "seg_change_rate_affected": 0.37777777777777777, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 4.646178610520893 |
| }, |
| "llama-3": { |
| "edits": 3915, |
| "docs_affected": 383, |
| "tokens": 540795, |
| "rel_token_change": -0.015621331071377215, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -2.509356725146199, |
| "seg_change_rate": 0.008656975356438007, |
| "seg_change_rate_affected": 0.4347953216374269, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 2.668656330032637 |
| }, |
| "llama-4": { |
| "edits": 3915, |
| "docs_affected": 383, |
| "tokens": 384660, |
| "rel_token_change": -0.01202287968644453, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -1.3687134502923977, |
| "seg_change_rate": 0.008645331811910182, |
| "seg_change_rate_affected": 0.4342105263157895, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 3.751874382571622 |
| }, |
| "qwen-3.5": { |
| "edits": 3915, |
| "docs_affected": 383, |
| "tokens": 398436, |
| "rel_token_change": -0.011972305983177273, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -1.4116959064327486, |
| "seg_change_rate": 0.0063515535399286255, |
| "seg_change_rate_affected": 0.31900584795321635, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 3.62215261673142 |
| }, |
| "deepseek-v3": { |
| "edits": 3915, |
| "docs_affected": 383, |
| "tokens": 425598, |
| "rel_token_change": -0.010899676495742387, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -1.371345029239766, |
| "seg_change_rate": 0.008773410801716258, |
| "seg_change_rate_affected": 0.44064327485380117, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 3.390983980187877 |
| }, |
| "bloom": { |
| "edits": 3915, |
| "docs_affected": 383, |
| "tokens": 279269, |
| "rel_token_change": -0.014263527584624616, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -1.181578947368421, |
| "seg_change_rate": 0.008686084217757569, |
| "seg_change_rate_affected": 0.43625730994152045, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 5.167762981211664 |
| }, |
| "roberta-urdu": { |
| "edits": 3915, |
| "docs_affected": 383, |
| "tokens": 250665, |
| "rel_token_change": -0.013778342585780215, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -1.0239766081871344, |
| "seg_change_rate": 0.009396340433954905, |
| "seg_change_rate_affected": 0.47192982456140353, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 5.757469132108591 |
| }, |
| "mt5": { |
| "edits": 3915, |
| "docs_affected": 383, |
| "tokens": 366789, |
| "rel_token_change": -0.011003855798527785, |
| "words_compared": 171769, |
| "words_affected": 3420, |
| "words_vanished": 0, |
| "extra_tokens_per_affected_word": -1.1932748538011695, |
| "seg_change_rate": 0.006398127718039926, |
| "seg_change_rate_affected": 0.3213450292397661, |
| "seg_changed_unaffected_words": 0, |
| "perturbed_bytes": 1443196, |
| "perturbed_bytes_per_token": 3.934676339802993 |
| } |
| } |
| }, |
| "perturbation_definitions": { |
| "harakat": { |
| "what": "harakat removed", |
| "source": "PLAN 4.2 (i)" |
| }, |
| "digits": { |
| "what": "digit script swapped (ASCII <-> Urdu)", |
| "source": "PLAN 4.2 (ii)" |
| }, |
| "punct_space": { |
| "what": "space before ۔ / ، toggled", |
| "source": "PLAN 4.2 (iii); on this data it inserts the space" |
| }, |
| "zwnj": { |
| "what": "ZWNJ inserted in compounds", |
| "source": "PLAN 4.2 (iv)" |
| }, |
| "arabic_keyboard": { |
| "what": "Arabic-keyboard letters ي ك ه for ی ک ہ", |
| "source": "new: realistic typing" |
| }, |
| "nr_nnu": { |
| "what": "retroflex nasal نڑ written ݨ", |
| "source": "new: Punjabi/Saraiki orthography" |
| }, |
| "nr_n": { |
| "what": "retroflex nasal نڑ written ن", |
| "source": "new: Urdu-style spelling" |
| } |
| }, |
| "arabic_keyboard_then_normalize": { |
| "released": { |
| "tokens": 320462, |
| "rel_token_change": 0.6072119965895982 |
| }, |
| "mingram48k": { |
| "tokens": 305680, |
| "rel_token_change": 0.553307282271219 |
| }, |
| "gpt-4o": { |
| "tokens": 338769, |
| "rel_token_change": 0.012689672489866199 |
| }, |
| "gemma-3": { |
| "tokens": 325430, |
| "rel_token_change": 0.03399410925520524 |
| }, |
| "llama-3": { |
| "tokens": 496906, |
| "rel_token_change": -0.09551000496926519 |
| }, |
| "llama-4": { |
| "tokens": 390443, |
| "rel_token_change": 0.0028304237159713463 |
| }, |
| "qwen-3.5": { |
| "tokens": 392263, |
| "rel_token_change": -0.02727989604824631 |
| }, |
| "deepseek-v3": { |
| "tokens": 418589, |
| "rel_token_change": -0.02718876659353735 |
| }, |
| "bloom": { |
| "tokens": 301426, |
| "rel_token_change": 0.06394408951325403 |
| }, |
| "roberta-urdu": { |
| "tokens": 427728, |
| "rel_token_change": 0.6828620552628784 |
| }, |
| "mt5": { |
| "tokens": 366633, |
| "rel_token_change": -0.011424488365195351 |
| } |
| }, |
| "arabic_keyboard_mitigation": { |
| "released": { |
| "clean": 199390, |
| "ak": 383125, |
| "ak_fold_yk": 250740, |
| "rel_ak": 0.9214855308691509, |
| "rel_ak_fold_yk": 0.2575354832238327 |
| }, |
| "mingram48k": { |
| "clean": 196793, |
| "ak": 362178, |
| "ak_fold_yk": 248035, |
| "rel_ak": 0.8404008272651975, |
| "rel_ak_fold_yk": 0.26038527793163374 |
| }, |
| "gpt-4o": { |
| "clean": 334524, |
| "ak": 355200, |
| "ak_fold_yk": 332301, |
| "rel_ak": 0.06180722459375112, |
| "rel_ak_fold_yk": -0.006645263120134878 |
| }, |
| "gemma-3": { |
| "clean": 314731, |
| "ak": 345517, |
| "ak_fold_yk": 314859, |
| "rel_ak": 0.09781686583145607, |
| "rel_ak_fold_yk": 0.0004066965122596757 |
| }, |
| "llama-3": { |
| "clean": 549377, |
| "ak": 498541, |
| "ak_fold_yk": 491651, |
| "rel_ak": -0.09253390658873596, |
| "rel_ak_fold_yk": -0.10507538539108845 |
| }, |
| "llama-4": { |
| "clean": 389341, |
| "ak": 403849, |
| "ak_fold_yk": 381301, |
| "rel_ak": 0.037262964856000266, |
| "rel_ak_fold_yk": -0.020650278290752836 |
| }, |
| "qwen-3.5": { |
| "clean": 403264, |
| "ak": 398567, |
| "ak_fold_yk": 393275, |
| "rel_ak": -0.011647456752896366, |
| "rel_ak_fold_yk": -0.02477037375019838 |
| }, |
| "deepseek-v3": { |
| "clean": 430288, |
| "ak": 425675, |
| "ak_fold_yk": 412337, |
| "rel_ak": -0.010720726583125721, |
| "rel_ak_fold_yk": -0.041718569888074965 |
| }, |
| "bloom": { |
| "clean": 283310, |
| "ak": 322135, |
| "ak_fold_yk": 295665, |
| "rel_ak": 0.13704069746920336, |
| "rel_ak_fold_yk": 0.04360947372136529 |
| }, |
| "roberta-urdu": { |
| "clean": 254167, |
| "ak": 512389, |
| "ak_fold_yk": 316036, |
| "rel_ak": 1.0159540774372755, |
| "rel_ak_fold_yk": 0.243418697155807 |
| }, |
| "mt5": { |
| "clean": 370870, |
| "ak": 380856, |
| "ak_fold_yk": 363346, |
| "rel_ak": 0.026925876991937877, |
| "rel_ak_fold_yk": -0.020287432253889504 |
| } |
| }, |
| "codemixed": { |
| "released": { |
| "mixed": { |
| "docs": 100, |
| "bytes": 17010, |
| "chars": 10311, |
| "words": 2059, |
| "tokens": 3005, |
| "pieces": 2956, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 5.6605657237936775, |
| "chars_per_token": 3.4312811980033278, |
| "fertility": 1.4356483729966003 |
| }, |
| "base": { |
| "docs": 100, |
| "bytes": 13409, |
| "chars": 7436, |
| "words": 1563, |
| "tokens": 1706, |
| "pieces": 1704, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 7.859906213364596, |
| "chars_per_token": 4.35873388042204, |
| "fertility": 1.09021113243762 |
| }, |
| "added_bytes": 3601, |
| "added_tokens": 1299 |
| }, |
| "mingram48k": { |
| "mixed": { |
| "docs": 100, |
| "bytes": 17010, |
| "chars": 10311, |
| "words": 2059, |
| "tokens": 2941, |
| "pieces": 2919, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 5.78374702482149, |
| "chars_per_token": 3.505950357021421, |
| "fertility": 1.4176784847013113 |
| }, |
| "base": { |
| "docs": 100, |
| "bytes": 13409, |
| "chars": 7436, |
| "words": 1563, |
| "tokens": 1734, |
| "pieces": 1734, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 7.732987312572088, |
| "chars_per_token": 4.288350634371396, |
| "fertility": 1.109404990403071 |
| }, |
| "added_bytes": 3601, |
| "added_tokens": 1207 |
| }, |
| "gpt-4o": { |
| "mixed": { |
| "docs": 100, |
| "bytes": 17010, |
| "chars": 10311, |
| "words": 2059, |
| "tokens": 3631, |
| "pieces": 3631, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.684659873313137, |
| "chars_per_token": 2.8397135775268523, |
| "fertility": 1.7634774162214668 |
| }, |
| "base": { |
| "docs": 100, |
| "bytes": 13409, |
| "chars": 7436, |
| "words": 1563, |
| "tokens": 3024, |
| "pieces": 3024, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.434193121693122, |
| "chars_per_token": 2.458994708994709, |
| "fertility": 1.9347408829174664 |
| }, |
| "added_bytes": 3601, |
| "added_tokens": 607 |
| }, |
| "gemma-3": { |
| "mixed": { |
| "docs": 100, |
| "bytes": 17010, |
| "chars": 10311, |
| "words": 2059, |
| "tokens": 3374, |
| "pieces": 3374, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 5.04149377593361, |
| "chars_per_token": 3.0560165975103732, |
| "fertility": 1.6386595434677027 |
| }, |
| "base": { |
| "docs": 100, |
| "bytes": 13409, |
| "chars": 7436, |
| "words": 1563, |
| "tokens": 2818, |
| "pieces": 2818, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.758339247693399, |
| "chars_per_token": 2.6387508871540097, |
| "fertility": 1.8029430582213692 |
| }, |
| "added_bytes": 3601, |
| "added_tokens": 556 |
| }, |
| "llama-3": { |
| "mixed": { |
| "docs": 100, |
| "bytes": 17010, |
| "chars": 10311, |
| "words": 2059, |
| "tokens": 5880, |
| "pieces": 5770, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.892857142857143, |
| "chars_per_token": 1.7535714285714286, |
| "fertility": 2.802331228751821 |
| }, |
| "base": { |
| "docs": 100, |
| "bytes": 13409, |
| "chars": 7436, |
| "words": 1563, |
| "tokens": 4844, |
| "pieces": 4764, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 2.7681668042939718, |
| "chars_per_token": 1.5350949628406276, |
| "fertility": 3.047984644913628 |
| }, |
| "added_bytes": 3601, |
| "added_tokens": 1036 |
| }, |
| "llama-4": { |
| "mixed": { |
| "docs": 100, |
| "bytes": 17010, |
| "chars": 10311, |
| "words": 2059, |
| "tokens": 4178, |
| "pieces": 4178, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.071325993298228, |
| "chars_per_token": 2.467927237912877, |
| "fertility": 2.0291403593977657 |
| }, |
| "base": { |
| "docs": 100, |
| "bytes": 13409, |
| "chars": 7436, |
| "words": 1563, |
| "tokens": 3510, |
| "pieces": 3510, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.82022792022792, |
| "chars_per_token": 2.1185185185185187, |
| "fertility": 2.2456813819577737 |
| }, |
| "added_bytes": 3601, |
| "added_tokens": 668 |
| }, |
| "qwen-3.5": { |
| "mixed": { |
| "docs": 100, |
| "bytes": 17010, |
| "chars": 10311, |
| "words": 2059, |
| "tokens": 4340, |
| "pieces": 4339, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.9193548387096775, |
| "chars_per_token": 2.3758064516129034, |
| "fertility": 2.1073336571151042 |
| }, |
| "base": { |
| "docs": 100, |
| "bytes": 13409, |
| "chars": 7436, |
| "words": 1563, |
| "tokens": 3623, |
| "pieces": 3622, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.7010764559757106, |
| "chars_per_token": 2.052442727021805, |
| "fertility": 2.3173384516954574 |
| }, |
| "added_bytes": 3601, |
| "added_tokens": 717 |
| }, |
| "deepseek-v3": { |
| "mixed": { |
| "docs": 100, |
| "bytes": 17010, |
| "chars": 10311, |
| "words": 2059, |
| "tokens": 4637, |
| "pieces": 4628, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.6683200345050677, |
| "chars_per_token": 2.223635971533319, |
| "fertility": 2.2476930548810103 |
| }, |
| "base": { |
| "docs": 100, |
| "bytes": 13409, |
| "chars": 7436, |
| "words": 1563, |
| "tokens": 3835, |
| "pieces": 3826, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 3.496479791395046, |
| "chars_per_token": 1.9389830508474577, |
| "fertility": 2.4478566858605246 |
| }, |
| "added_bytes": 3601, |
| "added_tokens": 802 |
| }, |
| "bloom": { |
| "mixed": { |
| "docs": 100, |
| "bytes": 17010, |
| "chars": 10311, |
| "words": 2059, |
| "tokens": 3067, |
| "pieces": 3064, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 5.546136289533746, |
| "chars_per_token": 3.3619171829149006, |
| "fertility": 1.4881010199125788 |
| }, |
| "base": { |
| "docs": 100, |
| "bytes": 13409, |
| "chars": 7436, |
| "words": 1563, |
| "tokens": 2470, |
| "pieces": 2467, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 5.4287449392712555, |
| "chars_per_token": 3.0105263157894737, |
| "fertility": 1.5783749200255919 |
| }, |
| "added_bytes": 3601, |
| "added_tokens": 597 |
| }, |
| "roberta-urdu": { |
| "mixed": { |
| "docs": 100, |
| "bytes": 17010, |
| "chars": 10311, |
| "words": 2059, |
| "tokens": 3466, |
| "pieces": 3456, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 4.907674552798615, |
| "chars_per_token": 2.9748990190421236, |
| "fertility": 1.6784847013113162 |
| }, |
| "base": { |
| "docs": 100, |
| "bytes": 13409, |
| "chars": 7436, |
| "words": 1563, |
| "tokens": 2242, |
| "pieces": 2242, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": null, |
| "bytes_per_token": 5.9808206958073145, |
| "chars_per_token": 3.3166815343443354, |
| "fertility": 1.434420985284709 |
| }, |
| "added_bytes": 3601, |
| "added_tokens": 1224 |
| }, |
| "mt5": { |
| "mixed": { |
| "docs": 100, |
| "bytes": 17010, |
| "chars": 10311, |
| "words": 2059, |
| "tokens": 4145, |
| "pieces": 3564, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 4.103739445114596, |
| "chars_per_token": 2.487575392038601, |
| "fertility": 1.7309373482272947 |
| }, |
| "base": { |
| "docs": 100, |
| "bytes": 13409, |
| "chars": 7436, |
| "words": 1563, |
| "tokens": 3397, |
| "pieces": 2902, |
| "lossless_docs": 100, |
| "unk": 0, |
| "byte_fallback_tokens": 0, |
| "bytes_per_token": 3.947306446864881, |
| "chars_per_token": 2.18899028554607, |
| "fertility": 1.8566858605246321 |
| }, |
| "added_bytes": 3601, |
| "added_tokens": 748 |
| } |
| }, |
| "summary_numbers": { |
| "bytes_per_token": { |
| "ooc_gotquestions_southern_hindko": { |
| "released": 6.18207455715167, |
| "mingram48k": 6.54365671641791, |
| "gpt-4o": 4.004566992064851, |
| "gemma-3": 4.149050689063701, |
| "llama-3": 2.4397607122982747, |
| "llama-4": 3.5698727735368956, |
| "qwen-3.5": 3.447076167076167, |
| "deepseek-v3": 3.262849434857435, |
| "bloom": 4.614392843046968, |
| "roberta-urdu": 4.7394094993581515, |
| "mt5": 3.6672940192388124 |
| }, |
| "ooc_translatewiki_hno": { |
| "released": 3.6206987128972945, |
| "mingram48k": 3.684576316492916, |
| "gpt-4o": 3.2183049264534205, |
| "gemma-3": 3.2108082925692987, |
| "llama-3": 2.424626209322779, |
| "llama-4": 2.971970677015955, |
| "qwen-3.5": 2.8263276604469962, |
| "deepseek-v3": 2.7818365287588294, |
| "bloom": 3.5608369930250583, |
| "roberta-urdu": 3.573761991184859, |
| "mt5": 3.7254054054054055 |
| }, |
| "ooc_chup_di_kahani_pdf": { |
| "released": 4.407791479820628, |
| "mingram48k": 4.733992689744141, |
| "gpt-4o": 3.39634108718455, |
| "gemma-3": 3.5027840529447327, |
| "llama-3": 2.4117466646219903, |
| "llama-4": 3.15026040176272, |
| "qwen-3.5": 3.023426342963858, |
| "deepseek-v3": 2.902425520696019, |
| "bloom": 3.814719844762466, |
| "roberta-urdu": 4.2031536346976175, |
| "mt5": 3.8792416928010147 |
| }, |
| "ooc_finepdfs_hindko_org": { |
| "released": 4.973251313243019, |
| "mingram48k": 5.1774680993955675, |
| "gpt-4o": 4.33904880598215, |
| "gemma-3": 4.532757969005922, |
| "llama-3": 2.9337836552183427, |
| "llama-4": 3.9183684583129117, |
| "qwen-3.5": 3.781824871228845, |
| "deepseek-v3": 3.52474649336229, |
| "bloom": 4.913368083217627, |
| "roberta-urdu": 4.888332804927759, |
| "mt5": 4.128665149283706 |
| }, |
| "lang_urdu": { |
| "released": 6.594187395025194, |
| "mingram48k": 6.910400536423356, |
| "gpt-4o": 5.346765564202335, |
| "gemma-3": 5.67378064516129, |
| "llama-3": 2.7946460916596227, |
| "llama-4": 4.543777405010316, |
| "qwen-3.5": 4.282800670103762, |
| "deepseek-v3": 3.591411022901761, |
| "bloom": 6.489951343346732, |
| "roberta-urdu": 6.956315850533132, |
| "mt5": 4.541790613121798 |
| }, |
| "lang_punjabi_shahmukhi": { |
| "released": 6.532921843687375, |
| "mingram48k": 6.802324804967894, |
| "gpt-4o": 4.380665875919602, |
| "gemma-3": 4.639859263982511, |
| "llama-3": 2.676207822799386, |
| "llama-4": 3.7797844315820717, |
| "qwen-3.5": 3.602831047805982, |
| "deepseek-v3": 3.3931945863125637, |
| "bloom": 5.102992987143096, |
| "roberta-urdu": 5.83149122514108, |
| "mt5": 3.991970166507202 |
| }, |
| "lang_saraiki": { |
| "released": 4.951328788538091, |
| "mingram48k": 5.150849603071481, |
| "gpt-4o": 3.7289791657816322, |
| "gemma-3": 3.9283944697514586, |
| "llama-3": 2.3937947168605818, |
| "llama-4": 3.3021542330935563, |
| "qwen-3.5": 3.1509206103386296, |
| "deepseek-v3": 2.996052885002182, |
| "bloom": 4.091137588595816, |
| "roberta-urdu": 4.416722374370144, |
| "mt5": 3.6746215190668274 |
| }, |
| "lang_pashto": { |
| "released": 3.2149393569216005, |
| "mingram48k": 3.338110553674364, |
| "gpt-4o": 4.855828473526408, |
| "gemma-3": 4.434776067193007, |
| "llama-3": 3.292977139745004, |
| "llama-4": 4.113751186289287, |
| "qwen-3.5": 3.6412864890239534, |
| "deepseek-v3": 3.3988209688810733, |
| "bloom": 3.4807178515231763, |
| "roberta-urdu": 2.9981564031297916, |
| "mt5": 4.183671731401117 |
| }, |
| "lang_english": { |
| "released": 1.8235120511559273, |
| "mingram48k": 2.04388576469291, |
| "gpt-4o": 4.7165394402035625, |
| "gemma-3": 4.640961442163245, |
| "llama-3": 4.691470513793976, |
| "llama-4": 4.758921694480103, |
| "qwen-3.5": 4.642123716503882, |
| "deepseek-v3": 4.634, |
| "bloom": 4.696224980998227, |
| "roberta-urdu": 2.0036752783482865, |
| "mt5": 3.841658031088083 |
| }, |
| "lang_python_code": { |
| "released": 1.366190979006093, |
| "mingram48k": 1.4908623394067075, |
| "gpt-4o": 4.141571095422164, |
| "gemma-3": 3.619428903257507, |
| "llama-3": 4.163062675409583, |
| "llama-4": 4.144934393943269, |
| "qwen-3.5": 3.8558010885405825, |
| "deepseek-v3": 3.9802405762039554, |
| "bloom": 4.147161940833248, |
| "roberta-urdu": 1.5030131307112102, |
| "mt5": 3.6160973786532336 |
| }, |
| "test_strict": { |
| "released": 7.277325843823662, |
| "mingram48k": 7.373361857383139, |
| "gpt-4o": 4.337584149418278, |
| "gemma-3": 4.61036885467272, |
| "llama-3": 2.64122087382617, |
| "llama-4": 3.7268769536216326, |
| "qwen-3.5": 3.5982036581495, |
| "deepseek-v3": 3.372220466292344, |
| "bloom": 5.121690021531185, |
| "roberta-urdu": 5.708947266954404, |
| "mt5": 3.9124922479575055 |
| } |
| }, |
| "best_external": { |
| "ooc_gotquestions_southern_hindko": "roberta-urdu", |
| "ooc_translatewiki_hno": "mt5", |
| "ooc_chup_di_kahani_pdf": "roberta-urdu", |
| "ooc_finepdfs_hindko_org": "bloom", |
| "lang_urdu": "roberta-urdu", |
| "lang_punjabi_shahmukhi": "roberta-urdu", |
| "lang_saraiki": "roberta-urdu", |
| "lang_pashto": "gpt-4o", |
| "lang_english": "llama-4", |
| "lang_python_code": "llama-3", |
| "test_strict": "roberta-urdu" |
| }, |
| "released_fewer_tokens_vs_best_external": { |
| "ooc_gotquestions_southern_hindko": 0.23336261063441655, |
| "ooc_translatewiki_hno": -0.028918918918918957, |
| "ooc_chup_di_kahani_pdf": 0.04642638973732449, |
| "ooc_finepdfs_hindko_org": 0.012041062527029855, |
| "lang_urdu": -0.05491631247561091, |
| "lang_punjabi_shahmukhi": 0.10736859177705804, |
| "lang_saraiki": 0.10797231147434927, |
| "lang_pashto": -0.5103950446443282, |
| "lang_english": -1.6097560975609757, |
| "lang_python_code": -2.047204043491943, |
| "test_strict": 0.21551578293012075 |
| }, |
| "released_rank_of_11": { |
| "ooc_gotquestions_southern_hindko": 2, |
| "ooc_translatewiki_hno": 3, |
| "ooc_chup_di_kahani_pdf": 2, |
| "ooc_finepdfs_hindko_org": 2, |
| "lang_urdu": 3, |
| "lang_punjabi_shahmukhi": 2, |
| "lang_saraiki": 2, |
| "lang_pashto": 10, |
| "lang_english": 11, |
| "lang_python_code": 11, |
| "test_strict": 2 |
| } |
| }, |
| "word_oov_vs_train_D2": { |
| "train_types": 85715, |
| "test_strict": { |
| "words": 172839, |
| "token_oov": 0.016211618905455367, |
| "types": 15890, |
| "type_oov": 0.14631843926998112 |
| }, |
| "ooc_gotquestions_southern_hindko": { |
| "words": 7472, |
| "token_oov": 0.06731798715203426, |
| "types": 1170, |
| "type_oov": 0.13333333333333333 |
| }, |
| "ooc_translatewiki_hno": { |
| "words": 1518, |
| "token_oov": 0.16798418972332016, |
| "types": 443, |
| "type_oov": 0.1873589164785553 |
| }, |
| "ooc_chup_di_kahani_pdf": { |
| "words": 5253, |
| "token_oov": 0.6809442223491339, |
| "types": 3765, |
| "type_oov": 0.8409030544488711 |
| }, |
| "ooc_finepdfs_hindko_org": { |
| "words": 25855, |
| "token_oov": 0.20266872945271708, |
| "types": 6708, |
| "type_oov": 0.48434704830053665 |
| } |
| } |
| } |
|
|
