| { |
| "transformers": "5.18.0", |
| "tokenizers": "0.23.2", |
| "sentencepiece": "0.2.2", |
| "autotokenizer_trust_remote_code": "MumeTokenizer", |
| "autotokenizer_default": "TokenizersBackend", |
| "sets": { |
| "fineweb_val": { |
| "group": "english", |
| "records": 14983, |
| "words": 7770317, |
| "utf8_bytes": 46515251, |
| "tokens": 11831667, |
| "tokens_per_word": 1.5227, |
| "bytes_per_token": 3.9314, |
| "ids_identical_exact": 14983, |
| "ids_identical_fast": 14959, |
| "roundtrip_exact": 14983, |
| "roundtrip_spm": 14983, |
| "roundtrip_fast": 14983, |
| "fast_differs_at": [ |
| 210, |
| 492, |
| 699, |
| 1298, |
| 3648 |
| ], |
| "gpt2_tokens": 10470777, |
| "gpt2_tokens_per_word": 1.3475 |
| }, |
| "fineweb_val_clean": { |
| "group": "english", |
| "records": 11048, |
| "words": 4782318, |
| "utf8_bytes": 28445808, |
| "tokens": 7177975, |
| "tokens_per_word": 1.5009, |
| "bytes_per_token": 3.9629, |
| "ids_identical_exact": 11048, |
| "ids_identical_fast": 11041, |
| "roundtrip_exact": 11048, |
| "roundtrip_spm": 11048, |
| "roundtrip_fast": 11048, |
| "fast_differs_at": [ |
| 155, |
| 2866, |
| 3549, |
| 4095, |
| 7837 |
| ], |
| "gpt2_tokens": 6392995, |
| "gpt2_tokens_per_word": 1.3368 |
| }, |
| "wikitext103_test": { |
| "group": "english", |
| "records": 62, |
| "words": 241211, |
| "utf8_bytes": 1287656, |
| "tokens": 363932, |
| "tokens_per_word": 1.5088, |
| "bytes_per_token": 3.5382, |
| "ids_identical_exact": 62, |
| "ids_identical_fast": 62, |
| "roundtrip_exact": 62, |
| "roundtrip_spm": 62, |
| "roundtrip_fast": 62, |
| "fast_differs_at": [], |
| "gpt2_tokens": 283287, |
| "gpt2_tokens_per_word": 1.1744 |
| }, |
| "enwik8_test": { |
| "group": "english", |
| "records": 1, |
| "words": 648280, |
| "utf8_bytes": 5000000, |
| "tokens": 1717932, |
| "tokens_per_word": 2.65, |
| "bytes_per_token": 2.9105, |
| "ids_identical_exact": 1, |
| "ids_identical_fast": 0, |
| "roundtrip_exact": 1, |
| "roundtrip_spm": 1, |
| "roundtrip_fast": 1, |
| "fast_differs_at": [ |
| 0 |
| ], |
| "gpt2_tokens": 1480280, |
| "gpt2_tokens_per_word": 2.2834 |
| }, |
| "text8_test": { |
| "group": "english", |
| "records": 1, |
| "words": 855233, |
| "utf8_bytes": 5000000, |
| "tokens": 1145389, |
| "tokens_per_word": 1.3393, |
| "bytes_per_token": 4.3653, |
| "ids_identical_exact": 1, |
| "ids_identical_fast": 0, |
| "roundtrip_exact": 1, |
| "roundtrip_spm": 1, |
| "roundtrip_fast": 1, |
| "fast_differs_at": [ |
| 0 |
| ], |
| "gpt2_tokens": 1003900, |
| "gpt2_tokens_per_word": 1.1738 |
| }, |
| "owm_val": { |
| "group": "math", |
| "records": 2704, |
| "words": 3157209, |
| "utf8_bytes": 20001168, |
| "tokens": 6124528, |
| "tokens_per_word": 1.9399, |
| "bytes_per_token": 3.2657, |
| "ids_identical_exact": 2704, |
| "ids_identical_fast": 2670, |
| "roundtrip_exact": 2703, |
| "roundtrip_spm": 2703, |
| "roundtrip_fast": 2703, |
| "fast_differs_at": [ |
| 13, |
| 81, |
| 82, |
| 297, |
| 431 |
| ], |
| "gpt2_tokens": 5540067, |
| "gpt2_tokens_per_word": 1.7547 |
| }, |
| "owm_val_clean": { |
| "group": "math", |
| "records": 1126, |
| "words": 768667, |
| "utf8_bytes": 4780645, |
| "tokens": 1436589, |
| "tokens_per_word": 1.8689, |
| "bytes_per_token": 3.3278, |
| "ids_identical_exact": 1126, |
| "ids_identical_fast": 1119, |
| "roundtrip_exact": 1126, |
| "roundtrip_spm": 1126, |
| "roundtrip_fast": 1126, |
| "fast_differs_at": [ |
| 7, |
| 36, |
| 235, |
| 313, |
| 547 |
| ], |
| "gpt2_tokens": 1307204, |
| "gpt2_tokens_per_word": 1.7006 |
| }, |
| "gsm8k_test": { |
| "group": "math", |
| "records": 1319, |
| "words": 130627, |
| "utf8_bytes": 652230, |
| "tokens": 224032, |
| "tokens_per_word": 1.7151, |
| "bytes_per_token": 2.9113, |
| "ids_identical_exact": 1319, |
| "ids_identical_fast": 1319, |
| "roundtrip_exact": 1319, |
| "roundtrip_spm": 1319, |
| "roundtrip_fast": 1319, |
| "fast_differs_at": [], |
| "gpt2_tokens": 173987, |
| "gpt2_tokens_per_word": 1.3319 |
| }, |
| "math_test": { |
| "group": "math", |
| "records": 5000, |
| "words": 577396, |
| "utf8_bytes": 3565468, |
| "tokens": 1531141, |
| "tokens_per_word": 2.6518, |
| "bytes_per_token": 2.3286, |
| "ids_identical_exact": 5000, |
| "ids_identical_fast": 4997, |
| "roundtrip_exact": 5000, |
| "roundtrip_spm": 5000, |
| "roundtrip_fast": 5000, |
| "fast_differs_at": [ |
| 180, |
| 510, |
| 1083 |
| ], |
| "gpt2_tokens": 1457380, |
| "gpt2_tokens_per_word": 2.5241 |
| }, |
| "math_test_clean": { |
| "group": "math", |
| "records": 3754, |
| "words": 393713, |
| "utf8_bytes": 2418527, |
| "tokens": 1014818, |
| "tokens_per_word": 2.5776, |
| "bytes_per_token": 2.3832, |
| "ids_identical_exact": 3754, |
| "ids_identical_fast": 3751, |
| "roundtrip_exact": 3754, |
| "roundtrip_spm": 3754, |
| "roundtrip_fast": 3754, |
| "fast_differs_at": [ |
| 140, |
| 393, |
| 838 |
| ], |
| "gpt2_tokens": 958028, |
| "gpt2_tokens_per_word": 2.4333 |
| }, |
| "sanskrit_dcs_gold_slp1": { |
| "group": "sanskrit", |
| "records": 3000, |
| "words": 16596, |
| "utf8_bytes": 139276, |
| "tokens": 39143, |
| "tokens_per_word": 2.3586, |
| "bytes_per_token": 3.5581, |
| "ids_identical_exact": 3000, |
| "ids_identical_fast": 3000, |
| "roundtrip_exact": 3000, |
| "roundtrip_spm": 3000, |
| "roundtrip_fast": 3000, |
| "fast_differs_at": [] |
| }, |
| "sanskrit_dcs_gold_devanagari": { |
| "group": "sanskrit", |
| "records": 3000, |
| "words": 16596, |
| "utf8_bytes": 370794, |
| "tokens": 164598, |
| "tokens_per_word": 9.9179, |
| "bytes_per_token": 2.2527, |
| "ids_identical_exact": 3000, |
| "ids_identical_fast": 3000, |
| "roundtrip_exact": 3000, |
| "roundtrip_spm": 3000, |
| "roundtrip_fast": 3000, |
| "fast_differs_at": [] |
| }, |
| "sanskrit_gita_slp1": { |
| "group": "sanskrit", |
| "records": 700, |
| "words": 7814, |
| "utf8_bytes": 67003, |
| "tokens": 18460, |
| "tokens_per_word": 2.3624, |
| "bytes_per_token": 3.6296, |
| "ids_identical_exact": 700, |
| "ids_identical_fast": 700, |
| "roundtrip_exact": 700, |
| "roundtrip_spm": 700, |
| "roundtrip_fast": 700, |
| "fast_differs_at": [] |
| }, |
| "sanskrit_gita_devanagari": { |
| "group": "sanskrit", |
| "records": 700, |
| "words": 7814, |
| "utf8_bytes": 166845, |
| "tokens": 72265, |
| "tokens_per_word": 9.2481, |
| "bytes_per_token": 2.3088, |
| "ids_identical_exact": 700, |
| "ids_identical_fast": 700, |
| "roundtrip_exact": 700, |
| "roundtrip_spm": 700, |
| "roundtrip_fast": 700, |
| "fast_differs_at": [] |
| }, |
| "sanskrit_ood_slp1": { |
| "group": "sanskrit", |
| "records": 2000, |
| "words": 132896, |
| "utf8_bytes": 1220801, |
| "tokens": 347994, |
| "tokens_per_word": 2.6185, |
| "bytes_per_token": 3.5081, |
| "ids_identical_exact": 2000, |
| "ids_identical_fast": 2000, |
| "roundtrip_exact": 2000, |
| "roundtrip_spm": 2000, |
| "roundtrip_fast": 2000, |
| "fast_differs_at": [] |
| }, |
| "sanskrit_ood_devanagari": { |
| "group": "sanskrit", |
| "records": 2000, |
| "words": 132896, |
| "utf8_bytes": 2921785, |
| "tokens": 1288809, |
| "tokens_per_word": 9.6979, |
| "bytes_per_token": 2.267, |
| "ids_identical_exact": 2000, |
| "ids_identical_fast": 2000, |
| "roundtrip_exact": 2000, |
| "roundtrip_spm": 2000, |
| "roundtrip_fast": 2000, |
| "fast_differs_at": [] |
| }, |
| "sanskrit_prose_slp1": { |
| "group": "sanskrit", |
| "records": 2470, |
| "words": 55243, |
| "utf8_bytes": 420458, |
| "tokens": 118897, |
| "tokens_per_word": 2.1523, |
| "bytes_per_token": 3.5363, |
| "ids_identical_exact": 2470, |
| "ids_identical_fast": 2469, |
| "roundtrip_exact": 2470, |
| "roundtrip_spm": 2470, |
| "roundtrip_fast": 2470, |
| "fast_differs_at": [ |
| 1495 |
| ] |
| }, |
| "sanskrit_prose_devanagari": { |
| "group": "sanskrit", |
| "records": 2470, |
| "words": 55243, |
| "utf8_bytes": 1014824, |
| "tokens": 464012, |
| "tokens_per_word": 8.3995, |
| "bytes_per_token": 2.1871, |
| "ids_identical_exact": 2470, |
| "ids_identical_fast": 2470, |
| "roundtrip_exact": 2470, |
| "roundtrip_spm": 2470, |
| "roundtrip_fast": 2470, |
| "fast_differs_at": [] |
| }, |
| "sanskrit_vedic_slp1": { |
| "group": "sanskrit", |
| "records": 1000, |
| "words": 4079, |
| "utf8_bytes": 29125, |
| "tokens": 13336, |
| "tokens_per_word": 3.2694, |
| "bytes_per_token": 2.1839, |
| "ids_identical_exact": 1000, |
| "ids_identical_fast": 1000, |
| "roundtrip_exact": 1000, |
| "roundtrip_spm": 1000, |
| "roundtrip_fast": 1000, |
| "fast_differs_at": [] |
| }, |
| "sanskrit_vedic_devanagari": { |
| "group": "sanskrit", |
| "records": 1000, |
| "words": 4079, |
| "utf8_bytes": 77606, |
| "tokens": 41505, |
| "tokens_per_word": 10.1753, |
| "bytes_per_token": 1.8698, |
| "ids_identical_exact": 1000, |
| "ids_identical_fast": 1000, |
| "roundtrip_exact": 1000, |
| "roundtrip_spm": 1000, |
| "roundtrip_fast": 1000, |
| "fast_differs_at": [] |
| } |
| }, |
| "edge_cases": [ |
| { |
| "text": "", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": " ", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": " ", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "a", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": " a", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "a ", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": " two leading spaces", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "trailing spaces ", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "tab\tseparated\tvalues", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "line one\nline two\n\nafter a blank line\n", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "\n", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "\r\n windows line ends \r\n", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "2026", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "x2y", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "3.14159", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "1,234,567.89 and -42", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "$12 for 3 x 4 = 12", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "Let $f(x) = x^2 + 1$. Then \\frac{a}{b} \\geq \\sqrt{2}", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "\\begin{align} a &= b \\\\ c &= d \\end{align}", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "</s> <s> <unk> literal special-token text", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "<0x41>", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "▁ already has a meta-symbol", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": false |
| }, |
| { |
| "text": "Ünïcödé café naïve façade", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "日本語のテキスト", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "emoji 🙂👍🏽", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "Ελληνικά and русский", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "संस्कृतं नाम दैवी वाक् । ॥ १२३ ॥", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "saMskftaM nAma dEvI vAk", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "zerowidth", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "combining é vs é", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": " no-break space", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "word word word word word word word word word word word word ", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "012345678901234567890123456789012345678901234567890123456789", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| }, |
| { |
| "text": "''' \"\"\" ``` ~~~", |
| "ids_equal_exact": true, |
| "ids_equal_fast": true, |
| "decode_equal_spm_decode_exact": true, |
| "decode_equal_spm_decode_fast": true, |
| "spm_roundtrip": true |
| } |
| ], |
| "edge_cases_ids_equal_exact": "35/35", |
| "edge_cases_ids_equal_fast": "35/35", |
| "edge_cases_decode_equal_spm_decode_exact": "35/35", |
| "edge_cases_decode_equal_spm_decode_fast": "35/35", |
| "edge_cases_spm_roundtrip": "34/35", |
| "all_texts": 58338, |
| "all_ids_identical_exact": "58338/58338", |
| "all_ids_identical_fast": "58257/58338", |
| "seconds": 99.9 |
| } |
|
|
