{ "transformers": "5.18.0", "tokenizers": "0.23.2", "sentencepiece": "0.2.2", "autotokenizer_trust_remote_code": "MumeTokenizer", "autotokenizer_default": "TokenizersBackend", "sets": { "fineweb_val": { "group": "english", "records": 14983, "words": 7770317, "utf8_bytes": 46515251, "tokens": 11831667, "tokens_per_word": 1.5227, "bytes_per_token": 3.9314, "ids_identical_exact": 14983, "ids_identical_fast": 14959, "roundtrip_exact": 14983, "roundtrip_spm": 14983, "roundtrip_fast": 14983, "fast_differs_at": [ 210, 492, 699, 1298, 3648 ], "gpt2_tokens": 10470777, "gpt2_tokens_per_word": 1.3475 }, "fineweb_val_clean": { "group": "english", "records": 11048, "words": 4782318, "utf8_bytes": 28445808, "tokens": 7177975, "tokens_per_word": 1.5009, "bytes_per_token": 3.9629, "ids_identical_exact": 11048, "ids_identical_fast": 11041, "roundtrip_exact": 11048, "roundtrip_spm": 11048, "roundtrip_fast": 11048, "fast_differs_at": [ 155, 2866, 3549, 4095, 7837 ], "gpt2_tokens": 6392995, "gpt2_tokens_per_word": 1.3368 }, "wikitext103_test": { "group": "english", "records": 62, "words": 241211, "utf8_bytes": 1287656, "tokens": 363932, "tokens_per_word": 1.5088, "bytes_per_token": 3.5382, "ids_identical_exact": 62, "ids_identical_fast": 62, "roundtrip_exact": 62, "roundtrip_spm": 62, "roundtrip_fast": 62, "fast_differs_at": [], "gpt2_tokens": 283287, "gpt2_tokens_per_word": 1.1744 }, "enwik8_test": { "group": "english", "records": 1, "words": 648280, "utf8_bytes": 5000000, "tokens": 1717932, "tokens_per_word": 2.65, "bytes_per_token": 2.9105, "ids_identical_exact": 1, "ids_identical_fast": 0, "roundtrip_exact": 1, "roundtrip_spm": 1, "roundtrip_fast": 1, "fast_differs_at": [ 0 ], "gpt2_tokens": 1480280, "gpt2_tokens_per_word": 2.2834 }, "text8_test": { "group": "english", "records": 1, "words": 855233, "utf8_bytes": 5000000, "tokens": 1145389, "tokens_per_word": 1.3393, "bytes_per_token": 4.3653, "ids_identical_exact": 1, "ids_identical_fast": 0, "roundtrip_exact": 1, "roundtrip_spm": 1, "roundtrip_fast": 1, "fast_differs_at": [ 0 ], "gpt2_tokens": 1003900, "gpt2_tokens_per_word": 1.1738 }, "owm_val": { "group": "math", "records": 2704, "words": 3157209, "utf8_bytes": 20001168, "tokens": 6124528, "tokens_per_word": 1.9399, "bytes_per_token": 3.2657, "ids_identical_exact": 2704, "ids_identical_fast": 2670, "roundtrip_exact": 2703, "roundtrip_spm": 2703, "roundtrip_fast": 2703, "fast_differs_at": [ 13, 81, 82, 297, 431 ], "gpt2_tokens": 5540067, "gpt2_tokens_per_word": 1.7547 }, "owm_val_clean": { "group": "math", "records": 1126, "words": 768667, "utf8_bytes": 4780645, "tokens": 1436589, "tokens_per_word": 1.8689, "bytes_per_token": 3.3278, "ids_identical_exact": 1126, "ids_identical_fast": 1119, "roundtrip_exact": 1126, "roundtrip_spm": 1126, "roundtrip_fast": 1126, "fast_differs_at": [ 7, 36, 235, 313, 547 ], "gpt2_tokens": 1307204, "gpt2_tokens_per_word": 1.7006 }, "gsm8k_test": { "group": "math", "records": 1319, "words": 130627, "utf8_bytes": 652230, "tokens": 224032, "tokens_per_word": 1.7151, "bytes_per_token": 2.9113, "ids_identical_exact": 1319, "ids_identical_fast": 1319, "roundtrip_exact": 1319, "roundtrip_spm": 1319, "roundtrip_fast": 1319, "fast_differs_at": [], "gpt2_tokens": 173987, "gpt2_tokens_per_word": 1.3319 }, "math_test": { "group": "math", "records": 5000, "words": 577396, "utf8_bytes": 3565468, "tokens": 1531141, "tokens_per_word": 2.6518, "bytes_per_token": 2.3286, "ids_identical_exact": 5000, "ids_identical_fast": 4997, "roundtrip_exact": 5000, "roundtrip_spm": 5000, "roundtrip_fast": 5000, "fast_differs_at": [ 180, 510, 1083 ], "gpt2_tokens": 1457380, "gpt2_tokens_per_word": 2.5241 }, "math_test_clean": { "group": "math", "records": 3754, "words": 393713, "utf8_bytes": 2418527, "tokens": 1014818, "tokens_per_word": 2.5776, "bytes_per_token": 2.3832, "ids_identical_exact": 3754, "ids_identical_fast": 3751, "roundtrip_exact": 3754, "roundtrip_spm": 3754, "roundtrip_fast": 3754, "fast_differs_at": [ 140, 393, 838 ], "gpt2_tokens": 958028, "gpt2_tokens_per_word": 2.4333 }, "sanskrit_dcs_gold_slp1": { "group": "sanskrit", "records": 3000, "words": 16596, "utf8_bytes": 139276, "tokens": 39143, "tokens_per_word": 2.3586, "bytes_per_token": 3.5581, "ids_identical_exact": 3000, "ids_identical_fast": 3000, "roundtrip_exact": 3000, "roundtrip_spm": 3000, "roundtrip_fast": 3000, "fast_differs_at": [] }, "sanskrit_dcs_gold_devanagari": { "group": "sanskrit", "records": 3000, "words": 16596, "utf8_bytes": 370794, "tokens": 164598, "tokens_per_word": 9.9179, "bytes_per_token": 2.2527, "ids_identical_exact": 3000, "ids_identical_fast": 3000, "roundtrip_exact": 3000, "roundtrip_spm": 3000, "roundtrip_fast": 3000, "fast_differs_at": [] }, "sanskrit_gita_slp1": { "group": "sanskrit", "records": 700, "words": 7814, "utf8_bytes": 67003, "tokens": 18460, "tokens_per_word": 2.3624, "bytes_per_token": 3.6296, "ids_identical_exact": 700, "ids_identical_fast": 700, "roundtrip_exact": 700, "roundtrip_spm": 700, "roundtrip_fast": 700, "fast_differs_at": [] }, "sanskrit_gita_devanagari": { "group": "sanskrit", "records": 700, "words": 7814, "utf8_bytes": 166845, "tokens": 72265, "tokens_per_word": 9.2481, "bytes_per_token": 2.3088, "ids_identical_exact": 700, "ids_identical_fast": 700, "roundtrip_exact": 700, "roundtrip_spm": 700, "roundtrip_fast": 700, "fast_differs_at": [] }, "sanskrit_ood_slp1": { "group": "sanskrit", "records": 2000, "words": 132896, "utf8_bytes": 1220801, "tokens": 347994, "tokens_per_word": 2.6185, "bytes_per_token": 3.5081, "ids_identical_exact": 2000, "ids_identical_fast": 2000, "roundtrip_exact": 2000, "roundtrip_spm": 2000, "roundtrip_fast": 2000, "fast_differs_at": [] }, "sanskrit_ood_devanagari": { "group": "sanskrit", "records": 2000, "words": 132896, "utf8_bytes": 2921785, "tokens": 1288809, "tokens_per_word": 9.6979, "bytes_per_token": 2.267, "ids_identical_exact": 2000, "ids_identical_fast": 2000, "roundtrip_exact": 2000, "roundtrip_spm": 2000, "roundtrip_fast": 2000, "fast_differs_at": [] }, "sanskrit_prose_slp1": { "group": "sanskrit", "records": 2470, "words": 55243, "utf8_bytes": 420458, "tokens": 118897, "tokens_per_word": 2.1523, "bytes_per_token": 3.5363, "ids_identical_exact": 2470, "ids_identical_fast": 2469, "roundtrip_exact": 2470, "roundtrip_spm": 2470, "roundtrip_fast": 2470, "fast_differs_at": [ 1495 ] }, "sanskrit_prose_devanagari": { "group": "sanskrit", "records": 2470, "words": 55243, "utf8_bytes": 1014824, "tokens": 464012, "tokens_per_word": 8.3995, "bytes_per_token": 2.1871, "ids_identical_exact": 2470, "ids_identical_fast": 2470, "roundtrip_exact": 2470, "roundtrip_spm": 2470, "roundtrip_fast": 2470, "fast_differs_at": [] }, "sanskrit_vedic_slp1": { "group": "sanskrit", "records": 1000, "words": 4079, "utf8_bytes": 29125, "tokens": 13336, "tokens_per_word": 3.2694, "bytes_per_token": 2.1839, "ids_identical_exact": 1000, "ids_identical_fast": 1000, "roundtrip_exact": 1000, "roundtrip_spm": 1000, "roundtrip_fast": 1000, "fast_differs_at": [] }, "sanskrit_vedic_devanagari": { "group": "sanskrit", "records": 1000, "words": 4079, "utf8_bytes": 77606, "tokens": 41505, "tokens_per_word": 10.1753, "bytes_per_token": 1.8698, "ids_identical_exact": 1000, "ids_identical_fast": 1000, "roundtrip_exact": 1000, "roundtrip_spm": 1000, "roundtrip_fast": 1000, "fast_differs_at": [] } }, "edge_cases": [ { "text": "", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": " ", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": " ", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "a", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": " a", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "a ", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": " two leading spaces", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "trailing spaces ", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "tab\tseparated\tvalues", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "line one\nline two\n\nafter a blank line\n", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "\n", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "\r\n windows line ends \r\n", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "2026", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "x2y", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "3.14159", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "1,234,567.89 and -42", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "$12 for 3 x 4 = 12", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "Let $f(x) = x^2 + 1$. Then \\frac{a}{b} \\geq \\sqrt{2}", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "\\begin{align} a &= b \\\\ c &= d \\end{align}", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": " literal special-token text", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "<0x41>", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "▁ already has a meta-symbol", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": false }, { "text": "Ünïcödé café naïve façade", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "日本語のテキスト", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "emoji 🙂👍🏽", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "Ελληνικά and русский", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "संस्कृतं नाम दैवी वाक् । ॥ १२३ ॥", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "saMskftaM nAma dEvI vAk", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "zero​width", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "combining é vs é", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": " no-break space", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "word word word word word word word word word word word word ", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "012345678901234567890123456789012345678901234567890123456789", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true }, { "text": "''' \"\"\" ``` ~~~", "ids_equal_exact": true, "ids_equal_fast": true, "decode_equal_spm_decode_exact": true, "decode_equal_spm_decode_fast": true, "spm_roundtrip": true } ], "edge_cases_ids_equal_exact": "35/35", "edge_cases_ids_equal_fast": "35/35", "edge_cases_decode_equal_spm_decode_exact": "35/35", "edge_cases_decode_equal_spm_decode_fast": "35/35", "edge_cases_spm_roundtrip": "34/35", "all_texts": 58338, "all_ids_identical_exact": "58338/58338", "all_ids_identical_fast": "58257/58338", "seconds": 99.9 }