| { |
| "what": "Held-out Hindko (test_strict) token-count ranking of the released tokenizer against every external tokenizer measured; compact copy of sota/competitors/competitors_final.json (full per-behaviour table: competitors_test_strict.csv). Screening metrics only; a tokenizer is not a model.", |
| "generated_utc_source": "2026-09-27T13:07:05Z", |
| "dataset": { |
| "name": "test_strict", |
| "path": "data/test_strict.jsonl (study folder; not published)", |
| "sha256": "a74c33ad008f4b0e4a6f7779b0de24e265b0f4edf6f94e89885c7943d3a08468", |
| "docs": 491, |
| "test_docs": 491 |
| }, |
| "method": { |
| "metrics": "bytes/token = UTF-8 bytes / tokens; fertility = harness definition: sum over whitespace words of the number of tokens overlapping the word, divided by words (standalone '▁' and newline tokens belong to no word; the plain ratio tokens / whitespace words is reported separately as tokens_per_word, released 1.161 vs fertility 1.135); every tokenizer scored with the same definition; each tokenizer with its own native encoder, no BOS/EOS/CLS (eval/harness.py)", |
| "lossless": "G1: decode(encode(doc)) == doc for all 491 test documents", |
| "dedup": "behaviour = sha256 of the per-document (token count, round-trip ok) vector over the 491 docs (same key as scripts/screen.py); identical vectors are treated as identical encodings (a strong proxy, not a token-by-token proof)", |
| "family": "vocabulary family = sha256 of vocabulary entries at ids 0-299 and 1000-1099 (heuristic lineage key; fine-tunes and appended vocab extensions stay in the base family)", |
| "metric_check": "all 550 rows re-read from their own summary.json: 0 mismatches with competitors_all.json" |
| }, |
| "released": { |
| "name": "hindko-tokenizer-1.0.0-released", |
| "sha256_tokenizer_json": "49f301c52363a09a1fc1925359af3ddabd7887495de02354cfa4092cba51da41", |
| "vocab_size": 32768, |
| "bytes_per_token": 7.277325843823662, |
| "fertility": 1.1350127205723965, |
| "strr": 0.8908184829625835, |
| "tokens": 199390, |
| "g1": "491/491" |
| }, |
| "verdict": { |
| "any_lossless_external_beats_released_on_bytes_per_token": false, |
| "any_lossless_external_beats_released_on_fertility": false, |
| "lossless_threats_bytes_per_token": [], |
| "lossless_threats_fertility": [], |
| "released_rank_among_lossless_behaviours": 1, |
| "n_lossless_behaviours": 130, |
| "best_lossless_external": { |
| "representative": "ProximaAI__urnova-95m", |
| "aliases": [], |
| "n_repos": 1, |
| "origin": "new", |
| "provider": "ProximaAI", |
| "repo": "ProximaAI/urnova-95m", |
| "vocab_family_label": "ProximaAI/urnova-95m (BPE 50,048)", |
| "model_type": "BPE", |
| "vocab_size": 50048, |
| "bytes_per_token": 6.061296952279107, |
| "fertility": 1.3725468507122938, |
| "strr": 0.7469741338658314, |
| "tokens": 239392, |
| "g1_pass_docs": 491, |
| "unk_share": 0.0, |
| "nonws_chars_lost": 0, |
| "released_uses_fewer_tokens": 0.16709831573319078 |
| }, |
| "best_lossless_external_by_fertility": { |
| "representative": "ProximaAI__urnova-95m", |
| "aliases": [], |
| "n_repos": 1, |
| "origin": "new", |
| "provider": "ProximaAI", |
| "repo": "ProximaAI/urnova-95m", |
| "vocab_family_label": "ProximaAI/urnova-95m (BPE 50,048)", |
| "model_type": "BPE", |
| "vocab_size": 50048, |
| "bytes_per_token": 6.061296952279107, |
| "fertility": 1.3725468507122938, |
| "strr": 0.7469741338658314, |
| "tokens": 239392, |
| "g1_pass_docs": 491, |
| "unk_share": 0.0, |
| "nonws_chars_lost": 0, |
| "released_uses_fewer_tokens": 0.16709831573319078 |
| }, |
| "lossy_with_higher_raw_bytes_per_token": 12, |
| "lossy_with_lower_or_equal_fertility": 57, |
| "best_lossy_with_zero_unk_and_zero_nonspace_loss": { |
| "representative": "mahwizzzz__UrduNER", |
| "aliases": [], |
| "n_repos": 2, |
| "origin": "new", |
| "provider": "mahwizzzz", |
| "repo": "mahwizzzz/UrduNER", |
| "vocab_family_label": "roberta-urdu", |
| "model_type": "BPE", |
| "vocab_size": 52000, |
| "bytes_per_token": 5.709890801770782, |
| "fertility": 1.4582840908429344, |
| "strr": 0.7213874447659356, |
| "tokens": 254125, |
| "g1_pass_docs": 0, |
| "unk_share": 0.0, |
| "nonws_chars_lost": 0, |
| "released_uses_fewer_tokens": 0.21538612887358588 |
| }, |
| "best_lossy_whitespace_only_failures": { |
| "representative": "mahwizzzz__UrduNER", |
| "aliases": [], |
| "n_repos": 2, |
| "origin": "new", |
| "provider": "mahwizzzz", |
| "repo": "mahwizzzz/UrduNER", |
| "vocab_family_label": "roberta-urdu", |
| "model_type": "BPE", |
| "vocab_size": 52000, |
| "bytes_per_token": 5.709890801770782, |
| "fertility": 1.4582840908429344, |
| "strr": 0.7213874447659356, |
| "tokens": 254125, |
| "g1_pass_docs": 0, |
| "unk_share": 0.0, |
| "nonws_chars_lost": 0, |
| "released_uses_fewer_tokens": 0.21538612887358588 |
| }, |
| "scope": "Token counts on the strict Hindko test split only. A tokenizer is not a model: this says nothing about generation, translation or downstream quality (LM bits-per-byte evidence is in analysis/TEST_RESULTS.md). Covers only tokenizers found and measured by 2026-09-27; closed tokenizers (Gemini, Claude 3+, newer OpenAI) are not public." |
| }, |
| "counts": { |
| "harness_rows_external": 550, |
| "harness_rows_new": 485, |
| "harness_rows_study_baselines": 65, |
| "excluded": { |
| "hindko-probe-bpe32k": "trained on the whole Hindko corpus incl. test (leaky probe; listed in study, never ranked)" |
| }, |
| "distinct_encoding_behaviours": 528, |
| "distinct_behaviours_lossless": 130, |
| "distinct_behaviours_lossy": 398, |
| "distinct_per_doc_token_count_profiles": 524, |
| "vocabulary_families": 466, |
| "vocabulary_families_with_a_lossless_member": 102, |
| "repos_represented_by_measured_rows": 2608, |
| "tokenizer_files_screened_ok_in_refresh": 957, |
| "new_files_equal_in_behaviour_to_a_baseline": 17 |
| }, |
| "top15_lossless_behaviours": [ |
| { |
| "representative": "ProximaAI__urnova-95m", |
| "aliases": [], |
| "provider": "ProximaAI", |
| "repo": "ProximaAI/urnova-95m", |
| "model_type": "BPE", |
| "vocab_size": 50048, |
| "bytes_per_token": 6.061296952279107, |
| "fertility": 1.3725468507122938, |
| "strr": 0.7469741338658314, |
| "tokens": 239392, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.16709831573319078, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "DunbaaBERT__DunbaaBERT_52k_base", |
| "aliases": [], |
| "provider": "DunbaaBERT", |
| "repo": "DunbaaBERT/DunbaaBERT_52k_base", |
| "model_type": "BPE", |
| "vocab_size": 52010, |
| "bytes_per_token": 5.964354270540893, |
| "fertility": 1.395176079502122, |
| "strr": 0.7324837427009531, |
| "tokens": 243283, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.1804195114331869, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "orature__ALIF-Base-100M", |
| "aliases": [], |
| "provider": "orature", |
| "repo": "orature/ALIF-Base-100M", |
| "model_type": "BPE", |
| "vocab_size": 32000, |
| "bytes_per_token": 5.911600544297506, |
| "fertility": 1.4047412513317303, |
| "strr": 0.7096507518818879, |
| "tokens": 245454, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.1876685651894041, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "DunbaaBERT__DunbaaBERT_32k_base", |
| "aliases": [], |
| "provider": "DunbaaBERT", |
| "repo": "DunbaaBERT/DunbaaBERT_32k_base", |
| "model_type": "BPE", |
| "vocab_size": 32010, |
| "bytes_per_token": 5.717765737365776, |
| "fertility": 1.456240648778301, |
| "strr": 0.6972969511378654, |
| "tokens": 253775, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.2143040094571963, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "roberta-urdu", |
| "aliases": [], |
| "provider": "UrduHack", |
| "repo": "urduhack/roberta-urdu-small", |
| "model_type": "BPE", |
| "vocab_size": 52000, |
| "bytes_per_token": 5.708947266954404, |
| "fertility": 1.4585286052780186, |
| "strr": 0.7212186133702821, |
| "tokens": 254167, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.21551578293012075, |
| "n_repos": 21 |
| }, |
| { |
| "representative": "ibm-research__ia-multilingual-original-script-roberta", |
| "aliases": [], |
| "provider": "ibm-research", |
| "repo": "ibm-research/ia-multilingual-original-script-roberta", |
| "model_type": "BPE", |
| "vocab_size": 110000, |
| "bytes_per_token": 5.667848912151869, |
| "fertility": 1.4692523097881458, |
| "strr": 0.6933148589093492, |
| "tokens": 256010, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.2211632358111011, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "mahwizzzz__urdu-dummy", |
| "aliases": [], |
| "provider": "mahwizzzz", |
| "repo": "mahwizzzz/urdu-dummy", |
| "model_type": "BPE", |
| "vocab_size": 25000, |
| "bytes_per_token": 5.585832027686137, |
| "fertility": 1.491147995272721, |
| "strr": 0.6707613131589518, |
| "tokens": 259769, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.23243343124083315, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "hadidev__robertaurdu", |
| "aliases": [], |
| "provider": "hadidev", |
| "repo": "hadidev/robertaurdu", |
| "model_type": "BPE", |
| "vocab_size": 59458, |
| "bytes_per_token": 5.523068197822024, |
| "fertility": 1.5011148693885392, |
| "strr": 0.7751573333954322, |
| "tokens": 262721, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.24105800449906933, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "bloom", |
| "aliases": [], |
| "provider": "BigScience", |
| "repo": "bigscience/bloom-560m", |
| "model_type": "BPE", |
| "vocab_size": 250680, |
| "bytes_per_token": 5.121690021531185, |
| "fertility": 1.6331468425618128, |
| "strr": 0.5723733618988293, |
| "tokens": 283310, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.2962126292753521, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "santhosh__madlad400-3b-ct2", |
| "aliases": [], |
| "provider": "santhosh", |
| "repo": "santhosh/madlad400-3b-ct2", |
| "model_type": "Unigram", |
| "vocab_size": 256000, |
| "bytes_per_token": 5.08016076967783, |
| "fertility": 1.5985888024032275, |
| "strr": 0.5621852604369822, |
| "tokens": 285626, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.30191929306155607, |
| "n_repos": 8 |
| }, |
| { |
| "representative": "mlx-community__K2-Horizon-MoVA-36B-A4B-oQ4e", |
| "aliases": [], |
| "provider": "mlx-community", |
| "repo": "mlx-community/K2-Horizon-MoVA-36B-A4B-oQ4e", |
| "model_type": "BPE", |
| "vocab_size": 250624, |
| "bytes_per_token": 5.063867804358827, |
| "fertility": 1.6512234454412613, |
| "strr": 0.5681583987797565, |
| "tokens": 286545, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.3041581601493657, |
| "n_repos": 3 |
| }, |
| { |
| "representative": "MaLA-LM__mala-500-10b-v2", |
| "aliases": [], |
| "provider": "MaLA-LM", |
| "repo": "MaLA-LM/mala-500-10b-v2", |
| "model_type": "BPE", |
| "vocab_size": 260164, |
| "bytes_per_token": 4.912820165563475, |
| "fertility": 1.698111999254813, |
| "strr": 0.5555891924619693, |
| "tokens": 295355, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.32491408643835384, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "gemma-3", |
| "aliases": [ |
| "gemma-4", |
| "sarvam-30b" |
| ], |
| "provider": "Google", |
| "repo": "unsloth/gemma-3-1b-it", |
| "model_type": "BPE", |
| "vocab_size": 262145, |
| "bytes_per_token": 4.61036885467272, |
| "fertility": 1.8071712590746876, |
| "strr": 0.48661865645139696, |
| "tokens": 314731, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.3664748626604942, |
| "n_repos": 3 |
| }, |
| { |
| "representative": "alphaedge-ai__gemma-3-270m-it-snd-32768", |
| "aliases": [], |
| "provider": "alphaedge-ai", |
| "repo": "alphaedge-ai/gemma-3-270m-it-snd-32768", |
| "model_type": "BPE", |
| "vocab_size": 32768, |
| "bytes_per_token": 4.531921206575072, |
| "fertility": 1.8388882743684833, |
| "strr": 0.47016050626131606, |
| "tokens": 320179, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.37725459820912677, |
| "n_repos": 3 |
| }, |
| { |
| "representative": "Palash123__indic_200K_all_tokenizer", |
| "aliases": [], |
| "provider": "Palash123", |
| "repo": "Palash123/indic_200K_all_tokenizer", |
| "model_type": "BPE", |
| "vocab_size": 200000, |
| "bytes_per_token": 4.528470178482818, |
| "fertility": 1.839511204000722, |
| "strr": 0.4848371941386397, |
| "tokens": 320423, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.377728814723038, |
| "n_repos": 1 |
| } |
| ], |
| "study_baselines_deduplicated": [ |
| { |
| "representative": "urdu-bert-64k", |
| "aliases": [], |
| "provider": "community (farahadeeba)", |
| "repo": "farahadeeba/urdu-bert-64k", |
| "model_type": "WordPiece", |
| "vocab_size": 64000, |
| "bytes_per_token": 6.221278018830712, |
| "fertility": 1.3578468757459146, |
| "strr": 0.7355867473176184, |
| "tokens": 233236, |
| "g1_pass_docs": 0, |
| "released_uses_fewer_tokens": 0.14511481932463255, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "roberta-urdu", |
| "aliases": [], |
| "provider": "UrduHack", |
| "repo": "urduhack/roberta-urdu-small", |
| "model_type": "BPE", |
| "vocab_size": 52000, |
| "bytes_per_token": 5.708947266954404, |
| "fertility": 1.4585286052780186, |
| "strr": 0.7212186133702821, |
| "tokens": 254167, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.21551578293012075, |
| "n_repos": 21 |
| }, |
| { |
| "representative": "urdu-gpt2-20k", |
| "aliases": [], |
| "provider": "community (aariciah)", |
| "repo": "aariciah/gpt2-urdu-20k-lc", |
| "model_type": "BPE", |
| "vocab_size": 20000, |
| "bytes_per_token": 5.632624383275559, |
| "fertility": 1.478421601103808, |
| "strr": 0.6588499671069867, |
| "tokens": 257611, |
| "g1_pass_docs": 166, |
| "released_uses_fewer_tokens": 0.22600354798514044, |
| "n_repos": 2 |
| }, |
| { |
| "representative": "muril", |
| "aliases": [], |
| "provider": "Google", |
| "repo": "google/muril-base-cased", |
| "model_type": "WordPiece", |
| "vocab_size": 197258, |
| "bytes_per_token": 5.392003924089674, |
| "fertility": 1.5666796686247226, |
| "strr": 0.615541803235741, |
| "tokens": 269107, |
| "g1_pass_docs": 0, |
| "released_uses_fewer_tokens": 0.25906795438245755, |
| "n_repos": 14 |
| }, |
| { |
| "representative": "indicbert-v2", |
| "aliases": [], |
| "provider": "AI4Bharat", |
| "repo": "ai4bharat/IndicBERTv2-MLM-only", |
| "model_type": "WordPiece", |
| "vocab_size": 250000, |
| "bytes_per_token": 5.298519289404977, |
| "fertility": 1.5943214433337796, |
| "strr": 0.5845699747917261, |
| "tokens": 273855, |
| "g1_pass_docs": 0, |
| "released_uses_fewer_tokens": 0.2719139690712238, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "bloom", |
| "aliases": [], |
| "provider": "BigScience", |
| "repo": "bigscience/bloom-560m", |
| "model_type": "BPE", |
| "vocab_size": 250680, |
| "bytes_per_token": 5.121690021531185, |
| "fertility": 1.6331468425618128, |
| "strr": 0.5723733618988293, |
| "tokens": 283310, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.2962126292753521, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "sindhi-xlmr", |
| "aliases": [], |
| "provider": "community (Kashif786)", |
| "repo": "Kashif786/xlm-roberta-base-sindhi-extended", |
| "model_type": "Unigram", |
| "vocab_size": 264635, |
| "bytes_per_token": 5.06933439539401, |
| "fertility": 1.5985771588586997, |
| "strr": 0.5864853378665533, |
| "tokens": 286236, |
| "g1_pass_docs": 172, |
| "released_uses_fewer_tokens": 0.30340697885660783, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "xlm-r", |
| "aliases": [], |
| "provider": "Meta", |
| "repo": "FacebookAI/xlm-roberta-base", |
| "model_type": "Unigram", |
| "vocab_size": 250002, |
| "bytes_per_token": 4.791568839179867, |
| "fertility": 1.6804254551170468, |
| "strr": 0.5677566964935465, |
| "tokens": 302829, |
| "g1_pass_docs": 173, |
| "released_uses_fewer_tokens": 0.341575608676844, |
| "n_repos": 259 |
| }, |
| { |
| "representative": "nllb-200", |
| "aliases": [], |
| "provider": "Meta", |
| "repo": "facebook/nllb-200-distilled-600M", |
| "model_type": "BPE", |
| "vocab_size": 256204, |
| "bytes_per_token": 4.617175840925073, |
| "fertility": 1.8244037049758688, |
| "strr": 0.4662773841612864, |
| "tokens": 314267, |
| "g1_pass_docs": 150, |
| "released_uses_fewer_tokens": 0.3655394934880213, |
| "n_repos": 4 |
| }, |
| { |
| "representative": "gemma-3", |
| "aliases": [ |
| "gemma-4", |
| "sarvam-30b" |
| ], |
| "provider": "Google", |
| "repo": "unsloth/gemma-3-1b-it", |
| "model_type": "BPE", |
| "vocab_size": 262145, |
| "bytes_per_token": 4.61036885467272, |
| "fertility": 1.8071712590746876, |
| "strr": 0.48661865645139696, |
| "tokens": 314731, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.3664748626604942, |
| "n_repos": 3 |
| }, |
| { |
| "representative": "falcon-h1", |
| "aliases": [], |
| "provider": "TII", |
| "repo": "tiiuae/Falcon-H1-34B-Instruct", |
| "model_type": "BPE", |
| "vocab_size": 261120, |
| "bytes_per_token": 4.506487859720609, |
| "fertility": 1.845618243105566, |
| "strr": 0.4560077778877446, |
| "tokens": 321986, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.38074947357959665, |
| "n_repos": 2 |
| }, |
| { |
| "representative": "gpt-4o", |
| "aliases": [ |
| "gpt-oss", |
| "phi-4-mini" |
| ], |
| "provider": "OpenAI", |
| "repo": "Xenova/gpt-4o", |
| "model_type": "BPE", |
| "vocab_size": 200000, |
| "bytes_per_token": 4.337584149418278, |
| "fertility": 1.9305346133469952, |
| "strr": 0.4149409963381053, |
| "tokens": 334524, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.4039590582439526, |
| "n_repos": 3 |
| }, |
| { |
| "representative": "mbert", |
| "aliases": [], |
| "provider": "Google", |
| "repo": "google-bert/bert-base-multilingual-cased", |
| "model_type": "WordPiece", |
| "vocab_size": 119547, |
| "bytes_per_token": 4.1505201644160055, |
| "fertility": 2.035297405236102, |
| "strr": 0.4189347321111493, |
| "tokens": 349601, |
| "g1_pass_docs": 0, |
| "released_uses_fewer_tokens": 0.42966410279146794, |
| "n_repos": 48 |
| }, |
| { |
| "representative": "tiny-aya", |
| "aliases": [], |
| "provider": "Cohere", |
| "repo": "mlx-community/tiny-aya-global-8bit-mlx", |
| "model_type": "BPE", |
| "vocab_size": 261010, |
| "bytes_per_token": 4.1062279976908185, |
| "fertility": 2.032299192520187, |
| "strr": 0.38365479219183907, |
| "tokens": 353372, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.4357504273117282, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "mt5", |
| "aliases": [], |
| "provider": "Google", |
| "repo": "google/mt5-small", |
| "model_type": "Unigram", |
| "vocab_size": 250100, |
| "bytes_per_token": 3.9124922479575055, |
| "fertility": 1.874470946445517, |
| "strr": 0.413025633263278, |
| "tokens": 370870, |
| "g1_pass_docs": 175, |
| "released_uses_fewer_tokens": 0.46237225982150076, |
| "n_repos": 3 |
| }, |
| { |
| "representative": "apertus", |
| "aliases": [ |
| "mistral-nemo", |
| "mistral-small-4", |
| "nemotron-3", |
| "sarvam-m" |
| ], |
| "provider": "Swiss AI", |
| "repo": "swiss-ai/Apertus-8B-Instruct-2509", |
| "model_type": "BPE", |
| "vocab_size": 131072, |
| "bytes_per_token": 3.901016504418474, |
| "fertility": 2.1491246965401207, |
| "strr": 0.3041293830667932, |
| "tokens": 371961, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.46394917746753017, |
| "n_repos": 14 |
| }, |
| { |
| "representative": "gemma-2", |
| "aliases": [], |
| "provider": "Google", |
| "repo": "unsloth/gemma-2-2b-it", |
| "model_type": "BPE", |
| "vocab_size": 256000, |
| "bytes_per_token": 3.7866022964509396, |
| "fertility": 2.205578422183281, |
| "strr": 0.30369857191926364, |
| "tokens": 383200, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.47967118997912317, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "llama-4", |
| "aliases": [], |
| "provider": "Meta", |
| "repo": "unsloth/Llama-4-Scout-17B-16E-Instruct", |
| "model_type": "BPE", |
| "vocab_size": 201135, |
| "bytes_per_token": 3.7268769536216326, |
| "fertility": 2.250545791149742, |
| "strr": 0.2749390169355355, |
| "tokens": 389341, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.4878782352744766, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "command-a-plus", |
| "aliases": [], |
| "provider": "Cohere", |
| "repo": "CohereLabs/command-a-plus-05-2026-bf16", |
| "model_type": "BPE", |
| "vocab_size": 255032, |
| "bytes_per_token": 3.6850237325497823, |
| "fertility": 2.2738736326112394, |
| "strr": 0.26122292148175746, |
| "tokens": 393763, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.49362941667957627, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "qwen-3.5", |
| "aliases": [ |
| "qwen-3.8" |
| ], |
| "provider": "Alibaba Qwen", |
| "repo": "Qwen/Qwen3.5-0.8B", |
| "model_type": "BPE", |
| "vocab_size": 248070, |
| "bytes_per_token": 3.5982036581495, |
| "fertility": 2.3226833712718826, |
| "strr": 0.25747952191606166, |
| "tokens": 403264, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.5055596333915251, |
| "n_repos": 88 |
| }, |
| { |
| "representative": "kimi-k2", |
| "aliases": [], |
| "provider": "Moonshot AI", |
| "repo": "moonshotai/Kimi-K2-Instruct", |
| "model_type": "BPE", |
| "vocab_size": 163840, |
| "bytes_per_token": 3.529626683596895, |
| "fertility": 2.375288905448597, |
| "strr": 0.2777975071171166, |
| "tokens": 411099, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.5149830089589126, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "ernie-4.5", |
| "aliases": [], |
| "provider": "Baidu", |
| "repo": "baidu/ERNIE-4.5-0.3B-PT", |
| "model_type": "BPE", |
| "vocab_size": 101304, |
| "bytes_per_token": 3.5275244077949357, |
| "fertility": 2.368884955958293, |
| "strr": 0.2724822290401644, |
| "tokens": 411344, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.5152718892216733, |
| "n_repos": 10 |
| }, |
| { |
| "representative": "pashto-lfm2.5", |
| "aliases": [], |
| "provider": "community (nassimjp)", |
| "repo": "nassimjp/LFM2.5-2.6B-Pashto-Zi", |
| "model_type": "BPE", |
| "vocab_size": 125039, |
| "bytes_per_token": 3.5092506154984693, |
| "fertility": 2.386105758314946, |
| "strr": 0.22505807217833254, |
| "tokens": 413486, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.5177829479111747, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "deepseek-r1", |
| "aliases": [ |
| "deepseek-v3", |
| "deepseek-v4", |
| "deepseek-v4.1" |
| ], |
| "provider": "DeepSeek", |
| "repo": "deepseek-ai/DeepSeek-R1", |
| "model_type": "BPE", |
| "vocab_size": 128815, |
| "bytes_per_token": 3.372220466292344, |
| "fertility": 2.4755281802886433, |
| "strr": 0.2324517229535015, |
| "tokens": 430288, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.5366126873164021, |
| "n_repos": 4 |
| }, |
| { |
| "representative": "glm-5", |
| "aliases": [], |
| "provider": "Zhipu AI / Z.ai", |
| "repo": "zai-org/GLM-5", |
| "model_type": "BPE", |
| "vocab_size": 154856, |
| "bytes_per_token": 3.0253785304137875, |
| "fertility": 2.762588127077645, |
| "strr": 0.1365613119945974, |
| "tokens": 479618, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.5842733175151892, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "command-r", |
| "aliases": [ |
| "command-r7b" |
| ], |
| "provider": "Cohere", |
| "repo": "adamo1139/aya-expanse-8b-ungated", |
| "model_type": "BPE", |
| "vocab_size": 255029, |
| "bytes_per_token": 2.645120833006725, |
| "fertility": 3.1073243716852286, |
| "strr": 0.21701238290960534, |
| "tokens": 548567, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.6365257115356922, |
| "n_repos": 2 |
| }, |
| { |
| "representative": "alif-1.0", |
| "aliases": [ |
| "llama-3", |
| "qalb-1.0", |
| "urdu-llama3-almanach", |
| "urdu-llama3.2-custom" |
| ], |
| "provider": "Traversaal.ai", |
| "repo": "large-traversaal/Alif-1.0-8B-Instruct", |
| "model_type": "BPE", |
| "vocab_size": 128256, |
| "bytes_per_token": 2.64122087382617, |
| "fertility": 3.1188980549458867, |
| "strr": 0.17801233051365498, |
| "tokens": 549377, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.6370616170680607, |
| "n_repos": 5 |
| }, |
| { |
| "representative": "qwen-2.5", |
| "aliases": [ |
| "qwen-3" |
| ], |
| "provider": "Alibaba Qwen", |
| "repo": "Qwen/Qwen2.5-0.5B", |
| "model_type": "BPE", |
| "vocab_size": 151665, |
| "bytes_per_token": 2.6325857842914915, |
| "fertility": 3.0821976025941815, |
| "strr": 0.02994719652556631, |
| "tokens": 551179, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.6382481916038165, |
| "n_repos": 60 |
| }, |
| { |
| "representative": "glm-4.5", |
| "aliases": [], |
| "provider": "Zhipu AI / Z.ai", |
| "repo": "zai-org/GLM-4.5", |
| "model_type": "BPE", |
| "vocab_size": 151365, |
| "bytes_per_token": 2.4823892097555635, |
| "fertility": 3.3733502552847137, |
| "strr": 0.13585105577840006, |
| "tokens": 584528, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.6588871705033805, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "urdu-llama-bilal", |
| "aliases": [], |
| "provider": "community (BilalKhan1)", |
| "repo": "BilalKhan1/llama-urdu-tokenizer", |
| "model_type": "BPE", |
| "vocab_size": 129435, |
| "bytes_per_token": 2.3721203204185057, |
| "fertility": 2.833980520350005, |
| "strr": 0.26064656602763014, |
| "tokens": 611700, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.674039561876737, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "minimax-m2", |
| "aliases": [ |
| "minimax-m3" |
| ], |
| "provider": "MiniMax", |
| "repo": "MiniMaxAI/MiniMax-M2", |
| "model_type": "BPE", |
| "vocab_size": 200054, |
| "bytes_per_token": 2.26764267842715, |
| "fertility": 3.604003050608666, |
| "strr": 0.06987291071147879, |
| "tokens": 639883, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.688396159922986, |
| "n_repos": 2 |
| }, |
| { |
| "representative": "eurollm", |
| "aliases": [], |
| "provider": "UTTER (EU)", |
| "repo": "utter-project/EuroLLM-9B-Instruct-2512", |
| "model_type": "BPE", |
| "vocab_size": 128000, |
| "bytes_per_token": 2.146942326557245, |
| "fertility": 3.6777649051924386, |
| "strr": 0.061378944978430335, |
| "tokens": 675857, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.7049819710382521, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "hunyuan", |
| "aliases": [], |
| "provider": "Tencent", |
| "repo": "tencent/Hunyuan-7B-Instruct", |
| "model_type": "BPE", |
| "vocab_size": 128166, |
| "bytes_per_token": 1.932331007297715, |
| "fertility": 4.245044216360345, |
| "strr": 0.00577519808580128, |
| "tokens": 750920, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.7344723805465296, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "gpt-4", |
| "aliases": [ |
| "olmo-2", |
| "phi-4" |
| ], |
| "provider": "OpenAI", |
| "repo": "Xenova/gpt-4", |
| "model_type": "BPE", |
| "vocab_size": 100263, |
| "bytes_per_token": 1.9192881698043578, |
| "fertility": 4.27475272022309, |
| "strr": 0.007050166211598135, |
| "tokens": 756023, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.736264637451506, |
| "n_repos": 20 |
| }, |
| { |
| "representative": "granite-4.2", |
| "aliases": [], |
| "provider": "IBM", |
| "repo": "ibm-granite/granite-4.2-8b", |
| "model_type": "BPE", |
| "vocab_size": 100352, |
| "bytes_per_token": 1.918772959406315, |
| "fertility": 4.274776007312146, |
| "strr": 0.00706180975612596, |
| "tokens": 756226, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.7363354341162562, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "lfm2", |
| "aliases": [], |
| "provider": "Liquid AI", |
| "repo": "LiquidAI/LFM2.5-1.2B-Instruct", |
| "model_type": "BPE", |
| "vocab_size": 64402, |
| "bytes_per_token": 1.9002286519487799, |
| "fertility": 4.181191018169751, |
| "strr": 0.03274164721224435, |
| "tokens": 763606, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.738883665136209, |
| "n_repos": 63 |
| }, |
| { |
| "representative": "falcon-3", |
| "aliases": [], |
| "provider": "TII", |
| "repo": "tiiuae/Falcon3-7B-Instruct", |
| "model_type": "BPE", |
| "vocab_size": 131072, |
| "bytes_per_token": 1.8130482154727336, |
| "fertility": 4.50383945880805, |
| "strr": 0.010793565777293923, |
| "tokens": 800324, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.7508634003228692, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "llama-2", |
| "aliases": [ |
| "urdu-llama2-almanach" |
| ], |
| "provider": "Meta", |
| "repo": "NousResearch/Llama-2-7b-hf", |
| "model_type": "BPE", |
| "vocab_size": 32000, |
| "bytes_per_token": 1.713758958393371, |
| "fertility": 3.937066641827105, |
| "strr": 0.006473810757470789, |
| "tokens": 846692, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.7645070462458603, |
| "n_repos": 99 |
| }, |
| { |
| "representative": "grok-1", |
| "aliases": [], |
| "provider": "xAI", |
| "repo": "Xenova/grok-1-tokenizer", |
| "model_type": "BPE", |
| "vocab_size": 131072, |
| "bytes_per_token": 1.618097837965627, |
| "fertility": 4.822173966198791, |
| "strr": 0.0056121884624117276, |
| "tokens": 896748, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.7776521386164229, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "claude-legacy", |
| "aliases": [], |
| "provider": "Anthropic", |
| "repo": "Xenova/claude-tokenizer", |
| "model_type": "BPE", |
| "vocab_size": 65000, |
| "bytes_per_token": 1.5541900255135377, |
| "fertility": 4.944966786789235, |
| "strr": 0.00631662290634515, |
| "tokens": 933622, |
| "g1_pass_docs": 467, |
| "released_uses_fewer_tokens": 0.7864339100835241, |
| "n_repos": 1 |
| }, |
| { |
| "representative": "gpt-2", |
| "aliases": [], |
| "provider": "OpenAI", |
| "repo": "openai-community/gpt2", |
| "model_type": "BPE", |
| "vocab_size": 50257, |
| "bytes_per_token": 1.3846658873488085, |
| "fertility": 5.7485751212384075, |
| "strr": 0.005128981364506983, |
| "tokens": 1047925, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.8097287496719708, |
| "n_repos": 122 |
| }, |
| { |
| "representative": "sarvam-1", |
| "aliases": [], |
| "provider": "Sarvam AI", |
| "repo": "sarvamai/sarvam-1", |
| "model_type": "BPE", |
| "vocab_size": 68096, |
| "bytes_per_token": 1.0054379986391144, |
| "fertility": 7.40664497086203, |
| "strr": 0.001304076987116418, |
| "tokens": 1443178, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.8618396344733636, |
| "n_repos": 2 |
| }, |
| { |
| "representative": "byt5", |
| "aliases": [], |
| "provider": "Google", |
| "repo": "google/byt5-small", |
| "model_type": "bytes", |
| "vocab_size": 384, |
| "bytes_per_token": 1.0, |
| "fertility": 7.45030244106911, |
| "strr": 0.000873265839586887, |
| "tokens": 1451026, |
| "g1_pass_docs": 491, |
| "released_uses_fewer_tokens": 0.8625868867959636, |
| "n_repos": 1 |
| } |
| ], |
| "search_coverage": { |
| "ranking_snapshot_utc": "2026-09-27T12:49:42Z", |
| "pass1": { |
| "file": "_search.json", |
| "generated_utc": "2026-09-27T08:17:53Z", |
| "n_keywords": 43, |
| "keywords": [ |
| "hindko", |
| "hindku", |
| "hinko", |
| "hazarewal", |
| "hazara", |
| "potohari", |
| "pothwari", |
| "pothohari", |
| "pahari", |
| "lahnda", |
| "lahanda", |
| "saraiki", |
| "seraiki", |
| "siraiki", |
| "multani", |
| "shahmukhi", |
| "punjabi", |
| "panjabi", |
| "pnb", |
| "urdu", |
| "pashto", |
| "pushto", |
| "pukhto", |
| "sindhi", |
| "kashmiri", |
| "balochi", |
| "baluchi", |
| "brahui", |
| "khowar", |
| "shina", |
| "gojri", |
| "pakistan", |
| "peshawar", |
| "nastaliq", |
| "nastaleeq", |
| "perso-arabic", |
| "persoarabic", |
| "indic", |
| "indo-aryan", |
| "indoaryan", |
| "south-asian", |
| "southasian", |
| "bharat" |
| ], |
| "n_lang_tags": 27, |
| "lang_tags": [ |
| "hno", |
| "hnd", |
| "ur", |
| "urd", |
| "pnb", |
| "skr", |
| "lah", |
| "phr", |
| "ps", |
| "pus", |
| "pbt", |
| "pbu", |
| "pst", |
| "sd", |
| "snd", |
| "ks", |
| "kas", |
| "bal", |
| "bcc", |
| "bgp", |
| "khw", |
| "scl", |
| "gju", |
| "pa", |
| "pan", |
| "trw", |
| "bft" |
| ], |
| "n_authors": 91, |
| "n_named_repos": 216, |
| "n_named_missing": 15, |
| "n_queries": 198, |
| "n_repos": 10995, |
| "n_with_tokenizer_files": 7309 |
| }, |
| "pass2": { |
| "file": "_search2.json", |
| "generated_utc": "2026-09-27T08:46:42Z", |
| "n_keywords": 34, |
| "keywords": [ |
| "markhor", |
| "pakmosaic", |
| "urdulm", |
| "alif-base", |
| "qehwa", |
| "zabaan", |
| "shaheen", |
| "baat", |
| "urdu-gpt", |
| "urdugpt", |
| "muse-glimmer", |
| "Nemotron-3.5", |
| "Atria-Dawn", |
| "Dots3", |
| "Ornith", |
| "Hy-MT2", |
| "Nex-N2.5", |
| "Bonsai-2", |
| "Ling-3.0", |
| "MiMo-V2.6", |
| "MiniCPM5", |
| "grok-2", |
| "grok-3", |
| "gpt-oss", |
| "o200k", |
| "BrahmicTokenizer", |
| "tokenizer-urdu", |
| "urdu-tokenizer", |
| "urdu_tokenizer", |
| "UrduTokenizer", |
| "punjabi-tokenizer", |
| "pashto-tokenizer", |
| "sindhi-tokenizer", |
| "saraiki-tokenizer" |
| ], |
| "n_authors": 33, |
| "authors": [ |
| "ma1993", |
| "mahwizzzz", |
| "themohal", |
| "ProximaAI", |
| "orature", |
| "adiled", |
| "hadidev", |
| "NeerjaK", |
| "aariciah", |
| "Kashif786", |
| "aakashMeghwar01", |
| "zirak-ai", |
| "tasal9", |
| "junaid008", |
| "meta-models", |
| "DeepReinforce", |
| "nex-agi", |
| "PrismML", |
| "dots-studio", |
| "rednote-hilab", |
| "OpenGVLab", |
| "theschoolofai", |
| "bisma10", |
| "Xhaheen", |
| "mirfan899", |
| "Mavkif", |
| "traversaal-ai", |
| "large-traversaal", |
| "enstazao", |
| "UrduHack", |
| "salmanpopa", |
| "SLPG", |
| "abdulwaheed1" |
| ], |
| "named": [ |
| "ProximaAI/PakMosaic", |
| "hadidev/gpt2-urdu-tokenizer-withgpt2", |
| "NeerjaK/Urdu_Model", |
| "meta-models/Muse-Glimmer-30B", |
| "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", |
| "internlm/Atria-Dawn-Preview", |
| "tencent/Hy-MT2-30B-A3B", |
| "XiaomiMiMo/MiMo-V2.6-Flash", |
| "openbmb/MiniCPM5-2B", |
| "alvarobartt/grok-2-tokenizer" |
| ], |
| "n_queries": 67, |
| "n_repos": 3861, |
| "n_kept_after_filter": 634, |
| "filter": "regional keywords, regional/frontier authors, named repos, official orgs of the 2026 releases, and the 3 most-downloaded derivative repos per frontier keyword (the other derivatives are quantisations / fine-tunes)" |
| }, |
| "pass3": { |
| "file": "_search3.json", |
| "generated_utc": "2026-09-27T12:54:36Z", |
| "authors": [ |
| "DunbaaBERT", |
| "eshaaftab900", |
| "Humair332", |
| "hafeez007", |
| "shah-bakhsh", |
| "ijazulhaq", |
| "Hammad712", |
| "alphaedge-ai", |
| "Omarrran", |
| "HPLT", |
| "orature", |
| "ProximaAI", |
| "goldfish-models", |
| "zuhri025", |
| "SabahNawab", |
| "BilalKhan1", |
| "farahadeeba", |
| "urduhack", |
| "Imran1", |
| "Khurram123", |
| "shaikhsalman", |
| "mfayazkhan", |
| "large-traversaal", |
| "enstazao", |
| "nassimjp", |
| "tasal9", |
| "zirak-ai", |
| "APARSIN", |
| "Abdullah104", |
| "aakashMeghwar01", |
| "themohal", |
| "mahwizzzz" |
| ], |
| "n_queries": 32, |
| "n_repos": 2789, |
| "n_with_tokenizer_files": 2564, |
| "n_kept_after_filter": 508, |
| "filter": "alphaedge-ai / goldfish-models / HPLT: only repos whose name carries a Perso-Arabic-script language code (urd, snd, pnb, pbt, pus, skr, hno, hnd, kas, bal, bcc, ur, sd, ps, arab); all repos of the other authors", |
| "status": "STOPPED: started after the ranking snapshot; the probe log (logs/probe_pass3.log) ends with 458 repos still to probe; nothing from this pass was screened or measured" |
| }, |
| "sweep_totals_at_snapshot": { |
| "n_repos_found": 14856, |
| "n_repos_found_pass1": 10995, |
| "n_repos_found_pass2": 3861, |
| "n_repos_kept_pass2": 634, |
| "pass2_filter": "regional keywords, regional/frontier authors, named repos, official orgs of the 2026 releases, and the 3 most-downloaded derivative repos per frontier keyword (the other derivatives are quantisations / fine-tunes)", |
| "n_repos_with_tokenizer_files": 7760, |
| "n_queries": 265, |
| "n_probed": 6895, |
| "skipped": { |
| "gated": 302, |
| "non_text_pipeline": 563 |
| }, |
| "n_probe_errors": 0, |
| "n_repos_all_files_already_known": 1765, |
| "n_new_file_signatures": 1759, |
| "n_downloaded": 880, |
| "n_reconstructed": 95, |
| "n_download_errors": 0, |
| "n_screened_ok": 957, |
| "n_screened": 970, |
| "n_behaviour_groups": 502, |
| "n_groups_equal_to_a_baseline": 17, |
| "n_new_behaviours_harnessed": 485 |
| }, |
| "download_log": { |
| "entries": 1026, |
| "after_snapshot": 47, |
| "after_snapshot_repos": [ |
| "Ayshhaa/orpheus-urdu-tts", |
| "ByteDance-Seed/Seed-OSS-36B-Instruct", |
| "Jarbas/m2v-256-multilingual-e5-small", |
| "JuliaKreutzerCohere/tiny-aya-global-prompt-tasktype", |
| "Kashif786/distilbert-base-multilingual-cased-sindhi-extended", |
| "Kashif786/rembert-sindhi-extended", |
| "Kashif786/sindhi_distilbert_colab", |
| "KnutJaegersberg/nllb-moe-54b-4bit", |
| "LingoIITGN/Ansh-160k", |
| "MISHANM/Kashmiri_text_generation_Llama3_8B_instruct", |
| "MoritzLaurer/xlm-v-base-mnli-xnli", |
| "MoritzLaurer/xlm-v-base-mnli-xnli", |
| "QCRI/TechniqueRAG-FS-Ministral-8B", |
| "Qrzysztof/functiongemma-270m-it-prepaid-cards-v2", |
| "Qrzysztof/functiongemma-270m-it-prepaid-cards-v2-mlx", |
| "SalehAhmad/llama3.1-urdu-8b-qlora_split0", |
| "ServiceNow-AI/Apriel-5B-Instruct", |
| "Shinapri/pratchya-3k", |
| "Soupis/Contra-200", |
| "aiana94/NaSE", |
| "bharatgenai/Param2-17B-A2.4B-Thinking", |
| "casawolice/small100-onnx", |
| "facebook/mgenre-wiki", |
| "facebook/xglm-564M", |
| "facebook/xlm-v-base", |
| "google/madlad400-3b-mt", |
| "google/rembert", |
| "google/umt5-xxl", |
| "huggingworld/m2m100_418M", |
| "ibm-granite/granite-embedding-97m-multilingual-r2", |
| "inclusionAI/Ling-mini-2.0", |
| "inclusionAI/Ring-mini-2.0", |
| "minishlab/M2V_multilingual_output", |
| "minishlab/potion-multilingual-128M", |
| "mistralai/Ministral-3-14B-Reasoning-2512", |
| "nvidia/NVIDIA-Nemotron-Nano-9B-v2", |
| "nvidia/Nemotron-Mini-4B-Instruct", |
| "panagoa/nllb-200-1.3b-kbd-pretrain", |
| "panagoa/nllb-200-1.3b-kbd-v0.1", |
| "panagoa/nllb-200-1.3b-kbd-v0.2", |
| "rahular/varta-t5", |
| "setu4993/LEALLA-base", |
| "shah-bakhsh/balochi-english-translator", |
| "tangledgroup/tangled-alpha-0.5-core", |
| "tiiuae/Falcon-H1-7B-Base", |
| "tiiuae/Falcon-H1-7B-Instruct", |
| "tiiuae/Falcon-H1R-7B" |
| ], |
| "errors": [ |
| { |
| "repo": "MoritzLaurer/xlm-v-base-mnli-xnli", |
| "error": "RemoteProtocolError: peer closed connection without sending complete message body (received 25494361 bytes, expected 61333645)" |
| } |
| ], |
| "note": "downloads logged after the snapshot were not screened or measured" |
| }, |
| "probe_cache_lines": 7050, |
| "github_and_web": "GitHub search API + web search (refresh, before the snapshot): no Hindko tokenizer or Hindko LM found; Urdu tokenizer projects on GitHub publish code only and were not run (no third-party code).", |
| "not_collected": [ |
| "gated repos (not logged into): 302 skipped at search, 65 unmatched after metadata check", |
| "closed tokenizers: Gemini, Claude 3+, OpenAI encodings newer than o200k_base / o200k_harmony", |
| "tiktoken rank files whose split regex exists only in repo Python code; xAI tokenizer.tok.json (Grok 2)", |
| "392 large files classified by range reads as identical / expected-identical to a measured tokenizer (not measured themselves)", |
| "13 downloaded tokenizers that could not be loaded" |
| ] |
| } |
| } |
|
|
