{
 "what": "Held-out Hindko (test_strict) token-count ranking of the released tokenizer against every external tokenizer measured; compact copy of sota/competitors/competitors_final.json (full per-behaviour table: competitors_test_strict.csv). Screening metrics only; a tokenizer is not a model.",
 "generated_utc_source": "2026-09-27T13:07:05Z",
 "dataset": {
  "name": "test_strict",
  "path": "data/test_strict.jsonl (study folder; not published)",
  "sha256": "a74c33ad008f4b0e4a6f7779b0de24e265b0f4edf6f94e89885c7943d3a08468",
  "docs": 491,
  "test_docs": 491
 },
 "method": {
  "metrics": "bytes/token = UTF-8 bytes / tokens; fertility = harness definition: sum over whitespace words of the number of tokens overlapping the word, divided by words (standalone '▁' and newline tokens belong to no word; the plain ratio tokens / whitespace words is reported separately as tokens_per_word, released 1.161 vs fertility 1.135); every tokenizer scored with the same definition; each tokenizer with its own native encoder, no BOS/EOS/CLS (eval/harness.py)",
  "lossless": "G1: decode(encode(doc)) == doc for all 491 test documents",
  "dedup": "behaviour = sha256 of the per-document (token count, round-trip ok) vector over the 491 docs (same key as scripts/screen.py); identical vectors are treated as identical encodings (a strong proxy, not a token-by-token proof)",
  "family": "vocabulary family = sha256 of vocabulary entries at ids 0-299 and 1000-1099 (heuristic lineage key; fine-tunes and appended vocab extensions stay in the base family)",
  "metric_check": "all 550 rows re-read from their own summary.json: 0 mismatches with competitors_all.json"
 },
 "released": {
  "name": "hindko-tokenizer-1.0.0-released",
  "sha256_tokenizer_json": "49f301c52363a09a1fc1925359af3ddabd7887495de02354cfa4092cba51da41",
  "vocab_size": 32768,
  "bytes_per_token": 7.277325843823662,
  "fertility": 1.1350127205723965,
  "strr": 0.8908184829625835,
  "tokens": 199390,
  "g1": "491/491"
 },
 "verdict": {
  "any_lossless_external_beats_released_on_bytes_per_token": false,
  "any_lossless_external_beats_released_on_fertility": false,
  "lossless_threats_bytes_per_token": [],
  "lossless_threats_fertility": [],
  "released_rank_among_lossless_behaviours": 1,
  "n_lossless_behaviours": 130,
  "best_lossless_external": {
   "representative": "ProximaAI__urnova-95m",
   "aliases": [],
   "n_repos": 1,
   "origin": "new",
   "provider": "ProximaAI",
   "repo": "ProximaAI/urnova-95m",
   "vocab_family_label": "ProximaAI/urnova-95m (BPE 50,048)",
   "model_type": "BPE",
   "vocab_size": 50048,
   "bytes_per_token": 6.061296952279107,
   "fertility": 1.3725468507122938,
   "strr": 0.7469741338658314,
   "tokens": 239392,
   "g1_pass_docs": 491,
   "unk_share": 0.0,
   "nonws_chars_lost": 0,
   "released_uses_fewer_tokens": 0.16709831573319078
  },
  "best_lossless_external_by_fertility": {
   "representative": "ProximaAI__urnova-95m",
   "aliases": [],
   "n_repos": 1,
   "origin": "new",
   "provider": "ProximaAI",
   "repo": "ProximaAI/urnova-95m",
   "vocab_family_label": "ProximaAI/urnova-95m (BPE 50,048)",
   "model_type": "BPE",
   "vocab_size": 50048,
   "bytes_per_token": 6.061296952279107,
   "fertility": 1.3725468507122938,
   "strr": 0.7469741338658314,
   "tokens": 239392,
   "g1_pass_docs": 491,
   "unk_share": 0.0,
   "nonws_chars_lost": 0,
   "released_uses_fewer_tokens": 0.16709831573319078
  },
  "lossy_with_higher_raw_bytes_per_token": 12,
  "lossy_with_lower_or_equal_fertility": 57,
  "best_lossy_with_zero_unk_and_zero_nonspace_loss": {
   "representative": "mahwizzzz__UrduNER",
   "aliases": [],
   "n_repos": 2,
   "origin": "new",
   "provider": "mahwizzzz",
   "repo": "mahwizzzz/UrduNER",
   "vocab_family_label": "roberta-urdu",
   "model_type": "BPE",
   "vocab_size": 52000,
   "bytes_per_token": 5.709890801770782,
   "fertility": 1.4582840908429344,
   "strr": 0.7213874447659356,
   "tokens": 254125,
   "g1_pass_docs": 0,
   "unk_share": 0.0,
   "nonws_chars_lost": 0,
   "released_uses_fewer_tokens": 0.21538612887358588
  },
  "best_lossy_whitespace_only_failures": {
   "representative": "mahwizzzz__UrduNER",
   "aliases": [],
   "n_repos": 2,
   "origin": "new",
   "provider": "mahwizzzz",
   "repo": "mahwizzzz/UrduNER",
   "vocab_family_label": "roberta-urdu",
   "model_type": "BPE",
   "vocab_size": 52000,
   "bytes_per_token": 5.709890801770782,
   "fertility": 1.4582840908429344,
   "strr": 0.7213874447659356,
   "tokens": 254125,
   "g1_pass_docs": 0,
   "unk_share": 0.0,
   "nonws_chars_lost": 0,
   "released_uses_fewer_tokens": 0.21538612887358588
  },
  "scope": "Token counts on the strict Hindko test split only. A tokenizer is not a model: this says nothing about generation, translation or downstream quality (LM bits-per-byte evidence is in analysis/TEST_RESULTS.md). Covers only tokenizers found and measured by 2026-09-27; closed tokenizers (Gemini, Claude 3+, newer OpenAI) are not public."
 },
 "counts": {
  "harness_rows_external": 550,
  "harness_rows_new": 485,
  "harness_rows_study_baselines": 65,
  "excluded": {
   "hindko-probe-bpe32k": "trained on the whole Hindko corpus incl. test (leaky probe; listed in study, never ranked)"
  },
  "distinct_encoding_behaviours": 528,
  "distinct_behaviours_lossless": 130,
  "distinct_behaviours_lossy": 398,
  "distinct_per_doc_token_count_profiles": 524,
  "vocabulary_families": 466,
  "vocabulary_families_with_a_lossless_member": 102,
  "repos_represented_by_measured_rows": 2608,
  "tokenizer_files_screened_ok_in_refresh": 957,
  "new_files_equal_in_behaviour_to_a_baseline": 17
 },
 "top15_lossless_behaviours": [
  {
   "representative": "ProximaAI__urnova-95m",
   "aliases": [],
   "provider": "ProximaAI",
   "repo": "ProximaAI/urnova-95m",
   "model_type": "BPE",
   "vocab_size": 50048,
   "bytes_per_token": 6.061296952279107,
   "fertility": 1.3725468507122938,
   "strr": 0.7469741338658314,
   "tokens": 239392,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.16709831573319078,
   "n_repos": 1
  },
  {
   "representative": "DunbaaBERT__DunbaaBERT_52k_base",
   "aliases": [],
   "provider": "DunbaaBERT",
   "repo": "DunbaaBERT/DunbaaBERT_52k_base",
   "model_type": "BPE",
   "vocab_size": 52010,
   "bytes_per_token": 5.964354270540893,
   "fertility": 1.395176079502122,
   "strr": 0.7324837427009531,
   "tokens": 243283,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.1804195114331869,
   "n_repos": 1
  },
  {
   "representative": "orature__ALIF-Base-100M",
   "aliases": [],
   "provider": "orature",
   "repo": "orature/ALIF-Base-100M",
   "model_type": "BPE",
   "vocab_size": 32000,
   "bytes_per_token": 5.911600544297506,
   "fertility": 1.4047412513317303,
   "strr": 0.7096507518818879,
   "tokens": 245454,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.1876685651894041,
   "n_repos": 1
  },
  {
   "representative": "DunbaaBERT__DunbaaBERT_32k_base",
   "aliases": [],
   "provider": "DunbaaBERT",
   "repo": "DunbaaBERT/DunbaaBERT_32k_base",
   "model_type": "BPE",
   "vocab_size": 32010,
   "bytes_per_token": 5.717765737365776,
   "fertility": 1.456240648778301,
   "strr": 0.6972969511378654,
   "tokens": 253775,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.2143040094571963,
   "n_repos": 1
  },
  {
   "representative": "roberta-urdu",
   "aliases": [],
   "provider": "UrduHack",
   "repo": "urduhack/roberta-urdu-small",
   "model_type": "BPE",
   "vocab_size": 52000,
   "bytes_per_token": 5.708947266954404,
   "fertility": 1.4585286052780186,
   "strr": 0.7212186133702821,
   "tokens": 254167,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.21551578293012075,
   "n_repos": 21
  },
  {
   "representative": "ibm-research__ia-multilingual-original-script-roberta",
   "aliases": [],
   "provider": "ibm-research",
   "repo": "ibm-research/ia-multilingual-original-script-roberta",
   "model_type": "BPE",
   "vocab_size": 110000,
   "bytes_per_token": 5.667848912151869,
   "fertility": 1.4692523097881458,
   "strr": 0.6933148589093492,
   "tokens": 256010,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.2211632358111011,
   "n_repos": 1
  },
  {
   "representative": "mahwizzzz__urdu-dummy",
   "aliases": [],
   "provider": "mahwizzzz",
   "repo": "mahwizzzz/urdu-dummy",
   "model_type": "BPE",
   "vocab_size": 25000,
   "bytes_per_token": 5.585832027686137,
   "fertility": 1.491147995272721,
   "strr": 0.6707613131589518,
   "tokens": 259769,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.23243343124083315,
   "n_repos": 1
  },
  {
   "representative": "hadidev__robertaurdu",
   "aliases": [],
   "provider": "hadidev",
   "repo": "hadidev/robertaurdu",
   "model_type": "BPE",
   "vocab_size": 59458,
   "bytes_per_token": 5.523068197822024,
   "fertility": 1.5011148693885392,
   "strr": 0.7751573333954322,
   "tokens": 262721,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.24105800449906933,
   "n_repos": 1
  },
  {
   "representative": "bloom",
   "aliases": [],
   "provider": "BigScience",
   "repo": "bigscience/bloom-560m",
   "model_type": "BPE",
   "vocab_size": 250680,
   "bytes_per_token": 5.121690021531185,
   "fertility": 1.6331468425618128,
   "strr": 0.5723733618988293,
   "tokens": 283310,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.2962126292753521,
   "n_repos": 1
  },
  {
   "representative": "santhosh__madlad400-3b-ct2",
   "aliases": [],
   "provider": "santhosh",
   "repo": "santhosh/madlad400-3b-ct2",
   "model_type": "Unigram",
   "vocab_size": 256000,
   "bytes_per_token": 5.08016076967783,
   "fertility": 1.5985888024032275,
   "strr": 0.5621852604369822,
   "tokens": 285626,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.30191929306155607,
   "n_repos": 8
  },
  {
   "representative": "mlx-community__K2-Horizon-MoVA-36B-A4B-oQ4e",
   "aliases": [],
   "provider": "mlx-community",
   "repo": "mlx-community/K2-Horizon-MoVA-36B-A4B-oQ4e",
   "model_type": "BPE",
   "vocab_size": 250624,
   "bytes_per_token": 5.063867804358827,
   "fertility": 1.6512234454412613,
   "strr": 0.5681583987797565,
   "tokens": 286545,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.3041581601493657,
   "n_repos": 3
  },
  {
   "representative": "MaLA-LM__mala-500-10b-v2",
   "aliases": [],
   "provider": "MaLA-LM",
   "repo": "MaLA-LM/mala-500-10b-v2",
   "model_type": "BPE",
   "vocab_size": 260164,
   "bytes_per_token": 4.912820165563475,
   "fertility": 1.698111999254813,
   "strr": 0.5555891924619693,
   "tokens": 295355,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.32491408643835384,
   "n_repos": 1
  },
  {
   "representative": "gemma-3",
   "aliases": [
    "gemma-4",
    "sarvam-30b"
   ],
   "provider": "Google",
   "repo": "unsloth/gemma-3-1b-it",
   "model_type": "BPE",
   "vocab_size": 262145,
   "bytes_per_token": 4.61036885467272,
   "fertility": 1.8071712590746876,
   "strr": 0.48661865645139696,
   "tokens": 314731,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.3664748626604942,
   "n_repos": 3
  },
  {
   "representative": "alphaedge-ai__gemma-3-270m-it-snd-32768",
   "aliases": [],
   "provider": "alphaedge-ai",
   "repo": "alphaedge-ai/gemma-3-270m-it-snd-32768",
   "model_type": "BPE",
   "vocab_size": 32768,
   "bytes_per_token": 4.531921206575072,
   "fertility": 1.8388882743684833,
   "strr": 0.47016050626131606,
   "tokens": 320179,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.37725459820912677,
   "n_repos": 3
  },
  {
   "representative": "Palash123__indic_200K_all_tokenizer",
   "aliases": [],
   "provider": "Palash123",
   "repo": "Palash123/indic_200K_all_tokenizer",
   "model_type": "BPE",
   "vocab_size": 200000,
   "bytes_per_token": 4.528470178482818,
   "fertility": 1.839511204000722,
   "strr": 0.4848371941386397,
   "tokens": 320423,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.377728814723038,
   "n_repos": 1
  }
 ],
 "study_baselines_deduplicated": [
  {
   "representative": "urdu-bert-64k",
   "aliases": [],
   "provider": "community (farahadeeba)",
   "repo": "farahadeeba/urdu-bert-64k",
   "model_type": "WordPiece",
   "vocab_size": 64000,
   "bytes_per_token": 6.221278018830712,
   "fertility": 1.3578468757459146,
   "strr": 0.7355867473176184,
   "tokens": 233236,
   "g1_pass_docs": 0,
   "released_uses_fewer_tokens": 0.14511481932463255,
   "n_repos": 1
  },
  {
   "representative": "roberta-urdu",
   "aliases": [],
   "provider": "UrduHack",
   "repo": "urduhack/roberta-urdu-small",
   "model_type": "BPE",
   "vocab_size": 52000,
   "bytes_per_token": 5.708947266954404,
   "fertility": 1.4585286052780186,
   "strr": 0.7212186133702821,
   "tokens": 254167,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.21551578293012075,
   "n_repos": 21
  },
  {
   "representative": "urdu-gpt2-20k",
   "aliases": [],
   "provider": "community (aariciah)",
   "repo": "aariciah/gpt2-urdu-20k-lc",
   "model_type": "BPE",
   "vocab_size": 20000,
   "bytes_per_token": 5.632624383275559,
   "fertility": 1.478421601103808,
   "strr": 0.6588499671069867,
   "tokens": 257611,
   "g1_pass_docs": 166,
   "released_uses_fewer_tokens": 0.22600354798514044,
   "n_repos": 2
  },
  {
   "representative": "muril",
   "aliases": [],
   "provider": "Google",
   "repo": "google/muril-base-cased",
   "model_type": "WordPiece",
   "vocab_size": 197258,
   "bytes_per_token": 5.392003924089674,
   "fertility": 1.5666796686247226,
   "strr": 0.615541803235741,
   "tokens": 269107,
   "g1_pass_docs": 0,
   "released_uses_fewer_tokens": 0.25906795438245755,
   "n_repos": 14
  },
  {
   "representative": "indicbert-v2",
   "aliases": [],
   "provider": "AI4Bharat",
   "repo": "ai4bharat/IndicBERTv2-MLM-only",
   "model_type": "WordPiece",
   "vocab_size": 250000,
   "bytes_per_token": 5.298519289404977,
   "fertility": 1.5943214433337796,
   "strr": 0.5845699747917261,
   "tokens": 273855,
   "g1_pass_docs": 0,
   "released_uses_fewer_tokens": 0.2719139690712238,
   "n_repos": 1
  },
  {
   "representative": "bloom",
   "aliases": [],
   "provider": "BigScience",
   "repo": "bigscience/bloom-560m",
   "model_type": "BPE",
   "vocab_size": 250680,
   "bytes_per_token": 5.121690021531185,
   "fertility": 1.6331468425618128,
   "strr": 0.5723733618988293,
   "tokens": 283310,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.2962126292753521,
   "n_repos": 1
  },
  {
   "representative": "sindhi-xlmr",
   "aliases": [],
   "provider": "community (Kashif786)",
   "repo": "Kashif786/xlm-roberta-base-sindhi-extended",
   "model_type": "Unigram",
   "vocab_size": 264635,
   "bytes_per_token": 5.06933439539401,
   "fertility": 1.5985771588586997,
   "strr": 0.5864853378665533,
   "tokens": 286236,
   "g1_pass_docs": 172,
   "released_uses_fewer_tokens": 0.30340697885660783,
   "n_repos": 1
  },
  {
   "representative": "xlm-r",
   "aliases": [],
   "provider": "Meta",
   "repo": "FacebookAI/xlm-roberta-base",
   "model_type": "Unigram",
   "vocab_size": 250002,
   "bytes_per_token": 4.791568839179867,
   "fertility": 1.6804254551170468,
   "strr": 0.5677566964935465,
   "tokens": 302829,
   "g1_pass_docs": 173,
   "released_uses_fewer_tokens": 0.341575608676844,
   "n_repos": 259
  },
  {
   "representative": "nllb-200",
   "aliases": [],
   "provider": "Meta",
   "repo": "facebook/nllb-200-distilled-600M",
   "model_type": "BPE",
   "vocab_size": 256204,
   "bytes_per_token": 4.617175840925073,
   "fertility": 1.8244037049758688,
   "strr": 0.4662773841612864,
   "tokens": 314267,
   "g1_pass_docs": 150,
   "released_uses_fewer_tokens": 0.3655394934880213,
   "n_repos": 4
  },
  {
   "representative": "gemma-3",
   "aliases": [
    "gemma-4",
    "sarvam-30b"
   ],
   "provider": "Google",
   "repo": "unsloth/gemma-3-1b-it",
   "model_type": "BPE",
   "vocab_size": 262145,
   "bytes_per_token": 4.61036885467272,
   "fertility": 1.8071712590746876,
   "strr": 0.48661865645139696,
   "tokens": 314731,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.3664748626604942,
   "n_repos": 3
  },
  {
   "representative": "falcon-h1",
   "aliases": [],
   "provider": "TII",
   "repo": "tiiuae/Falcon-H1-34B-Instruct",
   "model_type": "BPE",
   "vocab_size": 261120,
   "bytes_per_token": 4.506487859720609,
   "fertility": 1.845618243105566,
   "strr": 0.4560077778877446,
   "tokens": 321986,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.38074947357959665,
   "n_repos": 2
  },
  {
   "representative": "gpt-4o",
   "aliases": [
    "gpt-oss",
    "phi-4-mini"
   ],
   "provider": "OpenAI",
   "repo": "Xenova/gpt-4o",
   "model_type": "BPE",
   "vocab_size": 200000,
   "bytes_per_token": 4.337584149418278,
   "fertility": 1.9305346133469952,
   "strr": 0.4149409963381053,
   "tokens": 334524,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.4039590582439526,
   "n_repos": 3
  },
  {
   "representative": "mbert",
   "aliases": [],
   "provider": "Google",
   "repo": "google-bert/bert-base-multilingual-cased",
   "model_type": "WordPiece",
   "vocab_size": 119547,
   "bytes_per_token": 4.1505201644160055,
   "fertility": 2.035297405236102,
   "strr": 0.4189347321111493,
   "tokens": 349601,
   "g1_pass_docs": 0,
   "released_uses_fewer_tokens": 0.42966410279146794,
   "n_repos": 48
  },
  {
   "representative": "tiny-aya",
   "aliases": [],
   "provider": "Cohere",
   "repo": "mlx-community/tiny-aya-global-8bit-mlx",
   "model_type": "BPE",
   "vocab_size": 261010,
   "bytes_per_token": 4.1062279976908185,
   "fertility": 2.032299192520187,
   "strr": 0.38365479219183907,
   "tokens": 353372,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.4357504273117282,
   "n_repos": 1
  },
  {
   "representative": "mt5",
   "aliases": [],
   "provider": "Google",
   "repo": "google/mt5-small",
   "model_type": "Unigram",
   "vocab_size": 250100,
   "bytes_per_token": 3.9124922479575055,
   "fertility": 1.874470946445517,
   "strr": 0.413025633263278,
   "tokens": 370870,
   "g1_pass_docs": 175,
   "released_uses_fewer_tokens": 0.46237225982150076,
   "n_repos": 3
  },
  {
   "representative": "apertus",
   "aliases": [
    "mistral-nemo",
    "mistral-small-4",
    "nemotron-3",
    "sarvam-m"
   ],
   "provider": "Swiss AI",
   "repo": "swiss-ai/Apertus-8B-Instruct-2509",
   "model_type": "BPE",
   "vocab_size": 131072,
   "bytes_per_token": 3.901016504418474,
   "fertility": 2.1491246965401207,
   "strr": 0.3041293830667932,
   "tokens": 371961,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.46394917746753017,
   "n_repos": 14
  },
  {
   "representative": "gemma-2",
   "aliases": [],
   "provider": "Google",
   "repo": "unsloth/gemma-2-2b-it",
   "model_type": "BPE",
   "vocab_size": 256000,
   "bytes_per_token": 3.7866022964509396,
   "fertility": 2.205578422183281,
   "strr": 0.30369857191926364,
   "tokens": 383200,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.47967118997912317,
   "n_repos": 1
  },
  {
   "representative": "llama-4",
   "aliases": [],
   "provider": "Meta",
   "repo": "unsloth/Llama-4-Scout-17B-16E-Instruct",
   "model_type": "BPE",
   "vocab_size": 201135,
   "bytes_per_token": 3.7268769536216326,
   "fertility": 2.250545791149742,
   "strr": 0.2749390169355355,
   "tokens": 389341,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.4878782352744766,
   "n_repos": 1
  },
  {
   "representative": "command-a-plus",
   "aliases": [],
   "provider": "Cohere",
   "repo": "CohereLabs/command-a-plus-05-2026-bf16",
   "model_type": "BPE",
   "vocab_size": 255032,
   "bytes_per_token": 3.6850237325497823,
   "fertility": 2.2738736326112394,
   "strr": 0.26122292148175746,
   "tokens": 393763,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.49362941667957627,
   "n_repos": 1
  },
  {
   "representative": "qwen-3.5",
   "aliases": [
    "qwen-3.8"
   ],
   "provider": "Alibaba Qwen",
   "repo": "Qwen/Qwen3.5-0.8B",
   "model_type": "BPE",
   "vocab_size": 248070,
   "bytes_per_token": 3.5982036581495,
   "fertility": 2.3226833712718826,
   "strr": 0.25747952191606166,
   "tokens": 403264,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.5055596333915251,
   "n_repos": 88
  },
  {
   "representative": "kimi-k2",
   "aliases": [],
   "provider": "Moonshot AI",
   "repo": "moonshotai/Kimi-K2-Instruct",
   "model_type": "BPE",
   "vocab_size": 163840,
   "bytes_per_token": 3.529626683596895,
   "fertility": 2.375288905448597,
   "strr": 0.2777975071171166,
   "tokens": 411099,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.5149830089589126,
   "n_repos": 1
  },
  {
   "representative": "ernie-4.5",
   "aliases": [],
   "provider": "Baidu",
   "repo": "baidu/ERNIE-4.5-0.3B-PT",
   "model_type": "BPE",
   "vocab_size": 101304,
   "bytes_per_token": 3.5275244077949357,
   "fertility": 2.368884955958293,
   "strr": 0.2724822290401644,
   "tokens": 411344,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.5152718892216733,
   "n_repos": 10
  },
  {
   "representative": "pashto-lfm2.5",
   "aliases": [],
   "provider": "community (nassimjp)",
   "repo": "nassimjp/LFM2.5-2.6B-Pashto-Zi",
   "model_type": "BPE",
   "vocab_size": 125039,
   "bytes_per_token": 3.5092506154984693,
   "fertility": 2.386105758314946,
   "strr": 0.22505807217833254,
   "tokens": 413486,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.5177829479111747,
   "n_repos": 1
  },
  {
   "representative": "deepseek-r1",
   "aliases": [
    "deepseek-v3",
    "deepseek-v4",
    "deepseek-v4.1"
   ],
   "provider": "DeepSeek",
   "repo": "deepseek-ai/DeepSeek-R1",
   "model_type": "BPE",
   "vocab_size": 128815,
   "bytes_per_token": 3.372220466292344,
   "fertility": 2.4755281802886433,
   "strr": 0.2324517229535015,
   "tokens": 430288,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.5366126873164021,
   "n_repos": 4
  },
  {
   "representative": "glm-5",
   "aliases": [],
   "provider": "Zhipu AI / Z.ai",
   "repo": "zai-org/GLM-5",
   "model_type": "BPE",
   "vocab_size": 154856,
   "bytes_per_token": 3.0253785304137875,
   "fertility": 2.762588127077645,
   "strr": 0.1365613119945974,
   "tokens": 479618,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.5842733175151892,
   "n_repos": 1
  },
  {
   "representative": "command-r",
   "aliases": [
    "command-r7b"
   ],
   "provider": "Cohere",
   "repo": "adamo1139/aya-expanse-8b-ungated",
   "model_type": "BPE",
   "vocab_size": 255029,
   "bytes_per_token": 2.645120833006725,
   "fertility": 3.1073243716852286,
   "strr": 0.21701238290960534,
   "tokens": 548567,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.6365257115356922,
   "n_repos": 2
  },
  {
   "representative": "alif-1.0",
   "aliases": [
    "llama-3",
    "qalb-1.0",
    "urdu-llama3-almanach",
    "urdu-llama3.2-custom"
   ],
   "provider": "Traversaal.ai",
   "repo": "large-traversaal/Alif-1.0-8B-Instruct",
   "model_type": "BPE",
   "vocab_size": 128256,
   "bytes_per_token": 2.64122087382617,
   "fertility": 3.1188980549458867,
   "strr": 0.17801233051365498,
   "tokens": 549377,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.6370616170680607,
   "n_repos": 5
  },
  {
   "representative": "qwen-2.5",
   "aliases": [
    "qwen-3"
   ],
   "provider": "Alibaba Qwen",
   "repo": "Qwen/Qwen2.5-0.5B",
   "model_type": "BPE",
   "vocab_size": 151665,
   "bytes_per_token": 2.6325857842914915,
   "fertility": 3.0821976025941815,
   "strr": 0.02994719652556631,
   "tokens": 551179,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.6382481916038165,
   "n_repos": 60
  },
  {
   "representative": "glm-4.5",
   "aliases": [],
   "provider": "Zhipu AI / Z.ai",
   "repo": "zai-org/GLM-4.5",
   "model_type": "BPE",
   "vocab_size": 151365,
   "bytes_per_token": 2.4823892097555635,
   "fertility": 3.3733502552847137,
   "strr": 0.13585105577840006,
   "tokens": 584528,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.6588871705033805,
   "n_repos": 1
  },
  {
   "representative": "urdu-llama-bilal",
   "aliases": [],
   "provider": "community (BilalKhan1)",
   "repo": "BilalKhan1/llama-urdu-tokenizer",
   "model_type": "BPE",
   "vocab_size": 129435,
   "bytes_per_token": 2.3721203204185057,
   "fertility": 2.833980520350005,
   "strr": 0.26064656602763014,
   "tokens": 611700,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.674039561876737,
   "n_repos": 1
  },
  {
   "representative": "minimax-m2",
   "aliases": [
    "minimax-m3"
   ],
   "provider": "MiniMax",
   "repo": "MiniMaxAI/MiniMax-M2",
   "model_type": "BPE",
   "vocab_size": 200054,
   "bytes_per_token": 2.26764267842715,
   "fertility": 3.604003050608666,
   "strr": 0.06987291071147879,
   "tokens": 639883,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.688396159922986,
   "n_repos": 2
  },
  {
   "representative": "eurollm",
   "aliases": [],
   "provider": "UTTER (EU)",
   "repo": "utter-project/EuroLLM-9B-Instruct-2512",
   "model_type": "BPE",
   "vocab_size": 128000,
   "bytes_per_token": 2.146942326557245,
   "fertility": 3.6777649051924386,
   "strr": 0.061378944978430335,
   "tokens": 675857,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.7049819710382521,
   "n_repos": 1
  },
  {
   "representative": "hunyuan",
   "aliases": [],
   "provider": "Tencent",
   "repo": "tencent/Hunyuan-7B-Instruct",
   "model_type": "BPE",
   "vocab_size": 128166,
   "bytes_per_token": 1.932331007297715,
   "fertility": 4.245044216360345,
   "strr": 0.00577519808580128,
   "tokens": 750920,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.7344723805465296,
   "n_repos": 1
  },
  {
   "representative": "gpt-4",
   "aliases": [
    "olmo-2",
    "phi-4"
   ],
   "provider": "OpenAI",
   "repo": "Xenova/gpt-4",
   "model_type": "BPE",
   "vocab_size": 100263,
   "bytes_per_token": 1.9192881698043578,
   "fertility": 4.27475272022309,
   "strr": 0.007050166211598135,
   "tokens": 756023,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.736264637451506,
   "n_repos": 20
  },
  {
   "representative": "granite-4.2",
   "aliases": [],
   "provider": "IBM",
   "repo": "ibm-granite/granite-4.2-8b",
   "model_type": "BPE",
   "vocab_size": 100352,
   "bytes_per_token": 1.918772959406315,
   "fertility": 4.274776007312146,
   "strr": 0.00706180975612596,
   "tokens": 756226,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.7363354341162562,
   "n_repos": 1
  },
  {
   "representative": "lfm2",
   "aliases": [],
   "provider": "Liquid AI",
   "repo": "LiquidAI/LFM2.5-1.2B-Instruct",
   "model_type": "BPE",
   "vocab_size": 64402,
   "bytes_per_token": 1.9002286519487799,
   "fertility": 4.181191018169751,
   "strr": 0.03274164721224435,
   "tokens": 763606,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.738883665136209,
   "n_repos": 63
  },
  {
   "representative": "falcon-3",
   "aliases": [],
   "provider": "TII",
   "repo": "tiiuae/Falcon3-7B-Instruct",
   "model_type": "BPE",
   "vocab_size": 131072,
   "bytes_per_token": 1.8130482154727336,
   "fertility": 4.50383945880805,
   "strr": 0.010793565777293923,
   "tokens": 800324,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.7508634003228692,
   "n_repos": 1
  },
  {
   "representative": "llama-2",
   "aliases": [
    "urdu-llama2-almanach"
   ],
   "provider": "Meta",
   "repo": "NousResearch/Llama-2-7b-hf",
   "model_type": "BPE",
   "vocab_size": 32000,
   "bytes_per_token": 1.713758958393371,
   "fertility": 3.937066641827105,
   "strr": 0.006473810757470789,
   "tokens": 846692,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.7645070462458603,
   "n_repos": 99
  },
  {
   "representative": "grok-1",
   "aliases": [],
   "provider": "xAI",
   "repo": "Xenova/grok-1-tokenizer",
   "model_type": "BPE",
   "vocab_size": 131072,
   "bytes_per_token": 1.618097837965627,
   "fertility": 4.822173966198791,
   "strr": 0.0056121884624117276,
   "tokens": 896748,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.7776521386164229,
   "n_repos": 1
  },
  {
   "representative": "claude-legacy",
   "aliases": [],
   "provider": "Anthropic",
   "repo": "Xenova/claude-tokenizer",
   "model_type": "BPE",
   "vocab_size": 65000,
   "bytes_per_token": 1.5541900255135377,
   "fertility": 4.944966786789235,
   "strr": 0.00631662290634515,
   "tokens": 933622,
   "g1_pass_docs": 467,
   "released_uses_fewer_tokens": 0.7864339100835241,
   "n_repos": 1
  },
  {
   "representative": "gpt-2",
   "aliases": [],
   "provider": "OpenAI",
   "repo": "openai-community/gpt2",
   "model_type": "BPE",
   "vocab_size": 50257,
   "bytes_per_token": 1.3846658873488085,
   "fertility": 5.7485751212384075,
   "strr": 0.005128981364506983,
   "tokens": 1047925,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.8097287496719708,
   "n_repos": 122
  },
  {
   "representative": "sarvam-1",
   "aliases": [],
   "provider": "Sarvam AI",
   "repo": "sarvamai/sarvam-1",
   "model_type": "BPE",
   "vocab_size": 68096,
   "bytes_per_token": 1.0054379986391144,
   "fertility": 7.40664497086203,
   "strr": 0.001304076987116418,
   "tokens": 1443178,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.8618396344733636,
   "n_repos": 2
  },
  {
   "representative": "byt5",
   "aliases": [],
   "provider": "Google",
   "repo": "google/byt5-small",
   "model_type": "bytes",
   "vocab_size": 384,
   "bytes_per_token": 1.0,
   "fertility": 7.45030244106911,
   "strr": 0.000873265839586887,
   "tokens": 1451026,
   "g1_pass_docs": 491,
   "released_uses_fewer_tokens": 0.8625868867959636,
   "n_repos": 1
  }
 ],
 "search_coverage": {
  "ranking_snapshot_utc": "2026-09-27T12:49:42Z",
  "pass1": {
   "file": "_search.json",
   "generated_utc": "2026-09-27T08:17:53Z",
   "n_keywords": 43,
   "keywords": [
    "hindko",
    "hindku",
    "hinko",
    "hazarewal",
    "hazara",
    "potohari",
    "pothwari",
    "pothohari",
    "pahari",
    "lahnda",
    "lahanda",
    "saraiki",
    "seraiki",
    "siraiki",
    "multani",
    "shahmukhi",
    "punjabi",
    "panjabi",
    "pnb",
    "urdu",
    "pashto",
    "pushto",
    "pukhto",
    "sindhi",
    "kashmiri",
    "balochi",
    "baluchi",
    "brahui",
    "khowar",
    "shina",
    "gojri",
    "pakistan",
    "peshawar",
    "nastaliq",
    "nastaleeq",
    "perso-arabic",
    "persoarabic",
    "indic",
    "indo-aryan",
    "indoaryan",
    "south-asian",
    "southasian",
    "bharat"
   ],
   "n_lang_tags": 27,
   "lang_tags": [
    "hno",
    "hnd",
    "ur",
    "urd",
    "pnb",
    "skr",
    "lah",
    "phr",
    "ps",
    "pus",
    "pbt",
    "pbu",
    "pst",
    "sd",
    "snd",
    "ks",
    "kas",
    "bal",
    "bcc",
    "bgp",
    "khw",
    "scl",
    "gju",
    "pa",
    "pan",
    "trw",
    "bft"
   ],
   "n_authors": 91,
   "n_named_repos": 216,
   "n_named_missing": 15,
   "n_queries": 198,
   "n_repos": 10995,
   "n_with_tokenizer_files": 7309
  },
  "pass2": {
   "file": "_search2.json",
   "generated_utc": "2026-09-27T08:46:42Z",
   "n_keywords": 34,
   "keywords": [
    "markhor",
    "pakmosaic",
    "urdulm",
    "alif-base",
    "qehwa",
    "zabaan",
    "shaheen",
    "baat",
    "urdu-gpt",
    "urdugpt",
    "muse-glimmer",
    "Nemotron-3.5",
    "Atria-Dawn",
    "Dots3",
    "Ornith",
    "Hy-MT2",
    "Nex-N2.5",
    "Bonsai-2",
    "Ling-3.0",
    "MiMo-V2.6",
    "MiniCPM5",
    "grok-2",
    "grok-3",
    "gpt-oss",
    "o200k",
    "BrahmicTokenizer",
    "tokenizer-urdu",
    "urdu-tokenizer",
    "urdu_tokenizer",
    "UrduTokenizer",
    "punjabi-tokenizer",
    "pashto-tokenizer",
    "sindhi-tokenizer",
    "saraiki-tokenizer"
   ],
   "n_authors": 33,
   "authors": [
    "ma1993",
    "mahwizzzz",
    "themohal",
    "ProximaAI",
    "orature",
    "adiled",
    "hadidev",
    "NeerjaK",
    "aariciah",
    "Kashif786",
    "aakashMeghwar01",
    "zirak-ai",
    "tasal9",
    "junaid008",
    "meta-models",
    "DeepReinforce",
    "nex-agi",
    "PrismML",
    "dots-studio",
    "rednote-hilab",
    "OpenGVLab",
    "theschoolofai",
    "bisma10",
    "Xhaheen",
    "mirfan899",
    "Mavkif",
    "traversaal-ai",
    "large-traversaal",
    "enstazao",
    "UrduHack",
    "salmanpopa",
    "SLPG",
    "abdulwaheed1"
   ],
   "named": [
    "ProximaAI/PakMosaic",
    "hadidev/gpt2-urdu-tokenizer-withgpt2",
    "NeerjaK/Urdu_Model",
    "meta-models/Muse-Glimmer-30B",
    "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16",
    "internlm/Atria-Dawn-Preview",
    "tencent/Hy-MT2-30B-A3B",
    "XiaomiMiMo/MiMo-V2.6-Flash",
    "openbmb/MiniCPM5-2B",
    "alvarobartt/grok-2-tokenizer"
   ],
   "n_queries": 67,
   "n_repos": 3861,
   "n_kept_after_filter": 634,
   "filter": "regional keywords, regional/frontier authors, named repos, official orgs of the 2026 releases, and the 3 most-downloaded derivative repos per frontier keyword (the other derivatives are quantisations / fine-tunes)"
  },
  "pass3": {
   "file": "_search3.json",
   "generated_utc": "2026-09-27T12:54:36Z",
   "authors": [
    "DunbaaBERT",
    "eshaaftab900",
    "Humair332",
    "hafeez007",
    "shah-bakhsh",
    "ijazulhaq",
    "Hammad712",
    "alphaedge-ai",
    "Omarrran",
    "HPLT",
    "orature",
    "ProximaAI",
    "goldfish-models",
    "zuhri025",
    "SabahNawab",
    "BilalKhan1",
    "farahadeeba",
    "urduhack",
    "Imran1",
    "Khurram123",
    "shaikhsalman",
    "mfayazkhan",
    "large-traversaal",
    "enstazao",
    "nassimjp",
    "tasal9",
    "zirak-ai",
    "APARSIN",
    "Abdullah104",
    "aakashMeghwar01",
    "themohal",
    "mahwizzzz"
   ],
   "n_queries": 32,
   "n_repos": 2789,
   "n_with_tokenizer_files": 2564,
   "n_kept_after_filter": 508,
   "filter": "alphaedge-ai / goldfish-models / HPLT: only repos whose name carries a Perso-Arabic-script language code (urd, snd, pnb, pbt, pus, skr, hno, hnd, kas, bal, bcc, ur, sd, ps, arab); all repos of the other authors",
   "status": "STOPPED: started after the ranking snapshot; the probe log (logs/probe_pass3.log) ends with 458 repos still to probe; nothing from this pass was screened or measured"
  },
  "sweep_totals_at_snapshot": {
   "n_repos_found": 14856,
   "n_repos_found_pass1": 10995,
   "n_repos_found_pass2": 3861,
   "n_repos_kept_pass2": 634,
   "pass2_filter": "regional keywords, regional/frontier authors, named repos, official orgs of the 2026 releases, and the 3 most-downloaded derivative repos per frontier keyword (the other derivatives are quantisations / fine-tunes)",
   "n_repos_with_tokenizer_files": 7760,
   "n_queries": 265,
   "n_probed": 6895,
   "skipped": {
    "gated": 302,
    "non_text_pipeline": 563
   },
   "n_probe_errors": 0,
   "n_repos_all_files_already_known": 1765,
   "n_new_file_signatures": 1759,
   "n_downloaded": 880,
   "n_reconstructed": 95,
   "n_download_errors": 0,
   "n_screened_ok": 957,
   "n_screened": 970,
   "n_behaviour_groups": 502,
   "n_groups_equal_to_a_baseline": 17,
   "n_new_behaviours_harnessed": 485
  },
  "download_log": {
   "entries": 1026,
   "after_snapshot": 47,
   "after_snapshot_repos": [
    "Ayshhaa/orpheus-urdu-tts",
    "ByteDance-Seed/Seed-OSS-36B-Instruct",
    "Jarbas/m2v-256-multilingual-e5-small",
    "JuliaKreutzerCohere/tiny-aya-global-prompt-tasktype",
    "Kashif786/distilbert-base-multilingual-cased-sindhi-extended",
    "Kashif786/rembert-sindhi-extended",
    "Kashif786/sindhi_distilbert_colab",
    "KnutJaegersberg/nllb-moe-54b-4bit",
    "LingoIITGN/Ansh-160k",
    "MISHANM/Kashmiri_text_generation_Llama3_8B_instruct",
    "MoritzLaurer/xlm-v-base-mnli-xnli",
    "MoritzLaurer/xlm-v-base-mnli-xnli",
    "QCRI/TechniqueRAG-FS-Ministral-8B",
    "Qrzysztof/functiongemma-270m-it-prepaid-cards-v2",
    "Qrzysztof/functiongemma-270m-it-prepaid-cards-v2-mlx",
    "SalehAhmad/llama3.1-urdu-8b-qlora_split0",
    "ServiceNow-AI/Apriel-5B-Instruct",
    "Shinapri/pratchya-3k",
    "Soupis/Contra-200",
    "aiana94/NaSE",
    "bharatgenai/Param2-17B-A2.4B-Thinking",
    "casawolice/small100-onnx",
    "facebook/mgenre-wiki",
    "facebook/xglm-564M",
    "facebook/xlm-v-base",
    "google/madlad400-3b-mt",
    "google/rembert",
    "google/umt5-xxl",
    "huggingworld/m2m100_418M",
    "ibm-granite/granite-embedding-97m-multilingual-r2",
    "inclusionAI/Ling-mini-2.0",
    "inclusionAI/Ring-mini-2.0",
    "minishlab/M2V_multilingual_output",
    "minishlab/potion-multilingual-128M",
    "mistralai/Ministral-3-14B-Reasoning-2512",
    "nvidia/NVIDIA-Nemotron-Nano-9B-v2",
    "nvidia/Nemotron-Mini-4B-Instruct",
    "panagoa/nllb-200-1.3b-kbd-pretrain",
    "panagoa/nllb-200-1.3b-kbd-v0.1",
    "panagoa/nllb-200-1.3b-kbd-v0.2",
    "rahular/varta-t5",
    "setu4993/LEALLA-base",
    "shah-bakhsh/balochi-english-translator",
    "tangledgroup/tangled-alpha-0.5-core",
    "tiiuae/Falcon-H1-7B-Base",
    "tiiuae/Falcon-H1-7B-Instruct",
    "tiiuae/Falcon-H1R-7B"
   ],
   "errors": [
    {
     "repo": "MoritzLaurer/xlm-v-base-mnli-xnli",
     "error": "RemoteProtocolError: peer closed connection without sending complete message body (received 25494361 bytes, expected 61333645)"
    }
   ],
   "note": "downloads logged after the snapshot were not screened or measured"
  },
  "probe_cache_lines": 7050,
  "github_and_web": "GitHub search API + web search (refresh, before the snapshot): no Hindko tokenizer or Hindko LM found; Urdu tokenizer projects on GitHub publish code only and were not run (no third-party code).",
  "not_collected": [
   "gated repos (not logged into): 302 skipped at search, 65 unmatched after metadata check",
   "closed tokenizers: Gemini, Claude 3+, OpenAI encodings newer than o200k_base / o200k_harmony",
   "tiktoken rank files whose split regex exists only in repo Python code; xAI tokenizer.tok.json (Grok 2)",
   "392 large files classified by range reads as identical / expected-identical to a measured tokenizer (not measured themselves)",
   "13 downloaded tokenizers that could not be loaded"
  ]
 }
}