Add evaluation tables and charts

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+eval/chart_bytes_per_token_test.png filter=lfs diff=lfs merge=lfs -text
+eval/chart_lm_bpb_dev_test.png filter=lfs diff=lfs merge=lfs -text

eval/chart_data.json

@@ -0,0 +1,176 @@
+{
+ "chart_bytes_per_token_test.png": [
+  {
+   "label": "hindko-tokenizer (this, 32k)",
+   "bytes_per_token": 7.277325843823662,
+   "tokens_vs_released": 1.0
+  },
+  {
+   "label": "urnova-95m (best external)",
+   "bytes_per_token": 6.061296952279107,
+   "tokens_vs_released": 1.2006
+  },
+  {
+   "label": "RoBERTa-Urdu (UrduHack)",
+   "bytes_per_token": 5.708947266954404,
+   "tokens_vs_released": 1.2747
+  },
+  {
+   "label": "BLOOM",
+   "bytes_per_token": 5.121690021531185,
+   "tokens_vs_released": 1.4209
+  },
+  {
+   "label": "Gemma 3 / Gemma 4",
+   "bytes_per_token": 4.61036885467272,
+   "tokens_vs_released": 1.5785
+  },
+  {
+   "label": "GPT-4o (o200k)",
+   "bytes_per_token": 4.337584149418278,
+   "tokens_vs_released": 1.6777
+  },
+  {
+   "label": "Mistral Tekken (NeMo, Small 4)",
+   "bytes_per_token": 3.901016504418474,
+   "tokens_vs_released": 1.8655
+  },
+  {
+   "label": "Llama 4",
+   "bytes_per_token": 3.7268769536216326,
+   "tokens_vs_released": 1.9527
+  },
+  {
+   "label": "Qwen 3.5",
+   "bytes_per_token": 3.5982036581495,
+   "tokens_vs_released": 2.0225
+  },
+  {
+   "label": "DeepSeek-V3 / R1 / V4",
+   "bytes_per_token": 3.372220466292344,
+   "tokens_vs_released": 2.158
+  },
+  {
+   "label": "Llama 3",
+   "bytes_per_token": 2.64122087382617,
+   "tokens_vs_released": 2.7553
+  }
+ ],
+ "chart_lm_bpb_dev_test.png": {
+  "confirm": [
+   {
+    "id": "R2-A4-SPnat-D2-32k",
+    "dev_delta_pct": -1.623655,
+    "dev_ci95": [
+     -2.107054,
+     -1.236059
+    ],
+    "test_delta_pct": -1.0148512414859043,
+    "test_ci95": [
+     -1.3337703266599765,
+     -0.719411106580577
+    ]
+   },
+   {
+    "id": "R2-A10-MinGram-P1r3-D2-48k",
+    "dev_delta_pct": -1.58176,
+    "dev_ci95": [
+     -1.867521,
+     -1.355869
+    ],
+    "test_delta_pct": -1.3160397372781005,
+    "test_ci95": [
+     -1.467033141440053,
+     -1.1643594058582694
+    ]
+   },
+   {
+    "id": "R2-A10-MinGram-P1r3-D2-32k",
+    "dev_delta_pct": -1.414409,
+    "dev_ci95": [
+     -1.66217,
+     -1.19142
+    ],
+    "test_delta_pct": -1.253180889269175,
+    "test_ci95": [
+     -1.4797207661189518,
+     -1.03708384604859
+    ]
+   },
+   {
+    "id": "R2-A1-P1r3-D2-48k",
+    "dev_delta_pct": -0.98924,
+    "dev_ci95": [
+     -1.211704,
+     -0.778742
+    ],
+    "test_delta_pct": -0.850660355728833,
+    "test_ci95": [
+     -1.0468293493009588,
+     -0.6530838548134663
+    ]
+   },
+   {
+    "id": "A1-P1r3-D2-32k",
+    "dev_delta_pct": -0.810497,
+    "dev_ci95": [
+     -1.000743,
+     -0.657493
+    ],
+    "test_delta_pct": -0.5660213536703774,
+    "test_ci95": [
+     -0.7539943802074569,
+     -0.38067108247804304
+    ]
+   }
+  ],
+  "large": [
+   {
+    "id": "R2-A4-SPnat-D2-32k",
+    "dev_delta_pct": -1.335532,
+    "dev_ci95": [
+     -1.775676,
+     -1.026276
+    ],
+    "test_delta_pct": -0.7701554030632566,
+    "test_ci95": [
+     -1.1521793550485442,
+     -0.3981661969961223
+    ]
+   },
+   {
+    "id": "R2-A10-MinGram-P1r3-D2-32k",
+    "dev_delta_pct": -0.715001,
+    "dev_ci95": [
+     -0.910685,
+     -0.524984
+    ],
+    "test_delta_pct": -0.4672962320113417,
+    "test_ci95": null
+   },
+   {
+    "id": "R2-A10-MinGram-P1r3-D2-48k",
+    "dev_delta_pct": -0.436436,
+    "dev_ci95": [
+     -0.622803,
+     -0.286684
+    ],
+    "test_delta_pct": -0.25657526280652554,
+    "test_ci95": [
+     -0.4354746500167368,
+     -0.08159379637904507
+    ]
+   },
+   {
+    "id": "A1-P1r3-D2-32k",
+    "dev_delta_pct": 0.027361,
+    "dev_ci95": [
+     -0.296706,
+     0.317218
+    ],
+    "test_delta_pct": null,
+    "test_ci95": null
+   }
+  ]
+ }
+}

eval/competitors_summary.json

@@ -0,0 +1,1362 @@
+{
+ "what": "Held-out Hindko (test_strict) token-count ranking of the released tokenizer against every external tokenizer measured; compact copy of sota/competitors/competitors_final.json (full per-behaviour table: competitors_test_strict.csv). Screening metrics only; a tokenizer is not a model.",
+ "generated_utc_source": "2026-09-27T13:07:05Z",
+ "dataset": {
+  "name": "test_strict",
+  "path": "data/test_strict.jsonl (study folder; not published)",
+  "sha256": "a74c33ad008f4b0e4a6f7779b0de24e265b0f4edf6f94e89885c7943d3a08468",
+  "docs": 491,
+  "test_docs": 491
+ },
+ "method": {
+  "metrics": "bytes/token = UTF-8 bytes / tokens; fertility = harness definition: sum over whitespace words of the number of tokens overlapping the word, divided by words (standalone '▁' and newline tokens belong to no word; the plain ratio tokens / whitespace words is reported separately as tokens_per_word, released 1.161 vs fertility 1.135); every tokenizer scored with the same definition; each tokenizer with its own native encoder, no BOS/EOS/CLS (eval/harness.py)",
+  "lossless": "G1: decode(encode(doc)) == doc for all 491 test documents",
+  "dedup": "behaviour = sha256 of the per-document (token count, round-trip ok) vector over the 491 docs (same key as scripts/screen.py); identical vectors are treated as identical encodings (a strong proxy, not a token-by-token proof)",
+  "family": "vocabulary family = sha256 of vocabulary entries at ids 0-299 and 1000-1099 (heuristic lineage key; fine-tunes and appended vocab extensions stay in the base family)",
+  "metric_check": "all 550 rows re-read from their own summary.json: 0 mismatches with competitors_all.json"
+ },
+ "released": {
+  "name": "hindko-tokenizer-1.0.0-released",
+  "sha256_tokenizer_json": "49f301c52363a09a1fc1925359af3ddabd7887495de02354cfa4092cba51da41",
+  "vocab_size": 32768,
+  "bytes_per_token": 7.277325843823662,
+  "fertility": 1.1350127205723965,
+  "strr": 0.8908184829625835,
+  "tokens": 199390,
+  "g1": "491/491"
+ },
+ "verdict": {
+  "any_lossless_external_beats_released_on_bytes_per_token": false,
+  "any_lossless_external_beats_released_on_fertility": false,
+  "lossless_threats_bytes_per_token": [],
+  "lossless_threats_fertility": [],
+  "released_rank_among_lossless_behaviours": 1,
+  "n_lossless_behaviours": 130,
+  "best_lossless_external": {
+   "representative": "ProximaAI__urnova-95m",
+   "aliases": [],
+   "n_repos": 1,
+   "origin": "new",
+   "provider": "ProximaAI",
+   "repo": "ProximaAI/urnova-95m",
+   "vocab_family_label": "ProximaAI/urnova-95m (BPE 50,048)",
+   "model_type": "BPE",
+   "vocab_size": 50048,
+   "bytes_per_token": 6.061296952279107,
+   "fertility": 1.3725468507122938,
+   "strr": 0.7469741338658314,
+   "tokens": 239392,
+   "g1_pass_docs": 491,
+   "unk_share": 0.0,
+   "nonws_chars_lost": 0,
+   "released_uses_fewer_tokens": 0.16709831573319078
+  },
+  "best_lossless_external_by_fertility": {
+   "representative": "ProximaAI__urnova-95m",
+   "aliases": [],
+   "n_repos": 1,
+   "origin": "new",
+   "provider": "ProximaAI",
+   "repo": "ProximaAI/urnova-95m",
+   "vocab_family_label": "ProximaAI/urnova-95m (BPE 50,048)",
+   "model_type": "BPE",
+   "vocab_size": 50048,
+   "bytes_per_token": 6.061296952279107,
+   "fertility": 1.3725468507122938,
+   "strr": 0.7469741338658314,
+   "tokens": 239392,
+   "g1_pass_docs": 491,
+   "unk_share": 0.0,
+   "nonws_chars_lost": 0,
+   "released_uses_fewer_tokens": 0.16709831573319078
+  },
+  "lossy_with_higher_raw_bytes_per_token": 12,
+  "lossy_with_lower_or_equal_fertility": 57,
+  "best_lossy_with_zero_unk_and_zero_nonspace_loss": {
+   "representative": "mahwizzzz__UrduNER",
+   "aliases": [],
+   "n_repos": 2,
+   "origin": "new",
+   "provider": "mahwizzzz",
+   "repo": "mahwizzzz/UrduNER",
+   "vocab_family_label": "roberta-urdu",
+   "model_type": "BPE",
+   "vocab_size": 52000,
+   "bytes_per_token": 5.709890801770782,
+   "fertility": 1.4582840908429344,
+   "strr": 0.7213874447659356,
+   "tokens": 254125,
+   "g1_pass_docs": 0,
+   "unk_share": 0.0,
+   "nonws_chars_lost": 0,
+   "released_uses_fewer_tokens": 0.21538612887358588
+  },
+  "best_lossy_whitespace_only_failures": {
+   "representative": "mahwizzzz__UrduNER",
+   "aliases": [],
+   "n_repos": 2,
+   "origin": "new",
+   "provider": "mahwizzzz",
+   "repo": "mahwizzzz/UrduNER",
+   "vocab_family_label": "roberta-urdu",
+   "model_type": "BPE",
+   "vocab_size": 52000,
+   "bytes_per_token": 5.709890801770782,
+   "fertility": 1.4582840908429344,
+   "strr": 0.7213874447659356,
+   "tokens": 254125,
+   "g1_pass_docs": 0,
+   "unk_share": 0.0,
+   "nonws_chars_lost": 0,
+   "released_uses_fewer_tokens": 0.21538612887358588
+  },
+  "scope": "Token counts on the strict Hindko test split only. A tokenizer is not a model: this says nothing about generation, translation or downstream quality (LM bits-per-byte evidence is in analysis/TEST_RESULTS.md). Covers only tokenizers found and measured by 2026-09-27; closed tokenizers (Gemini, Claude 3+, newer OpenAI) are not public."
+ },
+ "counts": {
+  "harness_rows_external": 550,
+  "harness_rows_new": 485,
+  "harness_rows_study_baselines": 65,
+  "excluded": {
+   "hindko-probe-bpe32k": "trained on the whole Hindko corpus incl. test (leaky probe; listed in study, never ranked)"
+  },
+  "distinct_encoding_behaviours": 528,
+  "distinct_behaviours_lossless": 130,
+  "distinct_behaviours_lossy": 398,
+  "distinct_per_doc_token_count_profiles": 524,
+  "vocabulary_families": 466,
+  "vocabulary_families_with_a_lossless_member": 102,
+  "repos_represented_by_measured_rows": 2608,
+  "tokenizer_files_screened_ok_in_refresh": 957,
+  "new_files_equal_in_behaviour_to_a_baseline": 17
+ },
+ "top15_lossless_behaviours": [
+  {
+   "representative": "ProximaAI__urnova-95m",
+   "aliases": [],
+   "provider": "ProximaAI",
+   "repo": "ProximaAI/urnova-95m",
+   "model_type": "BPE",
+   "vocab_size": 50048,
+   "bytes_per_token": 6.061296952279107,
+   "fertility": 1.3725468507122938,
+   "strr": 0.7469741338658314,
+   "tokens": 239392,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.16709831573319078,
+   "n_repos": 1
+  },
+  {
+   "representative": "DunbaaBERT__DunbaaBERT_52k_base",
+   "aliases": [],
+   "provider": "DunbaaBERT",
+   "repo": "DunbaaBERT/DunbaaBERT_52k_base",
+   "model_type": "BPE",
+   "vocab_size": 52010,
+   "bytes_per_token": 5.964354270540893,
+   "fertility": 1.395176079502122,
+   "strr": 0.7324837427009531,
+   "tokens": 243283,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.1804195114331869,
+   "n_repos": 1
+  },
+  {
+   "representative": "orature__ALIF-Base-100M",
+   "aliases": [],
+   "provider": "orature",
+   "repo": "orature/ALIF-Base-100M",
+   "model_type": "BPE",
+   "vocab_size": 32000,
+   "bytes_per_token": 5.911600544297506,
+   "fertility": 1.4047412513317303,
+   "strr": 0.7096507518818879,
+   "tokens": 245454,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.1876685651894041,
+   "n_repos": 1
+  },
+  {
+   "representative": "DunbaaBERT__DunbaaBERT_32k_base",
+   "aliases": [],
+   "provider": "DunbaaBERT",
+   "repo": "DunbaaBERT/DunbaaBERT_32k_base",
+   "model_type": "BPE",
+   "vocab_size": 32010,
+   "bytes_per_token": 5.717765737365776,
+   "fertility": 1.456240648778301,
+   "strr": 0.6972969511378654,
+   "tokens": 253775,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.2143040094571963,
+   "n_repos": 1
+  },
+  {
+   "representative": "roberta-urdu",
+   "aliases": [],
+   "provider": "UrduHack",
+   "repo": "urduhack/roberta-urdu-small",
+   "model_type": "BPE",
+   "vocab_size": 52000,
+   "bytes_per_token": 5.708947266954404,
+   "fertility": 1.4585286052780186,
+   "strr": 0.7212186133702821,
+   "tokens": 254167,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.21551578293012075,
+   "n_repos": 21
+  },
+  {
+   "representative": "ibm-research__ia-multilingual-original-script-roberta",
+   "aliases": [],
+   "provider": "ibm-research",
+   "repo": "ibm-research/ia-multilingual-original-script-roberta",
+   "model_type": "BPE",
+   "vocab_size": 110000,
+   "bytes_per_token": 5.667848912151869,
+   "fertility": 1.4692523097881458,
+   "strr": 0.6933148589093492,
+   "tokens": 256010,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.2211632358111011,
+   "n_repos": 1
+  },
+  {
+   "representative": "mahwizzzz__urdu-dummy",
+   "aliases": [],
+   "provider": "mahwizzzz",
+   "repo": "mahwizzzz/urdu-dummy",
+   "model_type": "BPE",
+   "vocab_size": 25000,
+   "bytes_per_token": 5.585832027686137,
+   "fertility": 1.491147995272721,
+   "strr": 0.6707613131589518,
+   "tokens": 259769,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.23243343124083315,
+   "n_repos": 1
+  },
+  {
+   "representative": "hadidev__robertaurdu",
+   "aliases": [],
+   "provider": "hadidev",
+   "repo": "hadidev/robertaurdu",
+   "model_type": "BPE",
+   "vocab_size": 59458,
+   "bytes_per_token": 5.523068197822024,
+   "fertility": 1.5011148693885392,
+   "strr": 0.7751573333954322,
+   "tokens": 262721,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.24105800449906933,
+   "n_repos": 1
+  },
+  {
+   "representative": "bloom",
+   "aliases": [],
+   "provider": "BigScience",
+   "repo": "bigscience/bloom-560m",
+   "model_type": "BPE",
+   "vocab_size": 250680,
+   "bytes_per_token": 5.121690021531185,
+   "fertility": 1.6331468425618128,
+   "strr": 0.5723733618988293,
+   "tokens": 283310,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.2962126292753521,
+   "n_repos": 1
+  },
+  {
+   "representative": "santhosh__madlad400-3b-ct2",
+   "aliases": [],
+   "provider": "santhosh",
+   "repo": "santhosh/madlad400-3b-ct2",
+   "model_type": "Unigram",
+   "vocab_size": 256000,
+   "bytes_per_token": 5.08016076967783,
+   "fertility": 1.5985888024032275,
+   "strr": 0.5621852604369822,
+   "tokens": 285626,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.30191929306155607,
+   "n_repos": 8
+  },
+  {
+   "representative": "mlx-community__K2-Horizon-MoVA-36B-A4B-oQ4e",
+   "aliases": [],
+   "provider": "mlx-community",
+   "repo": "mlx-community/K2-Horizon-MoVA-36B-A4B-oQ4e",
+   "model_type": "BPE",
+   "vocab_size": 250624,
+   "bytes_per_token": 5.063867804358827,
+   "fertility": 1.6512234454412613,
+   "strr": 0.5681583987797565,
+   "tokens": 286545,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.3041581601493657,
+   "n_repos": 3
+  },
+  {
+   "representative": "MaLA-LM__mala-500-10b-v2",
+   "aliases": [],
+   "provider": "MaLA-LM",
+   "repo": "MaLA-LM/mala-500-10b-v2",
+   "model_type": "BPE",
+   "vocab_size": 260164,
+   "bytes_per_token": 4.912820165563475,
+   "fertility": 1.698111999254813,
+   "strr": 0.5555891924619693,
+   "tokens": 295355,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.32491408643835384,
+   "n_repos": 1
+  },
+  {
+   "representative": "gemma-3",
+   "aliases": [
+    "gemma-4",
+    "sarvam-30b"
+   ],
+   "provider": "Google",
+   "repo": "unsloth/gemma-3-1b-it",
+   "model_type": "BPE",
+   "vocab_size": 262145,
+   "bytes_per_token": 4.61036885467272,
+   "fertility": 1.8071712590746876,
+   "strr": 0.48661865645139696,
+   "tokens": 314731,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.3664748626604942,
+   "n_repos": 3
+  },
+  {
+   "representative": "alphaedge-ai__gemma-3-270m-it-snd-32768",
+   "aliases": [],
+   "provider": "alphaedge-ai",
+   "repo": "alphaedge-ai/gemma-3-270m-it-snd-32768",
+   "model_type": "BPE",
+   "vocab_size": 32768,
+   "bytes_per_token": 4.531921206575072,
+   "fertility": 1.8388882743684833,
+   "strr": 0.47016050626131606,
+   "tokens": 320179,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.37725459820912677,
+   "n_repos": 3
+  },
+  {
+   "representative": "Palash123__indic_200K_all_tokenizer",
+   "aliases": [],
+   "provider": "Palash123",
+   "repo": "Palash123/indic_200K_all_tokenizer",
+   "model_type": "BPE",
+   "vocab_size": 200000,
+   "bytes_per_token": 4.528470178482818,
+   "fertility": 1.839511204000722,
+   "strr": 0.4848371941386397,
+   "tokens": 320423,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.377728814723038,
+   "n_repos": 1
+  }
+ ],
+ "study_baselines_deduplicated": [
+  {
+   "representative": "urdu-bert-64k",
+   "aliases": [],
+   "provider": "community (farahadeeba)",
+   "repo": "farahadeeba/urdu-bert-64k",
+   "model_type": "WordPiece",
+   "vocab_size": 64000,
+   "bytes_per_token": 6.221278018830712,
+   "fertility": 1.3578468757459146,
+   "strr": 0.7355867473176184,
+   "tokens": 233236,
+   "g1_pass_docs": 0,
+   "released_uses_fewer_tokens": 0.14511481932463255,
+   "n_repos": 1
+  },
+  {
+   "representative": "roberta-urdu",
+   "aliases": [],
+   "provider": "UrduHack",
+   "repo": "urduhack/roberta-urdu-small",
+   "model_type": "BPE",
+   "vocab_size": 52000,
+   "bytes_per_token": 5.708947266954404,
+   "fertility": 1.4585286052780186,
+   "strr": 0.7212186133702821,
+   "tokens": 254167,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.21551578293012075,
+   "n_repos": 21
+  },
+  {
+   "representative": "urdu-gpt2-20k",
+   "aliases": [],
+   "provider": "community (aariciah)",
+   "repo": "aariciah/gpt2-urdu-20k-lc",
+   "model_type": "BPE",
+   "vocab_size": 20000,
+   "bytes_per_token": 5.632624383275559,
+   "fertility": 1.478421601103808,
+   "strr": 0.6588499671069867,
+   "tokens": 257611,
+   "g1_pass_docs": 166,
+   "released_uses_fewer_tokens": 0.22600354798514044,
+   "n_repos": 2
+  },
+  {
+   "representative": "muril",
+   "aliases": [],
+   "provider": "Google",
+   "repo": "google/muril-base-cased",
+   "model_type": "WordPiece",
+   "vocab_size": 197258,
+   "bytes_per_token": 5.392003924089674,
+   "fertility": 1.5666796686247226,
+   "strr": 0.615541803235741,
+   "tokens": 269107,
+   "g1_pass_docs": 0,
+   "released_uses_fewer_tokens": 0.25906795438245755,
+   "n_repos": 14
+  },
+  {
+   "representative": "indicbert-v2",
+   "aliases": [],
+   "provider": "AI4Bharat",
+   "repo": "ai4bharat/IndicBERTv2-MLM-only",
+   "model_type": "WordPiece",
+   "vocab_size": 250000,
+   "bytes_per_token": 5.298519289404977,
+   "fertility": 1.5943214433337796,
+   "strr": 0.5845699747917261,
+   "tokens": 273855,
+   "g1_pass_docs": 0,
+   "released_uses_fewer_tokens": 0.2719139690712238,
+   "n_repos": 1
+  },
+  {
+   "representative": "bloom",
+   "aliases": [],
+   "provider": "BigScience",
+   "repo": "bigscience/bloom-560m",
+   "model_type": "BPE",
+   "vocab_size": 250680,
+   "bytes_per_token": 5.121690021531185,
+   "fertility": 1.6331468425618128,
+   "strr": 0.5723733618988293,
+   "tokens": 283310,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.2962126292753521,
+   "n_repos": 1
+  },
+  {
+   "representative": "sindhi-xlmr",
+   "aliases": [],
+   "provider": "community (Kashif786)",
+   "repo": "Kashif786/xlm-roberta-base-sindhi-extended",
+   "model_type": "Unigram",
+   "vocab_size": 264635,
+   "bytes_per_token": 5.06933439539401,
+   "fertility": 1.5985771588586997,
+   "strr": 0.5864853378665533,
+   "tokens": 286236,
+   "g1_pass_docs": 172,
+   "released_uses_fewer_tokens": 0.30340697885660783,
+   "n_repos": 1
+  },
+  {
+   "representative": "xlm-r",
+   "aliases": [],
+   "provider": "Meta",
+   "repo": "FacebookAI/xlm-roberta-base",
+   "model_type": "Unigram",
+   "vocab_size": 250002,
+   "bytes_per_token": 4.791568839179867,
+   "fertility": 1.6804254551170468,
+   "strr": 0.5677566964935465,
+   "tokens": 302829,
+   "g1_pass_docs": 173,
+   "released_uses_fewer_tokens": 0.341575608676844,
+   "n_repos": 259
+  },
+  {
+   "representative": "nllb-200",
+   "aliases": [],
+   "provider": "Meta",
+   "repo": "facebook/nllb-200-distilled-600M",
+   "model_type": "BPE",
+   "vocab_size": 256204,
+   "bytes_per_token": 4.617175840925073,
+   "fertility": 1.8244037049758688,
+   "strr": 0.4662773841612864,
+   "tokens": 314267,
+   "g1_pass_docs": 150,
+   "released_uses_fewer_tokens": 0.3655394934880213,
+   "n_repos": 4
+  },
+  {
+   "representative": "gemma-3",
+   "aliases": [
+    "gemma-4",
+    "sarvam-30b"
+   ],
+   "provider": "Google",
+   "repo": "unsloth/gemma-3-1b-it",
+   "model_type": "BPE",
+   "vocab_size": 262145,
+   "bytes_per_token": 4.61036885467272,
+   "fertility": 1.8071712590746876,
+   "strr": 0.48661865645139696,
+   "tokens": 314731,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.3664748626604942,
+   "n_repos": 3
+  },
+  {
+   "representative": "falcon-h1",
+   "aliases": [],
+   "provider": "TII",
+   "repo": "tiiuae/Falcon-H1-34B-Instruct",
+   "model_type": "BPE",
+   "vocab_size": 261120,
+   "bytes_per_token": 4.506487859720609,
+   "fertility": 1.845618243105566,
+   "strr": 0.4560077778877446,
+   "tokens": 321986,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.38074947357959665,
+   "n_repos": 2
+  },
+  {
+   "representative": "gpt-4o",
+   "aliases": [
+    "gpt-oss",
+    "phi-4-mini"
+   ],
+   "provider": "OpenAI",
+   "repo": "Xenova/gpt-4o",
+   "model_type": "BPE",
+   "vocab_size": 200000,
+   "bytes_per_token": 4.337584149418278,
+   "fertility": 1.9305346133469952,
+   "strr": 0.4149409963381053,
+   "tokens": 334524,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.4039590582439526,
+   "n_repos": 3
+  },
+  {
+   "representative": "mbert",
+   "aliases": [],
+   "provider": "Google",
+   "repo": "google-bert/bert-base-multilingual-cased",
+   "model_type": "WordPiece",
+   "vocab_size": 119547,
+   "bytes_per_token": 4.1505201644160055,
+   "fertility": 2.035297405236102,
+   "strr": 0.4189347321111493,
+   "tokens": 349601,
+   "g1_pass_docs": 0,
+   "released_uses_fewer_tokens": 0.42966410279146794,
+   "n_repos": 48
+  },
+  {
+   "representative": "tiny-aya",
+   "aliases": [],
+   "provider": "Cohere",
+   "repo": "mlx-community/tiny-aya-global-8bit-mlx",
+   "model_type": "BPE",
+   "vocab_size": 261010,
+   "bytes_per_token": 4.1062279976908185,
+   "fertility": 2.032299192520187,
+   "strr": 0.38365479219183907,
+   "tokens": 353372,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.4357504273117282,
+   "n_repos": 1
+  },
+  {
+   "representative": "mt5",
+   "aliases": [],
+   "provider": "Google",
+   "repo": "google/mt5-small",
+   "model_type": "Unigram",
+   "vocab_size": 250100,
+   "bytes_per_token": 3.9124922479575055,
+   "fertility": 1.874470946445517,
+   "strr": 0.413025633263278,
+   "tokens": 370870,
+   "g1_pass_docs": 175,
+   "released_uses_fewer_tokens": 0.46237225982150076,
+   "n_repos": 3
+  },
+  {
+   "representative": "apertus",
+   "aliases": [
+    "mistral-nemo",
+    "mistral-small-4",
+    "nemotron-3",
+    "sarvam-m"
+   ],
+   "provider": "Swiss AI",
+   "repo": "swiss-ai/Apertus-8B-Instruct-2509",
+   "model_type": "BPE",
+   "vocab_size": 131072,
+   "bytes_per_token": 3.901016504418474,
+   "fertility": 2.1491246965401207,
+   "strr": 0.3041293830667932,
+   "tokens": 371961,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.46394917746753017,
+   "n_repos": 14
+  },
+  {
+   "representative": "gemma-2",
+   "aliases": [],
+   "provider": "Google",
+   "repo": "unsloth/gemma-2-2b-it",
+   "model_type": "BPE",
+   "vocab_size": 256000,
+   "bytes_per_token": 3.7866022964509396,
+   "fertility": 2.205578422183281,
+   "strr": 0.30369857191926364,
+   "tokens": 383200,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.47967118997912317,
+   "n_repos": 1
+  },
+  {
+   "representative": "llama-4",
+   "aliases": [],
+   "provider": "Meta",
+   "repo": "unsloth/Llama-4-Scout-17B-16E-Instruct",
+   "model_type": "BPE",
+   "vocab_size": 201135,
+   "bytes_per_token": 3.7268769536216326,
+   "fertility": 2.250545791149742,
+   "strr": 0.2749390169355355,
+   "tokens": 389341,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.4878782352744766,
+   "n_repos": 1
+  },
+  {
+   "representative": "command-a-plus",
+   "aliases": [],
+   "provider": "Cohere",
+   "repo": "CohereLabs/command-a-plus-05-2026-bf16",
+   "model_type": "BPE",
+   "vocab_size": 255032,
+   "bytes_per_token": 3.6850237325497823,
+   "fertility": 2.2738736326112394,
+   "strr": 0.26122292148175746,
+   "tokens": 393763,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.49362941667957627,
+   "n_repos": 1
+  },
+  {
+   "representative": "qwen-3.5",
+   "aliases": [
+    "qwen-3.8"
+   ],
+   "provider": "Alibaba Qwen",
+   "repo": "Qwen/Qwen3.5-0.8B",
+   "model_type": "BPE",
+   "vocab_size": 248070,
+   "bytes_per_token": 3.5982036581495,
+   "fertility": 2.3226833712718826,
+   "strr": 0.25747952191606166,
+   "tokens": 403264,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.5055596333915251,
+   "n_repos": 88
+  },
+  {
+   "representative": "kimi-k2",
+   "aliases": [],
+   "provider": "Moonshot AI",
+   "repo": "moonshotai/Kimi-K2-Instruct",
+   "model_type": "BPE",
+   "vocab_size": 163840,
+   "bytes_per_token": 3.529626683596895,
+   "fertility": 2.375288905448597,
+   "strr": 0.2777975071171166,
+   "tokens": 411099,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.5149830089589126,
+   "n_repos": 1
+  },
+  {
+   "representative": "ernie-4.5",
+   "aliases": [],
+   "provider": "Baidu",
+   "repo": "baidu/ERNIE-4.5-0.3B-PT",
+   "model_type": "BPE",
+   "vocab_size": 101304,
+   "bytes_per_token": 3.5275244077949357,
+   "fertility": 2.368884955958293,
+   "strr": 0.2724822290401644,
+   "tokens": 411344,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.5152718892216733,
+   "n_repos": 10
+  },
+  {
+   "representative": "pashto-lfm2.5",
+   "aliases": [],
+   "provider": "community (nassimjp)",
+   "repo": "nassimjp/LFM2.5-2.6B-Pashto-Zi",
+   "model_type": "BPE",
+   "vocab_size": 125039,
+   "bytes_per_token": 3.5092506154984693,
+   "fertility": 2.386105758314946,
+   "strr": 0.22505807217833254,
+   "tokens": 413486,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.5177829479111747,
+   "n_repos": 1
+  },
+  {
+   "representative": "deepseek-r1",
+   "aliases": [
+    "deepseek-v3",
+    "deepseek-v4",
+    "deepseek-v4.1"
+   ],
+   "provider": "DeepSeek",
+   "repo": "deepseek-ai/DeepSeek-R1",
+   "model_type": "BPE",
+   "vocab_size": 128815,
+   "bytes_per_token": 3.372220466292344,
+   "fertility": 2.4755281802886433,
+   "strr": 0.2324517229535015,
+   "tokens": 430288,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.5366126873164021,
+   "n_repos": 4
+  },
+  {
+   "representative": "glm-5",
+   "aliases": [],
+   "provider": "Zhipu AI / Z.ai",
+   "repo": "zai-org/GLM-5",
+   "model_type": "BPE",
+   "vocab_size": 154856,
+   "bytes_per_token": 3.0253785304137875,
+   "fertility": 2.762588127077645,
+   "strr": 0.1365613119945974,
+   "tokens": 479618,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.5842733175151892,
+   "n_repos": 1
+  },
+  {
+   "representative": "command-r",
+   "aliases": [
+    "command-r7b"
+   ],
+   "provider": "Cohere",
+   "repo": "adamo1139/aya-expanse-8b-ungated",
+   "model_type": "BPE",
+   "vocab_size": 255029,
+   "bytes_per_token": 2.645120833006725,
+   "fertility": 3.1073243716852286,
+   "strr": 0.21701238290960534,
+   "tokens": 548567,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.6365257115356922,
+   "n_repos": 2
+  },
+  {
+   "representative": "alif-1.0",
+   "aliases": [
+    "llama-3",
+    "qalb-1.0",
+    "urdu-llama3-almanach",
+    "urdu-llama3.2-custom"
+   ],
+   "provider": "Traversaal.ai",
+   "repo": "large-traversaal/Alif-1.0-8B-Instruct",
+   "model_type": "BPE",
+   "vocab_size": 128256,
+   "bytes_per_token": 2.64122087382617,
+   "fertility": 3.1188980549458867,
+   "strr": 0.17801233051365498,
+   "tokens": 549377,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.6370616170680607,
+   "n_repos": 5
+  },
+  {
+   "representative": "qwen-2.5",
+   "aliases": [
+    "qwen-3"
+   ],
+   "provider": "Alibaba Qwen",
+   "repo": "Qwen/Qwen2.5-0.5B",
+   "model_type": "BPE",
+   "vocab_size": 151665,
+   "bytes_per_token": 2.6325857842914915,
+   "fertility": 3.0821976025941815,
+   "strr": 0.02994719652556631,
+   "tokens": 551179,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.6382481916038165,
+   "n_repos": 60
+  },
+  {
+   "representative": "glm-4.5",
+   "aliases": [],
+   "provider": "Zhipu AI / Z.ai",
+   "repo": "zai-org/GLM-4.5",
+   "model_type": "BPE",
+   "vocab_size": 151365,
+   "bytes_per_token": 2.4823892097555635,
+   "fertility": 3.3733502552847137,
+   "strr": 0.13585105577840006,
+   "tokens": 584528,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.6588871705033805,
+   "n_repos": 1
+  },
+  {
+   "representative": "urdu-llama-bilal",
+   "aliases": [],
+   "provider": "community (BilalKhan1)",
+   "repo": "BilalKhan1/llama-urdu-tokenizer",
+   "model_type": "BPE",
+   "vocab_size": 129435,
+   "bytes_per_token": 2.3721203204185057,
+   "fertility": 2.833980520350005,
+   "strr": 0.26064656602763014,
+   "tokens": 611700,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.674039561876737,
+   "n_repos": 1
+  },
+  {
+   "representative": "minimax-m2",
+   "aliases": [
+    "minimax-m3"
+   ],
+   "provider": "MiniMax",
+   "repo": "MiniMaxAI/MiniMax-M2",
+   "model_type": "BPE",
+   "vocab_size": 200054,
+   "bytes_per_token": 2.26764267842715,
+   "fertility": 3.604003050608666,
+   "strr": 0.06987291071147879,
+   "tokens": 639883,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.688396159922986,
+   "n_repos": 2
+  },
+  {
+   "representative": "eurollm",
+   "aliases": [],
+   "provider": "UTTER (EU)",
+   "repo": "utter-project/EuroLLM-9B-Instruct-2512",
+   "model_type": "BPE",
+   "vocab_size": 128000,
+   "bytes_per_token": 2.146942326557245,
+   "fertility": 3.6777649051924386,
+   "strr": 0.061378944978430335,
+   "tokens": 675857,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.7049819710382521,
+   "n_repos": 1
+  },
+  {
+   "representative": "hunyuan",
+   "aliases": [],
+   "provider": "Tencent",
+   "repo": "tencent/Hunyuan-7B-Instruct",
+   "model_type": "BPE",
+   "vocab_size": 128166,
+   "bytes_per_token": 1.932331007297715,
+   "fertility": 4.245044216360345,
+   "strr": 0.00577519808580128,
+   "tokens": 750920,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.7344723805465296,
+   "n_repos": 1
+  },
+  {
+   "representative": "gpt-4",
+   "aliases": [
+    "olmo-2",
+    "phi-4"
+   ],
+   "provider": "OpenAI",
+   "repo": "Xenova/gpt-4",
+   "model_type": "BPE",
+   "vocab_size": 100263,
+   "bytes_per_token": 1.9192881698043578,
+   "fertility": 4.27475272022309,
+   "strr": 0.007050166211598135,
+   "tokens": 756023,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.736264637451506,
+   "n_repos": 20
+  },
+  {
+   "representative": "granite-4.2",
+   "aliases": [],
+   "provider": "IBM",
+   "repo": "ibm-granite/granite-4.2-8b",
+   "model_type": "BPE",
+   "vocab_size": 100352,
+   "bytes_per_token": 1.918772959406315,
+   "fertility": 4.274776007312146,
+   "strr": 0.00706180975612596,
+   "tokens": 756226,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.7363354341162562,
+   "n_repos": 1
+  },
+  {
+   "representative": "lfm2",
+   "aliases": [],
+   "provider": "Liquid AI",
+   "repo": "LiquidAI/LFM2.5-1.2B-Instruct",
+   "model_type": "BPE",
+   "vocab_size": 64402,
+   "bytes_per_token": 1.9002286519487799,
+   "fertility": 4.181191018169751,
+   "strr": 0.03274164721224435,
+   "tokens": 763606,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.738883665136209,
+   "n_repos": 63
+  },
+  {
+   "representative": "falcon-3",
+   "aliases": [],
+   "provider": "TII",
+   "repo": "tiiuae/Falcon3-7B-Instruct",
+   "model_type": "BPE",
+   "vocab_size": 131072,
+   "bytes_per_token": 1.8130482154727336,
+   "fertility": 4.50383945880805,
+   "strr": 0.010793565777293923,
+   "tokens": 800324,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.7508634003228692,
+   "n_repos": 1
+  },
+  {
+   "representative": "llama-2",
+   "aliases": [
+    "urdu-llama2-almanach"
+   ],
+   "provider": "Meta",
+   "repo": "NousResearch/Llama-2-7b-hf",
+   "model_type": "BPE",
+   "vocab_size": 32000,
+   "bytes_per_token": 1.713758958393371,
+   "fertility": 3.937066641827105,
+   "strr": 0.006473810757470789,
+   "tokens": 846692,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.7645070462458603,
+   "n_repos": 99
+  },
+  {
+   "representative": "grok-1",
+   "aliases": [],
+   "provider": "xAI",
+   "repo": "Xenova/grok-1-tokenizer",
+   "model_type": "BPE",
+   "vocab_size": 131072,
+   "bytes_per_token": 1.618097837965627,
+   "fertility": 4.822173966198791,
+   "strr": 0.0056121884624117276,
+   "tokens": 896748,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.7776521386164229,
+   "n_repos": 1
+  },
+  {
+   "representative": "claude-legacy",
+   "aliases": [],
+   "provider": "Anthropic",
+   "repo": "Xenova/claude-tokenizer",
+   "model_type": "BPE",
+   "vocab_size": 65000,
+   "bytes_per_token": 1.5541900255135377,
+   "fertility": 4.944966786789235,
+   "strr": 0.00631662290634515,
+   "tokens": 933622,
+   "g1_pass_docs": 467,
+   "released_uses_fewer_tokens": 0.7864339100835241,
+   "n_repos": 1
+  },
+  {
+   "representative": "gpt-2",
+   "aliases": [],
+   "provider": "OpenAI",
+   "repo": "openai-community/gpt2",
+   "model_type": "BPE",
+   "vocab_size": 50257,
+   "bytes_per_token": 1.3846658873488085,
+   "fertility": 5.7485751212384075,
+   "strr": 0.005128981364506983,
+   "tokens": 1047925,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.8097287496719708,
+   "n_repos": 122
+  },
+  {
+   "representative": "sarvam-1",
+   "aliases": [],
+   "provider": "Sarvam AI",
+   "repo": "sarvamai/sarvam-1",
+   "model_type": "BPE",
+   "vocab_size": 68096,
+   "bytes_per_token": 1.0054379986391144,
+   "fertility": 7.40664497086203,
+   "strr": 0.001304076987116418,
+   "tokens": 1443178,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.8618396344733636,
+   "n_repos": 2
+  },
+  {
+   "representative": "byt5",
+   "aliases": [],
+   "provider": "Google",
+   "repo": "google/byt5-small",
+   "model_type": "bytes",
+   "vocab_size": 384,
+   "bytes_per_token": 1.0,
+   "fertility": 7.45030244106911,
+   "strr": 0.000873265839586887,
+   "tokens": 1451026,
+   "g1_pass_docs": 491,
+   "released_uses_fewer_tokens": 0.8625868867959636,
+   "n_repos": 1
+  }
+ ],
+ "search_coverage": {
+  "ranking_snapshot_utc": "2026-09-27T12:49:42Z",
+  "pass1": {
+   "file": "_search.json",
+   "generated_utc": "2026-09-27T08:17:53Z",
+   "n_keywords": 43,
+   "keywords": [
+    "hindko",
+    "hindku",
+    "hinko",
+    "hazarewal",
+    "hazara",
+    "potohari",
+    "pothwari",
+    "pothohari",
+    "pahari",
+    "lahnda",
+    "lahanda",
+    "saraiki",
+    "seraiki",
+    "siraiki",
+    "multani",
+    "shahmukhi",
+    "punjabi",
+    "panjabi",
+    "pnb",
+    "urdu",
+    "pashto",
+    "pushto",
+    "pukhto",
+    "sindhi",
+    "kashmiri",
+    "balochi",
+    "baluchi",
+    "brahui",
+    "khowar",
+    "shina",
+    "gojri",
+    "pakistan",
+    "peshawar",
+    "nastaliq",
+    "nastaleeq",
+    "perso-arabic",
+    "persoarabic",
+    "indic",
+    "indo-aryan",
+    "indoaryan",
+    "south-asian",
+    "southasian",
+    "bharat"
+   ],
+   "n_lang_tags": 27,
+   "lang_tags": [
+    "hno",
+    "hnd",
+    "ur",
+    "urd",
+    "pnb",
+    "skr",
+    "lah",
+    "phr",
+    "ps",
+    "pus",
+    "pbt",
+    "pbu",
+    "pst",
+    "sd",
+    "snd",
+    "ks",
+    "kas",
+    "bal",
+    "bcc",
+    "bgp",
+    "khw",
+    "scl",
+    "gju",
+    "pa",
+    "pan",
+    "trw",
+    "bft"
+   ],
+   "n_authors": 91,
+   "n_named_repos": 216,
+   "n_named_missing": 15,
+   "n_queries": 198,
+   "n_repos": 10995,
+   "n_with_tokenizer_files": 7309
+  },
+  "pass2": {
+   "file": "_search2.json",
+   "generated_utc": "2026-09-27T08:46:42Z",
+   "n_keywords": 34,
+   "keywords": [
+    "markhor",
+    "pakmosaic",
+    "urdulm",
+    "alif-base",
+    "qehwa",
+    "zabaan",
+    "shaheen",
+    "baat",
+    "urdu-gpt",
+    "urdugpt",
+    "muse-glimmer",
+    "Nemotron-3.5",
+    "Atria-Dawn",
+    "Dots3",
+    "Ornith",
+    "Hy-MT2",
+    "Nex-N2.5",
+    "Bonsai-2",
+    "Ling-3.0",
+    "MiMo-V2.6",
+    "MiniCPM5",
+    "grok-2",
+    "grok-3",
+    "gpt-oss",
+    "o200k",
+    "BrahmicTokenizer",
+    "tokenizer-urdu",
+    "urdu-tokenizer",
+    "urdu_tokenizer",
+    "UrduTokenizer",
+    "punjabi-tokenizer",
+    "pashto-tokenizer",
+    "sindhi-tokenizer",
+    "saraiki-tokenizer"
+   ],
+   "n_authors": 33,
+   "authors": [
+    "ma1993",
+    "mahwizzzz",
+    "themohal",
+    "ProximaAI",
+    "orature",
+    "adiled",
+    "hadidev",
+    "NeerjaK",
+    "aariciah",
+    "Kashif786",
+    "aakashMeghwar01",
+    "zirak-ai",
+    "tasal9",
+    "junaid008",
+    "meta-models",
+    "DeepReinforce",
+    "nex-agi",
+    "PrismML",
+    "dots-studio",
+    "rednote-hilab",
+    "OpenGVLab",
+    "theschoolofai",
+    "bisma10",
+    "Xhaheen",
+    "mirfan899",
+    "Mavkif",
+    "traversaal-ai",
+    "large-traversaal",
+    "enstazao",
+    "UrduHack",
+    "salmanpopa",
+    "SLPG",
+    "abdulwaheed1"
+   ],
+   "named": [
+    "ProximaAI/PakMosaic",
+    "hadidev/gpt2-urdu-tokenizer-withgpt2",
+    "NeerjaK/Urdu_Model",
+    "meta-models/Muse-Glimmer-30B",
+    "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16",
+    "internlm/Atria-Dawn-Preview",
+    "tencent/Hy-MT2-30B-A3B",
+    "XiaomiMiMo/MiMo-V2.6-Flash",
+    "openbmb/MiniCPM5-2B",
+    "alvarobartt/grok-2-tokenizer"
+   ],
+   "n_queries": 67,
+   "n_repos": 3861,
+   "n_kept_after_filter": 634,
+   "filter": "regional keywords, regional/frontier authors, named repos, official orgs of the 2026 releases, and the 3 most-downloaded derivative repos per frontier keyword (the other derivatives are quantisations / fine-tunes)"
+  },
+  "pass3": {
+   "file": "_search3.json",
+   "generated_utc": "2026-09-27T12:54:36Z",
+   "authors": [
+    "DunbaaBERT",
+    "eshaaftab900",
+    "Humair332",
+    "hafeez007",
+    "shah-bakhsh",
+    "ijazulhaq",
+    "Hammad712",
+    "alphaedge-ai",
+    "Omarrran",
+    "HPLT",
+    "orature",
+    "ProximaAI",
+    "goldfish-models",
+    "zuhri025",
+    "SabahNawab",
+    "BilalKhan1",
+    "farahadeeba",
+    "urduhack",
+    "Imran1",
+    "Khurram123",
+    "shaikhsalman",
+    "mfayazkhan",
+    "large-traversaal",
+    "enstazao",
+    "nassimjp",
+    "tasal9",
+    "zirak-ai",
+    "APARSIN",
+    "Abdullah104",
+    "aakashMeghwar01",
+    "themohal",
+    "mahwizzzz"
+   ],
+   "n_queries": 32,
+   "n_repos": 2789,
+   "n_with_tokenizer_files": 2564,
+   "n_kept_after_filter": 508,
+   "filter": "alphaedge-ai / goldfish-models / HPLT: only repos whose name carries a Perso-Arabic-script language code (urd, snd, pnb, pbt, pus, skr, hno, hnd, kas, bal, bcc, ur, sd, ps, arab); all repos of the other authors",
+   "status": "STOPPED: started after the ranking snapshot; the probe log (logs/probe_pass3.log) ends with 458 repos still to probe; nothing from this pass was screened or measured"
+  },
+  "sweep_totals_at_snapshot": {
+   "n_repos_found": 14856,
+   "n_repos_found_pass1": 10995,
+   "n_repos_found_pass2": 3861,
+   "n_repos_kept_pass2": 634,
+   "pass2_filter": "regional keywords, regional/frontier authors, named repos, official orgs of the 2026 releases, and the 3 most-downloaded derivative repos per frontier keyword (the other derivatives are quantisations / fine-tunes)",
+   "n_repos_with_tokenizer_files": 7760,
+   "n_queries": 265,
+   "n_probed": 6895,
+   "skipped": {
+    "gated": 302,
+    "non_text_pipeline": 563
+   },
+   "n_probe_errors": 0,
+   "n_repos_all_files_already_known": 1765,
+   "n_new_file_signatures": 1759,
+   "n_downloaded": 880,
+   "n_reconstructed": 95,
+   "n_download_errors": 0,
+   "n_screened_ok": 957,
+   "n_screened": 970,
+   "n_behaviour_groups": 502,
+   "n_groups_equal_to_a_baseline": 17,
+   "n_new_behaviours_harnessed": 485
+  },
+  "download_log": {
+   "entries": 1026,
+   "after_snapshot": 47,
+   "after_snapshot_repos": [
+    "Ayshhaa/orpheus-urdu-tts",
+    "ByteDance-Seed/Seed-OSS-36B-Instruct",
+    "Jarbas/m2v-256-multilingual-e5-small",
+    "JuliaKreutzerCohere/tiny-aya-global-prompt-tasktype",
+    "Kashif786/distilbert-base-multilingual-cased-sindhi-extended",
+    "Kashif786/rembert-sindhi-extended",
+    "Kashif786/sindhi_distilbert_colab",
+    "KnutJaegersberg/nllb-moe-54b-4bit",
+    "LingoIITGN/Ansh-160k",
+    "MISHANM/Kashmiri_text_generation_Llama3_8B_instruct",
+    "MoritzLaurer/xlm-v-base-mnli-xnli",
+    "MoritzLaurer/xlm-v-base-mnli-xnli",
+    "QCRI/TechniqueRAG-FS-Ministral-8B",
+    "Qrzysztof/functiongemma-270m-it-prepaid-cards-v2",
+    "Qrzysztof/functiongemma-270m-it-prepaid-cards-v2-mlx",
+    "SalehAhmad/llama3.1-urdu-8b-qlora_split0",
+    "ServiceNow-AI/Apriel-5B-Instruct",
+    "Shinapri/pratchya-3k",
+    "Soupis/Contra-200",
+    "aiana94/NaSE",
+    "bharatgenai/Param2-17B-A2.4B-Thinking",
+    "casawolice/small100-onnx",
+    "facebook/mgenre-wiki",
+    "facebook/xglm-564M",
+    "facebook/xlm-v-base",
+    "google/madlad400-3b-mt",
+    "google/rembert",
+    "google/umt5-xxl",
+    "huggingworld/m2m100_418M",
+    "ibm-granite/granite-embedding-97m-multilingual-r2",
+    "inclusionAI/Ling-mini-2.0",
+    "inclusionAI/Ring-mini-2.0",
+    "minishlab/M2V_multilingual_output",
+    "minishlab/potion-multilingual-128M",
+    "mistralai/Ministral-3-14B-Reasoning-2512",
+    "nvidia/NVIDIA-Nemotron-Nano-9B-v2",
+    "nvidia/Nemotron-Mini-4B-Instruct",
+    "panagoa/nllb-200-1.3b-kbd-pretrain",
+    "panagoa/nllb-200-1.3b-kbd-v0.1",
+    "panagoa/nllb-200-1.3b-kbd-v0.2",
+    "rahular/varta-t5",
+    "setu4993/LEALLA-base",
+    "shah-bakhsh/balochi-english-translator",
+    "tangledgroup/tangled-alpha-0.5-core",
+    "tiiuae/Falcon-H1-7B-Base",
+    "tiiuae/Falcon-H1-7B-Instruct",
+    "tiiuae/Falcon-H1R-7B"
+   ],
+   "errors": [
+    {
+     "repo": "MoritzLaurer/xlm-v-base-mnli-xnli",
+     "error": "RemoteProtocolError: peer closed connection without sending complete message body (received 25494361 bytes, expected 61333645)"
+    }
+   ],
+   "note": "downloads logged after the snapshot were not screened or measured"
+  },
+  "probe_cache_lines": 7050,
+  "github_and_web": "GitHub search API + web search (refresh, before the snapshot): no Hindko tokenizer or Hindko LM found; Urdu tokenizer projects on GitHub publish code only and were not run (no third-party code).",
+  "not_collected": [
+   "gated repos (not logged into): 302 skipped at search, 65 unmatched after metadata check",
+   "closed tokenizers: Gemini, Claude 3+, OpenAI encodings newer than o200k_base / o200k_harmony",
+   "tiktoken rank files whose split regex exists only in repo Python code; xAI tokenizer.tok.json (Grok 2)",
+   "392 large files classified by range reads as identical / expected-identical to a measured tokenizer (not measured themselves)",
+   "13 downloaded tokenizers that could not be loaded"
+  ]
+ }
+}

eval/competitors_test_strict.csv

@@ -0,0 +1,530 @@
+rank_lossless,tokenizer,repo,provider,origin,algorithm,vocab_size,bytes_per_token,fertility,strr,tokens,g1_lossless_docs_of_491,lossless,unk_share,released_uses_fewer_tokens_pct,aliases,n_hub_repos
+1,hindko-tokenizer 1.0.0 (released),junaid008/hindko-tokenizer,this release,released,Unigram,32768,7.2773,1.135,0.8908,199390,491,True,0.0,0.0,,
+2,ProximaAI__urnova-95m,ProximaAI/urnova-95m,ProximaAI,new,BPE,50048,6.0613,1.3725,0.747,239392,491,True,0.0,16.71,,1
+3,DunbaaBERT__DunbaaBERT_52k_base,DunbaaBERT/DunbaaBERT_52k_base,DunbaaBERT,new,BPE,52010,5.9644,1.3952,0.7325,243283,491,True,0.0,18.04,,1
+4,orature__ALIF-Base-100M,orature/ALIF-Base-100M,orature,new,BPE,32000,5.9116,1.4047,0.7097,245454,491,True,0.0,18.77,,1
+5,DunbaaBERT__DunbaaBERT_32k_base,DunbaaBERT/DunbaaBERT_32k_base,DunbaaBERT,new,BPE,32010,5.7178,1.4562,0.6973,253775,491,True,0.0,21.43,,1
+6,roberta-urdu,urduhack/roberta-urdu-small,UrduHack,baseline,BPE,52000,5.7089,1.4585,0.7212,254167,491,True,0.0,21.55,,21
+7,ibm-research__ia-multilingual-original-script-roberta,ibm-research/ia-multilingual-original-script-roberta,ibm-research,new,BPE,110000,5.6678,1.4693,0.6933,256010,491,True,0.0,22.12,,1
+8,mahwizzzz__urdu-dummy,mahwizzzz/urdu-dummy,mahwizzzz,new,BPE,25000,5.5858,1.4911,0.6708,259769,491,True,0.0,23.24,,1
+9,hadidev__robertaurdu,hadidev/robertaurdu,hadidev,new,BPE,59458,5.5231,1.5011,0.7752,262721,491,True,0.0,24.11,,1
+10,bloom,bigscience/bloom-560m,BigScience,baseline,BPE,250680,5.1217,1.6331,0.5724,283310,491,True,0.0,29.62,,1
+11,santhosh__madlad400-3b-ct2,santhosh/madlad400-3b-ct2,santhosh,new,Unigram,256000,5.0802,1.5986,0.5622,285626,491,True,0.0,30.19,,8
+12,mlx-community__K2-Horizon-MoVA-36B-A4B-oQ4e,mlx-community/K2-Horizon-MoVA-36B-A4B-oQ4e,mlx-community,new,BPE,250624,5.0639,1.6512,0.5682,286545,491,True,0.0,30.42,,3
+13,MaLA-LM__mala-500-10b-v2,MaLA-LM/mala-500-10b-v2,MaLA-LM,new,BPE,260164,4.9128,1.6981,0.5556,295355,491,True,0.0,32.49,,1
+14,gemma-3,unsloth/gemma-3-1b-it,Google,baseline,BPE,262145,4.6104,1.8072,0.4866,314731,491,True,0.0,36.65,gemma-4;sarvam-30b,3
+15,alphaedge-ai__gemma-3-270m-it-snd-32768,alphaedge-ai/gemma-3-270m-it-snd-32768,alphaedge-ai,new,BPE,32768,4.5319,1.8389,0.4702,320179,491,True,0.0,37.73,,3
+16,Palash123__indic_200K_all_tokenizer,Palash123/indic_200K_all_tokenizer,Palash123,new,BPE,200000,4.5285,1.8395,0.4848,320423,491,True,0.0,37.77,,1
+17,falcon-h1,tiiuae/Falcon-H1-34B-Instruct,TII,baseline,BPE,261120,4.5065,1.8456,0.456,321986,491,True,0.0,38.07,,2
+18,soketlabs__Indic-Llama-2-7b,soketlabs/Indic-Llama-2-7b,soketlabs,new,BPE,67991,4.3937,2.2381,0.1983,330250,491,True,0.0,39.62,,2
+19,gpt-4o,Xenova/gpt-4o,OpenAI,baseline,BPE,200000,4.3376,1.9305,0.4149,334524,491,True,0.0,40.4,gpt-oss;phi-4-mini,3
+20,Palash123__indic_200K_tokenizer,Palash123/indic_200K_tokenizer,Palash123,new,BPE,199598,4.2724,1.9517,0.4143,339626,491,True,0.0,41.29,,1
+21,Palash123__indic_128K_tokenizer,Palash123/indic_128K_tokenizer,Palash123,new,BPE,127694,4.2194,1.9748,0.4142,343895,491,True,0.0,42.02,,1
+22,alphaedge-ai__gemma-3-4b-it-snd-16384,alphaedge-ai/gemma-3-4b-it-snd-16384,alphaedge-ai,new,BPE,16384,4.2136,1.9797,0.3898,344369,491,True,0.0,42.1,,3
+23,Palash123__indic_128K_all_tokenizer,Palash123/indic_128K_all_tokenizer,Palash123,new,BPE,128000,4.1765,1.9967,0.3967,347423,491,True,0.0,42.61,,1
+24,agemagician__mlong-t5-tglobal-base,agemagician/mlong-t5-tglobal-base,agemagician,new,Unigram,256300,4.1533,1.8837,0.4193,349366,491,True,0.0,42.93,,3
+25,tiny-aya,mlx-community/tiny-aya-global-8bit-mlx,Cohere,baseline,BPE,261010,4.1062,2.0323,0.3837,353372,491,True,0.0,43.58,,1
+26,alphaedge-ai__gemma-3-1b-it-urd-32768,alphaedge-ai/gemma-3-1b-it-urd-32768,alphaedge-ai,new,BPE,32768,4.0633,2.0538,0.371,357106,491,True,0.0,44.17,,3
+27,apertus,swiss-ai/Apertus-8B-Instruct-2509,Swiss AI,baseline,BPE,131072,3.901,2.1491,0.3041,371961,491,True,0.0,46.39,mistral-nemo;mistral-small-4;nemotron-3;sarvam-m,14
+28,mistralai__Mistral-Small-3.1-24B-Instruct-2503,mistralai/Mistral-Small-3.1-24B-Instruct-2503,mistralai,new,BPE,131072,3.8883,2.1562,0.3039,373182,491,True,0.0,46.57,,16
+29,m3hrdadfi__zabanshenas-roberta-base-mix,m3hrdadfi/zabanshenas-roberta-base-mix,m3hrdadfi,new,BPE,32000,3.7876,2.2087,0.3767,383096,491,True,0.0,47.95,,1
+30,gemma-2,unsloth/gemma-2-2b-it,Google,baseline,BPE,256000,3.7866,2.2056,0.3037,383200,491,True,0.0,47.97,,1
+31,tiiuae__Falcon-H1-1.5B-Base,tiiuae/Falcon-H1-1.5B-Base,tiiuae,new,BPE,65536,3.7543,2.2211,0.2905,386495,491,True,0.0,48.41,,5
+32,llama-4,unsloth/Llama-4-Scout-17B-16E-Instruct,Meta,baseline,BPE,201135,3.7269,2.2505,0.2749,389341,491,True,0.0,48.79,,1
+33,command-a-plus,CohereLabs/command-a-plus-05-2026-bf16,Cohere,baseline,BPE,255032,3.685,2.2739,0.2612,393763,491,True,0.0,49.36,,1
+34,ai-forever__mGPT,ai-forever/mGPT,ai-forever,new,BPE,100000,3.6795,2.2737,0.2672,394357,491,True,0.0,49.44,,2
+35,qwen-3.5,Qwen/Qwen3.5-0.8B,Alibaba Qwen,baseline,BPE,248070,3.5982,2.3227,0.2575,403264,491,True,0.0,50.56,qwen-3.8,88
+36,alphaedge-ai__Qwen3.5-2B-snd-32768,alphaedge-ai/Qwen3.5-2B-snd-32768,alphaedge-ai,new,BPE,32768,3.5898,2.3282,0.2546,404208,491,True,0.0,50.67,,3
+37,kimi-k2,moonshotai/Kimi-K2-Instruct,Moonshot AI,baseline,BPE,163840,3.5296,2.3753,0.2778,411099,491,True,0.0,51.5,,1
+38,ernie-4.5,baidu/ERNIE-4.5-0.3B-PT,Baidu,baseline,BPE,101304,3.5275,2.3689,0.2725,411344,491,True,0.0,51.53,,10
+39,pashto-lfm2.5,nassimjp/LFM2.5-2.6B-Pashto-Zi,community (nassimjp),baseline,BPE,125039,3.5093,2.3861,0.2251,413486,491,True,0.0,51.78,,1
+40,LiquidAI__LFM2.5-VL-3B,LiquidAI/LFM2.5-VL-3B,LiquidAI,new,BPE,125017,3.5088,2.3864,0.2251,413541,491,True,0.0,51.78,,11
+41,alphaedge-ai__Qwen3.5-0.8B-urd-32768,alphaedge-ai/Qwen3.5-0.8B-urd-32768,alphaedge-ai,new,BPE,32768,3.505,2.3834,0.2384,413982,491,True,0.0,51.84,,3
+42,alphaedge-ai__Qwen3.5-2B-snd-16384,alphaedge-ai/Qwen3.5-2B-snd-16384,alphaedge-ai,new,BPE,16384,3.5041,2.3857,0.2501,414093,491,True,0.0,51.85,,3
+43,deepseek-r1,deepseek-ai/DeepSeek-R1,DeepSeek,baseline,BPE,128815,3.3722,2.4755,0.2325,430288,491,True,0.0,53.66,deepseek-v3;deepseek-v4;deepseek-v4.1,4
+44,cnmoro__tangled-llama-33m-32k-instruct-v0.1-fix,cnmoro/tangled-llama-33m-32k-instruct-v0.1-fix,cnmoro,new,BPE,32768,3.3396,2.508,0.2683,434494,491,True,0.0,54.11,,1
+45,alphaedge-ai__gemma-3-1b-it-urd-16384,alphaedge-ai/gemma-3-1b-it-urd-16384,alphaedge-ai,new,BPE,16384,3.3357,2.5069,0.2233,434993,491,True,0.0,54.16,,3
+46,alphaedge-ai__Qwen3.5-0.8B-urd-16384,alphaedge-ai/Qwen3.5-0.8B-urd-16384,alphaedge-ai,new,BPE,16384,3.3084,2.5266,0.1979,438591,491,True,0.0,54.54,,3
+47,mlx-community__Xing4.0-29B-A4B-OptiQ-4bit,mlx-community/Xing4.0-29B-A4B-OptiQ-4bit,mlx-community,new,BPE,131072,3.1277,2.6647,0.2013,463929,491,True,0.0,57.02,,1
+48,glm-5,zai-org/GLM-5,Zhipu AI / Z.ai,baseline,BPE,154856,3.0254,2.7626,0.1366,479618,491,True,0.0,58.43,,1
+49,APARSIN__PashtoEnglishBilingualSentimentCorpus_roberta-fa-zwnj-base,APARSIN/PashtoEnglishBilingualSentimentCorpus_roberta-fa-zwnj-base,APARSIN,new,BPE,42000,2.9372,2.8479,0.3367,494021,491,True,0.0,59.64,,2
+50,humain-ai__ALLaM-7B-Instruct-preview,humain-ai/ALLaM-7B-Instruct-preview,humain-ai,new,BPE,64000,2.8664,2.8571,0.1544,506226,491,True,0.0,60.61,,1
+51,internlm__internlm3-8b-instruct,internlm/internlm3-8b-instruct,internlm,new,BPE,128133,2.652,3.0442,0.1204,547154,491,True,0.0,63.56,,5
+52,command-r,adamo1139/aya-expanse-8b-ungated,Cohere,baseline,BPE,255029,2.6451,3.1073,0.217,548567,491,True,0.0,63.65,command-r7b,2
+53,alif-1.0,large-traversaal/Alif-1.0-8B-Instruct,Traversaal.ai,baseline,BPE,128256,2.6412,3.1189,0.178,549377,491,True,0.0,63.71,llama-3;qalb-1.0;urdu-llama3-almanach;urdu-llama3.2-custom,5
+54,sbintuitions__kana-whisper,sbintuitions/kana-whisper,sbintuitions,new,BPE,51866,2.6403,3.1708,0.0781,549576,491,True,0.0,63.72,,64
+55,kavyamanohar__AI4B-Indicwhisper-ml,kavyamanohar/AI4B-Indicwhisper-ml,kavyamanohar,new,BPE,50364,2.6402,3.1708,0.0781,549580,491,True,0.0,63.72,,21
+56,BidirLM__BidirLM-Omni-2.5B-Embedding,BidirLM/BidirLM-Omni-2.5B-Embedding,BidirLM,new,BPE,151705,2.6341,3.0803,0.0299,550854,491,True,0.0,63.8,,3
+57,qwen-2.5,Qwen/Qwen2.5-0.5B,Alibaba Qwen,baseline,BPE,151665,2.6326,3.0822,0.0299,551179,491,True,0.0,63.82,qwen-3,60
+58,alphaedge-ai__Qwen3-Embedding-snd-32768,alphaedge-ai/Qwen3-Embedding-snd-32768,alphaedge-ai,new,BPE,32768,2.6324,3.0825,0.0299,551227,491,True,0.0,63.83,,1
+59,alphaedge-ai__Qwen3-0.6B-snd-32768,alphaedge-ai/Qwen3-0.6B-snd-32768,alphaedge-ai,new,BPE,32768,2.6324,3.0825,0.0299,551228,491,True,0.0,63.83,,2
+60,alphaedge-ai__Qwen3-Embedding-snd-16384,alphaedge-ai/Qwen3-Embedding-snd-16384,alphaedge-ai,new,BPE,16384,2.6322,3.0826,0.0298,551254,491,True,0.0,63.83,,1
+61,alphaedge-ai__Qwen3-0.6B-snd-16384,alphaedge-ai/Qwen3-0.6B-snd-16384,alphaedge-ai,new,BPE,16384,2.6321,3.0828,0.0297,551286,491,True,0.0,63.83,,2
+62,jinaai__jina-code-embeddings-0.5b,jinaai/jina-code-embeddings-0.5b,jinaai,new,BPE,151644,2.6316,3.0822,0.03,551384,491,True,0.0,63.84,,2
+63,alphaedge-ai__Qwen3-Embedding-urd-32768,alphaedge-ai/Qwen3-Embedding-urd-32768,alphaedge-ai,new,BPE,32768,2.6274,3.0885,0.0292,552258,491,True,0.0,63.9,,1
+64,alphaedge-ai__Qwen3-0.6B-urd-32768,alphaedge-ai/Qwen3-0.6B-urd-32768,alphaedge-ai,new,BPE,32768,2.6274,3.0885,0.029,552268,491,True,0.0,63.9,,2
+65,alphaedge-ai__Qwen3-Embedding-urd-16384,alphaedge-ai/Qwen3-Embedding-urd-16384,alphaedge-ai,new,BPE,16384,2.603,3.1186,0.0257,557436,491,True,0.0,64.23,,1
+66,alphaedge-ai__Qwen3-1.7B-urd-16384,alphaedge-ai/Qwen3-1.7B-urd-16384,alphaedge-ai,new,BPE,16384,2.6026,3.1191,0.0249,557521,491,True,0.0,64.24,,2
+67,RudranshAgnihotri__INDIC-TAMIL,RudranshAgnihotri/INDIC-TAMIL,RudranshAgnihotri,new,BPE,100000,2.5647,3.2726,0.0285,565771,491,True,0.0,64.76,,2
+68,glm-4.5,zai-org/GLM-4.5,Zhipu AI / Z.ai,baseline,BPE,151365,2.4824,3.3734,0.1359,584528,491,True,0.0,65.89,,1
+69,alphaedge-ai__Qwen3-1.7B-pan-32768,alphaedge-ai/Qwen3-1.7B-pan-32768,alphaedge-ai,new,BPE,32768,2.4616,3.3051,0.0142,589462,491,True,0.0,66.17,,2
+70,alphaedge-ai__Qwen3-Embedding-pan-32768,alphaedge-ai/Qwen3-Embedding-pan-32768,alphaedge-ai,new,BPE,32768,2.4602,3.307,0.0141,589792,491,True,0.0,66.19,,1
+71,DavidCBaines__ebible_m2m-ms8-ie-shareable,DavidCBaines/ebible_m2m-ms8-ie-shareable,DavidCBaines,new,BPE,32000,2.4302,3.3918,0.1538,597078,491,True,0.0,66.61,,1
+72,Palash123__indic_dummy_tokenizer,Palash123/indic_dummy_tokenizer,Palash123,new,BPE,52000,2.4109,3.4249,0.0512,601852,491,True,0.0,66.87,,1
+73,DavidCBaines__ebible_m2m-ie-big-shareable,DavidCBaines/ebible_m2m-ie-big-shareable,DavidCBaines,new,BPE,32000,2.386,3.4551,0.1385,608131,491,True,0.0,67.21,,1
+74,urdu-llama-bilal,BilalKhan1/llama-urdu-tokenizer,community (BilalKhan1),baseline,BPE,129435,2.3721,2.834,0.2606,611700,491,True,0.0,67.4,,1
+75,alphaedge-ai__Qwen3.5-0.8B-pan-32768,alphaedge-ai/Qwen3.5-0.8B-pan-32768,alphaedge-ai,new,BPE,32768,2.3392,3.151,0.1167,620312,491,True,0.0,67.86,,3
+76,ByteDance-Seed__Seed-X-PPO-7B,ByteDance-Seed/Seed-X-PPO-7B,ByteDance-Seed,new,BPE,65269,2.2802,3.4139,0.013,636348,491,True,0.0,68.67,,3
+77,minimax-m2,MiniMaxAI/MiniMax-M2,MiniMax,baseline,BPE,200054,2.2676,3.604,0.0699,639883,491,True,0.0,68.84,minimax-m3,2
+78,NawazHanzla__Shaheen-Arabic-Instruct,NawazHanzla/Shaheen-Arabic-Instruct,NawazHanzla,new,BPE,84992,2.2472,3.594,0.1384,645696,491,True,0.0,69.12,,1
+79,alphaedge-ai__Qwen3-Embedding-pan-16384,alphaedge-ai/Qwen3-Embedding-pan-16384,alphaedge-ai,new,BPE,16384,2.2059,3.6965,0.0076,657795,491,True,0.0,69.69,,1
+80,alphaedge-ai__Qwen3-0.6B-pan-16384,alphaedge-ai/Qwen3-0.6B-pan-16384,alphaedge-ai,new,BPE,16384,2.1822,3.7381,0.007,664941,491,True,0.0,70.01,,2
+81,eurollm,utter-project/EuroLLM-9B-Instruct-2512,UTTER (EU),baseline,BPE,128000,2.1469,3.6778,0.0614,675857,491,True,0.0,70.5,,1
+82,alphaedge-ai__gemma-3-270m-it-pan-32768,alphaedge-ai/gemma-3-270m-it-pan-32768,alphaedge-ai,new,BPE,32768,2.0771,3.8212,0.1748,698583,491,True,0.0,71.46,,3
+83,hunyuan,tencent/Hunyuan-7B-Instruct,Tencent,baseline,BPE,128166,1.9323,4.245,0.0058,750920,491,True,0.0,73.45,,1
+84,gpt-4,Xenova/gpt-4,OpenAI,baseline,BPE,100263,1.9193,4.2748,0.0071,756023,491,True,0.0,73.63,olmo-2;phi-4,20
+85,granite-4.2,ibm-granite/granite-4.2-8b,IBM,baseline,BPE,100352,1.9188,4.2748,0.0071,756226,491,True,0.0,73.63,,1
+86,hgnoi__PnBJ7Z4wxY29v1kZ,hgnoi/PnBJ7Z4wxY29v1kZ,hgnoi,new,BPE,100289,1.9178,4.2782,0.0058,756612,491,True,0.0,73.65,,1
+87,akhooli__personachat-arabic,akhooli/personachat-arabic,akhooli,new,BPE,50262,1.9012,4.2302,0.1367,763207,491,True,0.0,73.87,,1
+88,lfm2,LiquidAI/LFM2.5-1.2B-Instruct,Liquid AI,baseline,BPE,64402,1.9002,4.1812,0.0327,763606,491,True,0.0,73.89,,63
+89,QCRI__Fanar-2-Diwan,QCRI/Fanar-2-Diwan,QCRI,new,BPE,65868,1.8649,4.2473,0.1449,778073,491,True,0.0,74.37,,1
+90,ibm-granite__granite-embedding-small-english-r2,ibm-granite/granite-embedding-small-english-r2,ibm-granite,new,BPE,50368,1.8472,4.3118,0.0101,785545,491,True,0.0,74.62,,137
+91,ibm-granite__granite-3.3-8b-instruct,ibm-granite/granite-3.3-8b-instruct,ibm-granite,new,BPE,49159,1.8456,4.3506,0.008,786204,491,True,0.0,74.64,,34
+92,ibm-research__ia-multilingual-transliterated-roberta,ibm-research/ia-multilingual-transliterated-roberta,ibm-research,new,BPE,110000,1.844,4.3741,0.0093,786870,491,True,0.0,74.66,,1
+93,mlx-community__XVERSE-7B-Chat-4bit,mlx-community/XVERSE-7B-Chat-4bit,mlx-community,new,BPE,100534,1.8436,3.6953,0.0071,787075,491,True,0.0,74.67,,1
+94,kdcyberdude__llama-punjabi-tokenizer,kdcyberdude/llama-punjabi-tokenizer,kdcyberdude,new,BPE,46449,1.8279,3.6879,0.0088,793814,491,True,0.0,74.88,,1
+95,falcon-3,tiiuae/Falcon3-7B-Instruct,TII,baseline,BPE,131072,1.813,4.5038,0.0108,800324,491,True,0.0,75.09,,1
+96,mlx-community__XORTRON.CriminalComputing.LARGE.2026.3-mlx-8Bit,mlx-community/XORTRON.CriminalComputing.LARGE.2026.3-mlx-8Bit,mlx-community,new,BPE,32768,1.7915,3.7819,0.0072,809955,491,True,0.0,75.38,,2
+97,upstage__SOLAR-10.7B-Instruct-v1.0,upstage/SOLAR-10.7B-Instruct-v1.0,upstage,new,BPE,32000,1.7906,3.7819,0.0072,810373,491,True,0.0,75.4,,92
+98,HaniaEmaan__urdu-trocr-model,HaniaEmaan/urdu-trocr-model,HaniaEmaan,new,BPE,50335,1.7865,3.7386,0.0088,812213,491,True,0.0,75.45,,1
+99,nassimjp__Ghanam-7B-Base-Pashto-v0.1,nassimjp/Ghanam-7B-Base-Pashto-v0.1,nassimjp,new,BPE,32010,1.7585,3.811,0.0072,825131,491,True,0.0,75.84,,1
+100,deepseek-ai__Janus-Pro-7B,deepseek-ai/Janus-Pro-7B,deepseek-ai,new,BPE,100603,1.7536,4.6161,0.0052,827452,491,True,0.0,75.9,,3
+101,BharatVLM__AssameseGPT2,BharatVLM/AssameseGPT2,BharatVLM,new,BPE,50000,1.7288,4.5464,0.0108,839336,491,True,0.0,76.24,,1
+102,Khurram123__urdu-poetry-trocr,Khurram123/urdu-poetry-trocr,Khurram123,new,BPE,50309,1.7279,3.8918,0.0065,839776,491,True,0.0,76.26,,1
+103,OpenGVLab__InternVL-14B-224px,OpenGVLab/InternVL-14B-224px,OpenGVLab,new,BPE,49954,1.7154,3.9347,0.0068,845879,491,True,0.0,76.43,,3
+104,llama-2,NousResearch/Llama-2-7b-hf,Meta,baseline,BPE,32000,1.7138,3.9371,0.0065,846692,491,True,0.0,76.45,urdu-llama2-almanach,99
+105,stepfun-ai__Step-Audio-EditX,stepfun-ai/Step-Audio-EditX,stepfun-ai,new,Unigram,74752,1.677,4.0415,0.0093,865243,491,True,0.0,76.96,,3
+106,grok-1,Xenova/grok-1-tokenizer,xAI,baseline,BPE,131072,1.6181,4.8222,0.0056,896748,491,True,0.0,77.77,,1
+107,alphaedge-ai__Qwen3.5-2B-pan-16384,alphaedge-ai/Qwen3.5-2B-pan-16384,alphaedge-ai,new,BPE,16384,1.6107,4.6927,0.0162,900889,491,True,0.0,77.87,,3
+108,HuggingFaceTB__SmolVLM2-500M-Video-Instruct,HuggingFaceTB/SmolVLM2-500M-Video-Instruct,HuggingFaceTB,new,BPE,49280,1.543,5.2377,0.0041,940418,491,True,0.0,78.8,,16
+109,openbmb__MiniCPM-Embedding-Light,openbmb/MiniCPM-Embedding-Light,openbmb,new,BPE,73440,1.5402,4.6454,0.0051,942125,491,True,0.0,78.84,,1
+110,openbmb__MiniCPM-V-4,openbmb/MiniCPM-V-4,openbmb,new,BPE,73448,1.5395,4.6454,0.0051,942530,491,True,0.0,78.85,,13
+111,sbintuitions__sarashina2.2-0.5b-instruct-v0.1,sbintuitions/sarashina2.2-0.5b-instruct-v0.1,sbintuitions,new,Unigram,102400,1.5357,4.5103,0.0083,944881,491,True,0.0,78.9,,6
+112,MBZUAI__GeoPixel-7B,MBZUAI/GeoPixel-7B,MBZUAI,new,BPE,92544,1.4921,4.6682,0.0062,972496,491,True,0.0,79.5,,4
+113,tiiuae__falcon-mamba-7b,tiiuae/falcon-mamba-7b,tiiuae,new,BPE,65024,1.4395,5.3458,0.0057,1008029,491,True,0.0,80.22,,2
+114,tiiuae__dense-500m-arch1__iter_0002000,tiiuae/dense-500m-arch1/iter_0002000,tiiuae,new,BPE,65536,1.4317,5.3215,0.0041,1013469,491,True,0.0,80.33,,51
+115,AinaShakeel__deepseek-urdu-5.7b-lora,AinaShakeel/deepseek-urdu-5.7b-lora,AinaShakeel,new,BPE,32022,1.4097,5.5409,0.0039,1029319,491,True,0.0,80.63,,1
+116,gpt-2,openai-community/gpt2,OpenAI,baseline,BPE,50257,1.3847,5.7486,0.0051,1047925,491,True,0.0,80.97,,122
+117,OpenGVLab__InternVL-Chat-V1-2,OpenGVLab/InternVL-Chat-V1-2,OpenGVLab,new,BPE,64000,1.3832,5.1176,0.0047,1049005,491,True,0.0,80.99,,4
+118,OpenGVLab__InternVL2-40B-AWQ,OpenGVLab/InternVL2-40B-AWQ,OpenGVLab,new,BPE,64007,1.3826,5.1175,0.0047,1049466,491,True,0.0,81.0,,1
+119,HPLT__NorOLMo-13B,HPLT/NorOLMo-13B,HPLT,new,BPE,51300,1.3321,5.9878,0.0032,1089238,491,True,0.0,81.69,,1
+120,HPLT__FinOLMo-13B,HPLT/FinOLMo-13B,HPLT,new,BPE,51200,1.3116,6.2125,0.0032,1106306,491,True,0.0,81.98,,1
+121,formulae__reBERT-multilingual,formulae/reBERT-multilingual,formulae,new,BPE,30000,1.212,6.6147,0.0031,1197222,491,True,0.0,83.35,,1
+122,tiiuae__Falcon-H1-0.5B-Base,tiiuae/Falcon-H1-0.5B-Base,tiiuae,new,BPE,32768,1.1458,6.7912,0.0019,1266440,491,True,0.0,84.26,,19
+123,BSC-LT__salamandraTA-2B-instruct-GGUF,BSC-LT/salamandraTA-2B-instruct-GGUF,BSC-LT,new,BPE,256000,1.0861,6.9116,0.0019,1335988,491,True,0.0,85.08,,1
+124,Ketak-ZoomRx__Indication_OLL_v2,Ketak-ZoomRx/Indication_OLL_v2,Ketak-ZoomRx,new,BPE,32000,1.0855,6.9157,0.0019,1336721,491,True,0.0,85.08,,1
+125,alphaedge-ai__gemma-3-4b-it-pan-16384,alphaedge-ai/gemma-3-4b-it-pan-16384,alphaedge-ai,new,BPE,16384,1.0645,7.0306,0.0107,1363153,491,True,0.0,85.37,,2
+126,mlx-community__lil-bard,mlx-community/lil-bard,mlx-community,new,BPE,8192,1.0068,7.4037,0.0016,1441223,491,True,0.0,86.17,,1
+127,sarvam-1,sarvamai/sarvam-1,Sarvam AI,baseline,BPE,68096,1.0054,7.4066,0.0013,1443178,491,True,0.0,86.18,,2
+128,openai__circuit-sparsity,openai/circuit-sparsity,openai,new,BPE,2047,1.0009,7.4458,0.002,1449670,491,True,0.0,86.25,,1
+129,zahidazam714__khowar_bpe_tokenizer,zahidazam714/khowar_bpe_tokenizer,zahidazam714,new,BPE,1500,1.0005,7.4483,0.0011,1450237,491,True,0.0,86.25,,1
+130,flax-community__mr-indicnlp-classifier,flax-community/mr-indicnlp-classifier,flax-community,new,BPE,50265,1.0,7.4503,0.0009,1451024,491,True,0.0,86.26,,1
+131,byt5,google/byt5-small,Google,baseline,bytes,384,1.0,7.4503,0.0009,1451026,491,True,0.0,86.26,,1
+,Ayesha-fatima__urdu-articulation-screener,Ayesha-fatima/urdu-articulation-screener,Ayesha-fatima,new,WordLevel,392,2955.2464,1.0,1.0,491,0,False,1.0,-40508.96,,1
+,internlm__Intern-S1-Pro,internlm/Intern-S1-Pro,internlm,new,BPE,1024,2118.2861,1.0007,0.9995,685,0,False,0.8496,-29008.03,,2
+,internlm__Intern-S1,internlm/Intern-S1,internlm,new,BPE,512,2102.9362,1.0007,0.9995,690,0,False,0.8333,-28797.1,,4
+,Omarrran__Kashmiri_Tokenizers__kashmiri_word_tokenizer,Omarrran/Kashmiri_Tokenizers/kashmiri_word_tokenizer,Omarrran,new,WordLevel,110995,7.801,1.0829,0.9308,186004,0,False,0.3922,-7.2,,2
+,ijazulhaq__bert-base-pashto-c,ijazulhaq/bert-base-pashto-c,ijazulhaq,new,WordPiece,703,7.6984,1.0973,0.9277,188483,0,False,0.9789,-5.79,,1
+,Kashif786__sindhi-universal-ultimate,Kashif786/sindhi-universal-ultimate,Kashif786,new,WordPiece,5,7.6977,1.0974,0.9278,188502,0,False,1.0,-5.78,,1
+,mirfan899__da-sentiment,mirfan899/da-sentiment,mirfan899,new,WordPiece,31748,7.691,1.0984,0.9272,188665,0,False,0.9877,-5.68,,1
+,pnbonillo__clasificador-muchocine,pnbonillo/clasificador-muchocine,pnbonillo,new,WordPiece,31002,7.6869,1.0989,0.9268,188765,0,False,0.9874,-5.63,,1
+,AnanthZeke__TaNER-1k-indic_glue,AnanthZeke/TaNER-1k-indic_glue,AnanthZeke,new,WordPiece,1023,7.6659,1.102,0.9257,189284,0,False,0.9854,-5.34,,4
+,tcepi__HelBERT-uncased-fs-lsg-13-indicios,tcepi/HelBERT-uncased-fs-lsg-13-indicios,tcepi,new,WordPiece,33013,7.6613,1.1026,0.9252,189397,0,False,0.9752,-5.28,,1
+,almanach__Biomed-Enriched-classifier,almanach/Biomed-Enriched-classifier,almanach,new,WordPiece,30522,7.6438,1.1052,0.9216,189831,0,False,0.9539,-5.04,,1
+,Ibrahimbaroud__personal-information-detection-arabic,Ibrahimbaroud/personal-information-detection-arabic,Ibrahimbaroud,new,WordPiece,64000,7.3789,1.1448,0.888,196645,0,False,0.745,-1.4,,1
+,eshaaftab900__urdu-bert-base-2,eshaaftab900/urdu-bert-base-2,eshaaftab900,new,WordPiece,50000,6.6916,1.2624,0.7967,216844,0,False,0.0005,8.05,,3
+,openbmb__chattts_tokenizer,openbmb/chattts_tokenizer,openbmb,new,WordPiece,21178,6.5855,1.2828,0.8266,220337,0,False,0.7587,9.51,,1
+,goldfish-models__pnb_arab_full,goldfish-models/pnb_arab_full,goldfish-models,new,Unigram,51200,6.45,1.3078,0.7897,224967,166,False,0.0001,11.37,,2
+,mahwizzzz__avey-b-ur,mahwizzzz/avey-b-ur,mahwizzzz,new,WordPiece,32000,6.3002,1.3408,0.7466,230315,0,False,0.0002,13.43,,1
+,themohal__trocr-small-saraiki,themohal/trocr-small-saraiki,themohal,new,Unigram,64202,6.2964,1.3096,0.7662,230454,169,False,0.0011,13.48,,1
+,eshaaftab900__urdu-bert-base,eshaaftab900/urdu-bert-base,eshaaftab900,new,WordPiece,60000,6.255,1.3505,0.73,231978,0,False,0.0336,14.05,,1
+,urdu-bert-64k,farahadeeba/urdu-bert-64k,community (farahadeeba),baseline,WordPiece,64000,6.2213,1.3578,0.7356,233236,0,False,0.0,14.51,,1
+,hafeez007__balochi-tokenizers,hafeez007/balochi-tokenizers,hafeez007,new,BPE,64000,6.1881,1.3647,0.719,234488,175,False,0.0,14.97,,1
+,goldfish-models__pnb_arab_10mb,goldfish-models/pnb_arab_10mb,goldfish-models,new,Unigram,51200,6.0993,1.3764,0.7257,237900,170,False,0.0,16.19,,1
+,goldfish-models__urd_arab_100mb,goldfish-models/urd_arab_100mb,goldfish-models,new,Unigram,51200,6.0551,1.3871,0.7342,239636,165,False,0.0001,16.79,,2
+,themohal__SaraikiRoBERTa-MTL-v1,themohal/SaraikiRoBERTa-MTL-v1,themohal,new,BPE,103345,5.9984,1.4033,0.8169,241904,41,False,0.0,17.57,,1
+,ijazulhaq__bert-base-pashto-v1,ijazulhaq/bert-base-pashto-v1,ijazulhaq,new,WordPiece,30000,5.9888,1.4106,0.6857,242289,0,False,0.2652,17.71,,1
+,goldfish-models__pnb_arab_5mb,goldfish-models/pnb_arab_5mb,goldfish-models,new,Unigram,40960,5.9504,1.4122,0.6952,243855,169,False,0.0,18.23,,1
+,Humair332__omnivoice-urdu__checkpoint-1000,Humair332/omnivoice-urdu/checkpoint-1000,Humair332,new,BPE,50007,5.8731,1.4172,0.7216,247065,467,False,0.0,19.3,,34
+,goldfish-models__urd_arab_10mb,goldfish-models/urd_arab_10mb,goldfish-models,new,Unigram,51200,5.8576,1.4337,0.683,247715,165,False,0.0001,19.51,,1
+,shah-bakhsh__balochi-tokenizer,shah-bakhsh/balochi-tokenizer,shah-bakhsh,new,BPE,48000,5.8058,1.455,0.6494,249926,0,False,0.014,20.22,,1
+,HPLT__hplt_bert_base_ur,HPLT/hplt_bert_base_ur,HPLT,new,WordPiece,32768,5.7545,1.4469,0.6811,252154,467,False,0.0,20.93,,3
+,mahwizzzz__UrduNER,mahwizzzz/UrduNER,mahwizzzz,new,BPE,52000,5.7099,1.4583,0.7214,254125,0,False,0.0,21.54,,2
+,goldfish-models__urd_arab_5mb,goldfish-models/urd_arab_5mb,goldfish-models,new,Unigram,38912,5.6874,1.4757,0.6576,255132,165,False,0.0001,21.85,,1
+,urdu-gpt2-20k,aariciah/gpt2-urdu-20k-lc,community (aariciah),baseline,BPE,20000,5.6326,1.4784,0.6588,257611,166,False,0.0141,22.6,,2
+,aariciah__gpt2-urdu-configC-20k__gpt2-urdu-configC-20k-GPT2TokenizerFast,aariciah/gpt2-urdu-configC-20k/gpt2-urdu-configC-20k-GPT2TokenizerFast,aariciah,new,BPE,20000,5.6317,1.4786,0.6588,257652,173,False,0.0141,22.61,,3
+,aariciah__gpt2-urdu-dutch-first,aariciah/gpt2-urdu-dutch-first,aariciah,new,BPE,38622,5.6285,1.4795,0.6585,257799,173,False,0.0141,22.66,,1
+,cis-lmu__glot500-base,cis-lmu/glot500-base,cis-lmu,new,Unigram,401143,5.4865,1.5087,0.6315,264472,174,False,0.0,24.61,,1
+,LingoIITGN__Ansh-128k,LingoIITGN/Ansh-128k,LingoIITGN,new,BPE,128002,5.4752,1.5429,0.61,265019,0,False,0.0001,24.76,,1
+,Omarrran__Kashmiri_Tokenizers__kashmiri_wordpiece_tokenizer,Omarrran/Kashmiri_Tokenizers/kashmiri_wordpiece_tokenizer,Omarrran,new,WordPiece,32000,5.4292,1.556,0.5779,267265,0,False,0.0485,25.4,,1
+,muril,google/muril-base-cased,Google,baseline,WordPiece,197258,5.392,1.5667,0.6155,269107,0,False,0.0001,25.91,,14
+,sharjilsharma__indictrans2-hi-sat-int8,sharjilsharma/indictrans2-hi-sat-int8,sharjilsharma,new,BPE,128000,5.3549,1.5722,0.6218,270972,153,False,0.0065,26.42,,1
+,theanugrah__muril-indic-ner,theanugrah/muril-indic-ner,theanugrah,new,WordPiece,197285,5.3172,1.5887,0.5994,272893,0,False,0.0,26.93,,1
+,snehagautam__Urdu_nlp_model,snehagautam/Urdu_nlp_model,snehagautam,new,BPE,32769,5.2986,1.5701,0.6165,273849,388,False,0.0002,27.19,,5
+,indicbert-v2,ai4bharat/IndicBERTv2-MLM-only,AI4Bharat,baseline,WordPiece,250000,5.2985,1.5943,0.5846,273855,0,False,0.0,27.19,,1
+,DanishMahdi__avey-b-snd,DanishMahdi/avey-b-snd,DanishMahdi,new,WordPiece,32000,5.2324,1.6145,0.5755,277317,0,False,0.2051,28.1,,1
+,Helsinki-NLP__opus-mt-ur-en,Helsinki-NLP/opus-mt-ur-en,Helsinki-NLP,new,Unigram,32000,5.2279,1.5738,0.6014,277553,173,False,0.0,28.16,,1
+,krutrim-ai-labs__Chitrarth,krutrim-ai-labs/Chitrarth,krutrim-ai-labs,new,BPE,70212,5.1677,1.6135,0.6053,280785,0,False,0.0,28.99,,4
+,Omarrran__Kashmiri_Tokenizers__kashmiri_bpe_tokenizer,Omarrran/Kashmiri_Tokenizers/kashmiri_bpe_tokenizer,Omarrran,new,BPE,32000,5.166,1.6352,0.5544,280879,0,False,0.0591,29.01,,1
+,sindhi-xlmr,Kashif786/xlm-roberta-base-sindhi-extended,community (Kashif786),baseline,Unigram,264635,5.0693,1.5986,0.5865,286236,172,False,0.0,30.34,,1
+,Omarrran__kashmiri-llm-bert-base-hnm,Omarrran/kashmiri-llm-bert-base-hnm,Omarrran,new,BPE,45026,5.0145,1.6634,0.5661,289368,0,False,0.001,31.09,,1
+,APARSIN__PashtoEnglishBilingualSentimentCorpus_bert-base-parsbert-uncased,APARSIN/PashtoEnglishBilingualSentimentCorpus_bert-base-parsbert-uncased,APARSIN,new,WordPiece,100000,4.9694,1.6999,0.5897,291991,0,False,0.1359,31.71,,2
+,livinNector__IndicBERTv2-MLM-only-NER,livinNector/IndicBERTv2-MLM-only-NER,livinNector,new,WordPiece,250000,4.9654,1.7013,0.5649,292225,0,False,0.0,31.77,,2
+,aariciah__gpt2-urdu-configC-6k,aariciah/gpt2-urdu-configC-6k,aariciah,new,BPE,6600,4.9162,1.6964,0.5467,295150,173,False,0.0123,32.44,,2
+,jhu-clsp__bernice,jhu-clsp/bernice,jhu-clsp,new,Unigram,249998,4.8821,1.5991,0.5867,297216,173,False,0.0,32.91,,1
+,MenteEAI__mentee-embed-v4,MenteEAI/mentee-embed-v4,MenteEAI,new,BPE,50000,4.8418,1.7213,0.5236,299689,0,False,0.0,33.47,,1
+,Omarrran__Kashmiri_WordPiece_Tokenizer,Omarrran/Kashmiri_WordPiece_Tokenizer,Omarrran,new,WordPiece,16000,4.8416,1.7448,0.4454,299698,0,False,0.0432,33.47,,1
+,alphaedge-ai__metaclip-2-worldwide-l14-snd-32768,alphaedge-ai/metaclip-2-worldwide-l14-snd-32768,alphaedge-ai,new,Unigram,32768,4.8353,1.6914,0.5551,300091,168,False,0.0009,33.56,,6
+,ai4bharat__IndicBERTv2-MLM-Sam-TLM-Large,ai4bharat/IndicBERTv2-MLM-Sam-TLM-Large,ai4bharat,new,WordPiece,128000,4.8002,1.7598,0.4861,302287,0,False,0.0,34.04,,2
+,Khubaib01__roman-urdu-emotion-xlmr-v2,Khubaib01/roman-urdu-emotion-xlmr-v2,Khubaib01,new,Unigram,250002,4.7918,1.6803,0.5678,302815,173,False,0.0006,34.15,,31
+,xlm-r,FacebookAI/xlm-roberta-base,Meta,baseline,Unigram,250002,4.7916,1.6804,0.5678,302829,173,False,0.0,34.16,,259
+,Omarrran__Kashmiri_BPE_Tokenizer,Omarrran/Kashmiri_BPE_Tokenizer,Omarrran,new,BPE,16000,4.7863,1.765,0.4921,303164,0,False,0.0547,34.23,,1
+,alphaedge-ai__multilingual-e5-large-instruct-snd-32768,alphaedge-ai/multilingual-e5-large-instruct-snd-32768,alphaedge-ai,new,Unigram,32768,4.7491,1.6946,0.5592,305534,169,False,0.0006,34.74,,1
+,alphaedge-ai__multilingual-e5-base-snd-32768,alphaedge-ai/multilingual-e5-base-snd-32768,alphaedge-ai,new,Unigram,32768,4.7488,1.6948,0.5591,305555,169,False,0.0006,34.74,,2
+,alphaedge-ai__multilingual-e5-small-snd-32768,alphaedge-ai/multilingual-e5-small-snd-32768,alphaedge-ai,new,Unigram,32768,4.7486,1.6949,0.559,305569,169,False,0.0001,34.75,,1
+,alphaedge-ai__bge-m3-snd-32768,alphaedge-ai/bge-m3-snd-32768,alphaedge-ai,new,Unigram,32768,4.7266,1.7023,0.5544,306990,169,False,0.0008,35.05,,1
+,zuhri025__tts-urdu-multi-scale,zuhri025/tts-urdu-multi-scale,zuhri025,new,BPE,23944,4.6908,1.7732,0.5334,309334,463,False,0.0006,35.54,,18
+,zuhri025__urdu-finetune-munch-v2,zuhri025/urdu-finetune-munch-v2,zuhri025,new,BPE,23944,4.6841,1.7687,0.5336,309780,0,False,0.0006,35.63,,32
+,impresso-project__nel-mgenre-multilingual,impresso-project/nel-mgenre-multilingual,impresso-project,new,BPE,256000,4.6794,1.8039,0.4784,310091,169,False,0.0001,35.7,,4
+,Helsinki-NLP__opus-mt-tc-bible-big-inc-deu_eng_nld,Helsinki-NLP/opus-mt-tc-bible-big-inc-deu_eng_nld,Helsinki-NLP,new,Unigram,32000,4.6547,1.7621,0.5373,311731,175,False,0.0,36.04,,2
+,Helsinki-NLP__opus-mt-tc-bible-big-iir-en,Helsinki-NLP/opus-mt-tc-bible-big-iir-en,Helsinki-NLP,new,Unigram,32000,4.6454,1.754,0.5287,312359,175,False,0.0,36.17,,2
+,Helsinki-NLP__opus-mt-tc-bible-big-inc-en,Helsinki-NLP/opus-mt-tc-bible-big-inc-en,Helsinki-NLP,new,Unigram,32000,4.6447,1.7404,0.5461,312407,175,False,0.0,36.18,,2
+,nllb-200,facebook/nllb-200-distilled-600M,Meta,baseline,BPE,256204,4.6172,1.8244,0.4663,314267,150,False,0.0057,36.55,,4
+,Hazaratun__bangla-emotion-bias-fixed,Hazaratun/bangla-emotion-bias-fixed,Hazaratun,new,WordPiece,32000,4.5964,1.8379,0.6052,315689,0,False,0.3904,36.84,,1
+,alphaedge-ai__metaclip-2-worldwide-l14-pus-32768,alphaedge-ai/metaclip-2-worldwide-l14-pus-32768,alphaedge-ai,new,Unigram,32768,4.561,1.7894,0.5129,318135,168,False,0.0009,37.33,,6
+,alphaedge-ai__embeddinggemma-snd-32768,alphaedge-ai/embeddinggemma-snd-32768,alphaedge-ai,new,BPE,32768,4.5592,1.8277,0.4758,318262,489,False,0.0,37.35,,1
+,alphaedge-ai__multilingual-e5-large-instruct-pus-32768,alphaedge-ai/multilingual-e5-large-instruct-pus-32768,alphaedge-ai,new,Unigram,32768,4.539,1.7699,0.5219,319682,169,False,0.0007,37.63,,1
+,alphaedge-ai__multilingual-e5-large-pus-32768,alphaedge-ai/multilingual-e5-large-pus-32768,alphaedge-ai,new,Unigram,32768,4.5388,1.77,0.5219,319696,169,False,0.0007,37.63,,2
+,alphaedge-ai__multilingual-e5-small-pus-32768,alphaedge-ai/multilingual-e5-small-pus-32768,alphaedge-ai,new,Unigram,32768,4.5386,1.7701,0.5218,319710,169,False,0.0002,37.63,,1
+,alphaedge-ai__bge-m3-pus-32768,alphaedge-ai/bge-m3-pus-32768,alphaedge-ai,new,Unigram,32768,4.5278,1.7737,0.5212,320468,169,False,0.0007,37.78,,1
+,ihanif__pashto-tokenizer,ihanif/pashto-tokenizer,ihanif,new,BPE,32000,4.5231,1.8676,0.4872,320804,0,False,0.0236,37.85,,1
+,Helsinki-NLP__opus-mt-iir-iir,Helsinki-NLP/opus-mt-iir-iir,Helsinki-NLP,new,Unigram,32000,4.4986,1.8105,0.5081,322550,169,False,0.0001,38.18,,1
+,mirfan899__punjabi_albert,mirfan899/punjabi_albert,mirfan899,new,Unigram,3204,4.4971,1.8491,0.4418,322660,103,False,0.0033,38.2,,1
+,alphaedge-ai__bge-m3-urd-32768,alphaedge-ai/bge-m3-urd-32768,alphaedge-ai,new,Unigram,32768,4.48,1.8068,0.5005,323892,157,False,0.0008,38.44,,1
+,alphaedge-ai__mbart-large-50-pus-32768,alphaedge-ai/mbart-large-50-pus-32768,alphaedge-ai,new,Unigram,32768,4.4743,1.7788,0.518,324303,169,False,0.0118,38.52,,1
+,alphaedge-ai__multilingual-e5-base-urd-32768,alphaedge-ai/multilingual-e5-base-urd-32768,alphaedge-ai,new,Unigram,32768,4.4724,1.8079,0.4988,324441,156,False,0.0003,38.54,,3
+,alphaedge-ai__multilingual-e5-large-instruct-urd-32768,alphaedge-ai/multilingual-e5-large-instruct-urd-32768,alphaedge-ai,new,Unigram,32768,4.4722,1.8079,0.4987,324456,156,False,0.0008,38.55,,1
+,Helsinki-NLP__opus-mt-tc-bible-big-iir-deu_eng_fra_por_spa,Helsinki-NLP/opus-mt-tc-bible-big-iir-deu_eng_fra_por_spa,Helsinki-NLP,new,Unigram,32000,4.4706,1.8334,0.4959,324570,175,False,0.0,38.57,,2
+,Helsinki-NLP__opus-mt-tc-bible-big-inc-deu_eng_fra_por_spa,Helsinki-NLP/opus-mt-tc-bible-big-inc-deu_eng_fra_por_spa,Helsinki-NLP,new,Unigram,32000,4.4536,1.8425,0.4999,325810,175,False,0.0,38.8,,2
+,facebook__m2m100_418M,facebook/m2m100_418M,facebook,new,BPE,128000,4.4414,1.9001,0.4366,326703,169,False,0.0001,38.97,,32
+,Helsinki-NLP__opus-mt-inc-inc,Helsinki-NLP/opus-mt-inc-inc,Helsinki-NLP,new,Unigram,32000,4.4293,1.8215,0.4967,327598,169,False,0.0001,39.14,,1
+,alphaedge-ai__multilingual-e5-large-snd-16384,alphaedge-ai/multilingual-e5-large-snd-16384,alphaedge-ai,new,Unigram,16384,4.4245,1.8227,0.4916,327951,169,False,0.0007,39.2,,2
+,alphaedge-ai__multilingual-e5-large-instruct-snd-16384,alphaedge-ai/multilingual-e5-large-instruct-snd-16384,alphaedge-ai,new,Unigram,16384,4.4244,1.8228,0.4918,327960,169,False,0.0007,39.2,,1
+,alphaedge-ai__multilingual-e5-small-snd-16384,alphaedge-ai/multilingual-e5-small-snd-16384,alphaedge-ai,new,Unigram,16384,4.4243,1.8229,0.4916,327965,169,False,0.0002,39.2,,1
+,alphaedge-ai__embeddinggemma-snd-16384,alphaedge-ai/embeddinggemma-snd-16384,alphaedge-ai,new,BPE,16384,4.424,1.8843,0.4486,327986,487,False,0.0,39.21,,1
+,alphaedge-ai__mbart-large-50-urd-32768,alphaedge-ai/mbart-large-50-urd-32768,alphaedge-ai,new,Unigram,32768,4.4199,1.8126,0.4973,328293,157,False,0.0118,39.26,,1
+,APARSIN__PashtoEnglishBilingualSentimentCorpus_TookaBERT-Base,APARSIN/PashtoEnglishBilingualSentimentCorpus_TookaBERT-Base,APARSIN,new,BPE,48000,4.4192,1.8737,0.4288,328344,0,False,0.221,39.27,,4
+,goldfish-models__snd_arab_full,goldfish-models/snd_arab_full,goldfish-models,new,Unigram,51200,4.3905,1.8812,0.4596,330494,164,False,0.0005,39.67,,2
+,alphaedge-ai__metaclip-2-worldwide-b16-384-snd-16384,alphaedge-ai/metaclip-2-worldwide-b16-384-snd-16384,alphaedge-ai,new,Unigram,16384,4.387,1.8689,0.4752,330758,166,False,0.0009,39.72,,6
+,alphaedge-ai__bge-m3-snd-16384,alphaedge-ai/bge-m3-snd-16384,alphaedge-ai,new,Unigram,16384,4.3697,1.8464,0.4811,332064,168,False,0.0008,39.95,,1
+,APARSIN__PashtoEnglishBilingualSentimentCorpus_bert-fa-zwnj-base,APARSIN/PashtoEnglishBilingualSentimentCorpus_bert-fa-zwnj-base,APARSIN,new,WordPiece,42000,4.3488,1.9425,0.4221,333658,0,False,0.0613,40.24,,2
+,laibamuzammal7__iqbal-urdu-transformer,laibamuzammal7/iqbal-urdu-transformer,laibamuzammal7,new,BPE,2767,4.3154,1.9575,0.409,336245,0,False,0.0315,40.7,,1
+,google-bert__bert-base-multilingual-uncased,google-bert/bert-base-multilingual-uncased,google-bert,new,WordPiece,105879,4.2939,1.9673,0.4269,337929,0,False,0.0107,41.0,,20
+,APARSIN__PashtoEnglishBilingualSentimentCorpus_bert-fa-base-uncased,APARSIN/PashtoEnglishBilingualSentimentCorpus_bert-fa-base-uncased,APARSIN,new,WordPiece,100000,4.2731,1.9769,0.399,339570,0,False,0.0118,41.28,,2
+,goldfish-models__snd_arab_10mb,goldfish-models/snd_arab_10mb,goldfish-models,new,Unigram,51200,4.2274,1.9736,0.3562,343243,164,False,0.0005,41.91,,1
+,Omarrran__Kashmiri_Unigram_Tokenizer,Omarrran/Kashmiri_Unigram_Tokenizer,Omarrran,new,Unigram,7,4.2238,1.0,1.0,343538,0,False,0.5,41.96,,1
+,jbhargav__gujarati-indicbart-5000,jbhargav/gujarati-indicbart-5000,jbhargav,new,Unigram,72,4.2231,1.0002,0.9999,343592,0,False,0.5,41.97,,1
+,alphaedge-ai__multilingual-e5-large-pus-16384,alphaedge-ai/multilingual-e5-large-pus-16384,alphaedge-ai,new,Unigram,16384,4.2214,1.8891,0.4754,343733,168,False,0.0008,41.99,,2
+,alphaedge-ai__multilingual-e5-large-instruct-pus-16384,alphaedge-ai/multilingual-e5-large-instruct-pus-16384,alphaedge-ai,new,Unigram,16384,4.2214,1.8891,0.4754,343734,168,False,0.0008,41.99,,1
+,alphaedge-ai__multilingual-e5-small-pus-16384,alphaedge-ai/multilingual-e5-small-pus-16384,alphaedge-ai,new,Unigram,16384,4.2212,1.8892,0.4753,343750,168,False,0.0004,42.0,,1
+,Helsinki-NLP__opus-mt-iir-en,Helsinki-NLP/opus-mt-iir-en,Helsinki-NLP,new,Unigram,32000,4.207,1.9221,0.4623,344906,169,False,0.0001,42.19,,1
+,goldfish-models__kas_deva_full,goldfish-models/kas_deva_full,goldfish-models,new,Unigram,51200,4.2065,1.0135,0.9901,344951,0,False,0.4974,42.2,,1
+,goldfish-models__kas_deva_5mb,goldfish-models/kas_deva_5mb,goldfish-models,new,Unigram,51200,4.2064,1.0135,0.9901,344953,0,False,0.4974,42.2,,1
+,Helsinki-NLP__opus-mt-synthetic-en-is,Helsinki-NLP/opus-mt-synthetic-en-is,Helsinki-NLP,new,Unigram,32000,4.2054,1.014,0.9914,345039,0,False,0.4971,42.21,,1
+,Helsinki-NLP__opus-mt_tiny_spa-eus,Helsinki-NLP/opus-mt_tiny_spa-eus,Helsinki-NLP,new,Unigram,32000,4.2051,1.0142,0.9899,345067,0,False,0.4973,42.22,,1
+,Helsinki-NLP__opus-mt_tiny_spa-glg,Helsinki-NLP/opus-mt_tiny_spa-glg,Helsinki-NLP,new,Unigram,32000,4.2051,1.0142,0.9899,345067,0,False,0.4973,42.22,,1
+,Helsinki-NLP__opus-mt-synthetic-en-eu,Helsinki-NLP/opus-mt-synthetic-en-eu,Helsinki-NLP,new,Unigram,32000,4.205,1.0142,0.9914,345072,0,False,0.4971,42.22,,1
+,Helsinki-NLP__opus-mt-synthetic-en-ka,Helsinki-NLP/opus-mt-synthetic-en-ka,Helsinki-NLP,new,Unigram,32000,4.2048,1.0143,0.9914,345092,0,False,0.4971,42.22,,1
+,Helsinki-NLP__opus-mt-pa-en,Helsinki-NLP/opus-mt-pa-en,Helsinki-NLP,new,Unigram,32000,4.2046,1.0143,0.99,345108,0,False,0.4972,42.22,,1
+,goldfish-models__pan_guru_1000mb,goldfish-models/pan_guru_1000mb,goldfish-models,new,Unigram,51200,4.2001,1.0215,0.9839,345475,0,False,0.4956,42.29,,2
+,goldfish-models__pan_guru_10mb,goldfish-models/pan_guru_10mb,goldfish-models,new,Unigram,51200,4.1999,1.0215,0.9838,345491,0,False,0.4956,42.29,,1
+,goldfish-models__pan_guru_5mb,goldfish-models/pan_guru_5mb,goldfish-models,new,Unigram,40960,4.1989,1.0219,0.9837,345572,0,False,0.4955,42.3,,1
+,nvidia__NVIDIA-NemotronLabs-VoiceChat-11B__rnnt_tokenizer,nvidia/NVIDIA-NemotronLabs-VoiceChat-11B/rnnt_tokenizer,nvidia,new,BPE,1024,4.1976,1.0129,0.9926,345677,0,False,0.4958,42.32,,6
+,Recag__Bharatai-v-2,Recag/Bharatai-v-2,Recag,new,Unigram,5000,4.1938,1.0182,0.9892,345994,0,False,0.4967,42.37,,1
+,alphaedge-ai__bge-m3-pus-16384,alphaedge-ai/bge-m3-pus-16384,alphaedge-ai,new,Unigram,16384,4.1931,1.9137,0.4637,346048,37,False,0.0075,42.38,,1
+,ibraheemmoosa__xlmindic-base-uniscript,ibraheemmoosa/xlmindic-base-uniscript,ibraheemmoosa,new,Unigram,50000,4.1924,1.0178,0.9887,346111,0,False,0.4958,42.39,,1
+,abdulwaheed1__pegasus-model-finetuned-on-samsum-data,abdulwaheed1/pegasus-model-finetuned-on-samsum-data,abdulwaheed1,new,Unigram,96000,4.1893,1.0262,0.9839,346362,0,False,0.4943,42.43,,1
+,Recag__BharatAI,Recag/BharatAI,Recag,new,Unigram,1000,4.1862,1.0201,0.9887,346621,0,False,0.4958,42.48,,1
+,Helsinki-NLP__opus-mt_tiny_eng-fra,Helsinki-NLP/opus-mt_tiny_eng-fra,Helsinki-NLP,new,Unigram,32000,4.1859,1.0234,0.9857,346648,0,False,0.4951,42.48,,1
+,Helsinki-NLP__opus-mt_tiny_fra-eng,Helsinki-NLP/opus-mt_tiny_fra-eng,Helsinki-NLP,new,Unigram,32000,4.1859,1.0235,0.9857,346650,0,False,0.4951,42.48,,1
+,Helsinki-NLP__opus-mt_tiny_cat-eng,Helsinki-NLP/opus-mt_tiny_cat-eng,Helsinki-NLP,new,Unigram,32000,4.1855,1.0235,0.9857,346679,0,False,0.4951,42.49,,1
+,Helsinki-NLP__opus-mt_tiny_eng-cat,Helsinki-NLP/opus-mt_tiny_eng-cat,Helsinki-NLP,new,Unigram,32000,4.1854,1.0236,0.9857,346687,0,False,0.495,42.49,,1
+,Helsinki-NLP__opus-mt_tiny_eng-tur,Helsinki-NLP/opus-mt_tiny_eng-tur,Helsinki-NLP,new,Unigram,32000,4.1854,1.0237,0.9857,346690,0,False,0.495,42.49,,1
+,Helsinki-NLP__opus-mt_tiny_eng-rus,Helsinki-NLP/opus-mt_tiny_eng-rus,Helsinki-NLP,new,Unigram,32000,4.1854,1.0237,0.9857,346691,0,False,0.4951,42.49,,1
+,Helsinki-NLP__opus-mt_tiny_rus-eng,Helsinki-NLP/opus-mt_tiny_rus-eng,Helsinki-NLP,new,Unigram,32000,4.1853,1.0237,0.9856,346693,0,False,0.4951,42.49,,1
+,Helsinki-NLP__opus-mt_tiny_tur-eng,Helsinki-NLP/opus-mt_tiny_tur-eng,Helsinki-NLP,new,Unigram,32000,4.1853,1.0237,0.9857,346695,0,False,0.495,42.49,,1
+,Helsinki-NLP__opus-mt-synthetic-en-gd,Helsinki-NLP/opus-mt-synthetic-en-gd,Helsinki-NLP,new,Unigram,32000,4.1851,1.0242,0.9867,346712,0,False,0.495,42.49,,1
+,Helsinki-NLP__opus-mt-inc-en,Helsinki-NLP/opus-mt-inc-en,Helsinki-NLP,new,Unigram,32000,4.1849,1.905,0.4721,346728,169,False,0.0001,42.49,,1
+,Helsinki-NLP__opus-mt_tiny_cat-spa,Helsinki-NLP/opus-mt_tiny_cat-spa,Helsinki-NLP,new,Unigram,32000,4.1844,1.0241,0.9855,346767,0,False,0.4949,42.5,,1
+,Helsinki-NLP__opus-mt_tiny_spa-cat,Helsinki-NLP/opus-mt_tiny_spa-cat,Helsinki-NLP,new,Unigram,32000,4.1844,1.0242,0.9854,346774,0,False,0.4949,42.5,,1
+,Helsinki-NLP__opus-mt-synthetic-en-so,Helsinki-NLP/opus-mt-synthetic-en-so,Helsinki-NLP,new,Unigram,32000,4.184,1.0243,0.9867,346807,0,False,0.4949,42.51,,1
+,goldfish-models__snd_arab_5mb,goldfish-models/snd_arab_5mb,goldfish-models,new,Unigram,47104,4.1839,1.9561,0.3713,346808,164,False,0.0005,42.51,,1
+,Helsinki-NLP__opus-mt-synthetic-en-mk,Helsinki-NLP/opus-mt-synthetic-en-mk,Helsinki-NLP,new,Unigram,32000,4.1837,1.0244,0.9867,346827,0,False,0.4948,42.51,,1
+,Helsinki-NLP__opus-mt-synthetic-en-uk,Helsinki-NLP/opus-mt-synthetic-en-uk,Helsinki-NLP,new,Unigram,32000,4.1831,1.0247,0.9866,346882,0,False,0.4948,42.52,,1
+,BharathP08__longt5-custom-summarizer,BharathP08/longt5-custom-summarizer,BharathP08,new,Unigram,32100,4.1798,1.0303,0.9818,347156,0,False,0.4931,42.56,,1
+,Helsinki-NLP__opus-mt_tiny_spa-eng,Helsinki-NLP/opus-mt_tiny_spa-eng,Helsinki-NLP,new,Unigram,32000,4.1787,1.0314,0.981,347247,0,False,0.4931,42.58,,1
+,Helsinki-NLP__opus-mt_tiny_eng-spa,Helsinki-NLP/opus-mt_tiny_eng-spa,Helsinki-NLP,new,Unigram,32000,4.1786,1.0315,0.981,347253,0,False,0.4931,42.58,,1
+,Helsinki-NLP__opus-mt_tiny_deu-eng,Helsinki-NLP/opus-mt_tiny_deu-eng,Helsinki-NLP,new,Unigram,32000,4.1785,1.0315,0.9809,347262,0,False,0.4931,42.58,,1
+,Helsinki-NLP__opus-mt_tiny_ita-eng,Helsinki-NLP/opus-mt_tiny_ita-eng,Helsinki-NLP,new,Unigram,32000,4.1785,1.0315,0.981,347263,0,False,0.4931,42.58,,1
+,Helsinki-NLP__opus-mt_tiny_eng-deu,Helsinki-NLP/opus-mt_tiny_eng-deu,Helsinki-NLP,new,Unigram,32000,4.1784,1.0315,0.9809,347266,0,False,0.4931,42.58,,1
+,Helsinki-NLP__opus-mt_tiny_nld-eng,Helsinki-NLP/opus-mt_tiny_nld-eng,Helsinki-NLP,new,Unigram,32000,4.1784,1.0316,0.9809,347269,0,False,0.493,42.58,,1
+,Helsinki-NLP__opus-mt_tiny_eng-nld,Helsinki-NLP/opus-mt_tiny_eng-nld,Helsinki-NLP,new,Unigram,32000,4.1783,1.0316,0.9809,347276,0,False,0.493,42.58,,1
+,Helsinki-NLP__opus-mt_tiny_eng-ita,Helsinki-NLP/opus-mt_tiny_eng-ita,Helsinki-NLP,new,Unigram,32000,4.1783,1.0316,0.981,347280,0,False,0.493,42.59,,1
+,Helsinki-NLP__opus-mt_tiny_eng-ell,Helsinki-NLP/opus-mt_tiny_eng-ell,Helsinki-NLP,new,Unigram,32000,4.1782,1.0316,0.9809,347284,0,False,0.493,42.59,,1
+,Helsinki-NLP__opus-mt_tiny_ell-eng,Helsinki-NLP/opus-mt_tiny_ell-eng,Helsinki-NLP,new,Unigram,32000,4.1781,1.0317,0.9809,347296,0,False,0.493,42.59,,1
+,Snowflake__snowflake-arctic-tilt-v1.3,Snowflake/snowflake-arctic-tilt-v1.3,Snowflake,new,Unigram,32100,4.1772,1.0314,0.981,347369,0,False,0.4929,42.6,,33
+,mbert,google-bert/bert-base-multilingual-cased,Google,baseline,WordPiece,119547,4.1505,2.0353,0.4189,349601,0,False,0.0107,42.97,,48
+,goldfish-models__pbt_arab_100mb,goldfish-models/pbt_arab_100mb,goldfish-models,new,Unigram,51200,4.1465,2.0023,0.4288,349941,0,False,0.1272,43.02,,2
+,alphaedge-ai__mbart-large-50-pus-16384,alphaedge-ai/mbart-large-50-pus-16384,alphaedge-ai,new,Unigram,16384,4.1444,1.9183,0.461,350116,37,False,0.0177,43.05,,1
+,upstage__TFLOP,upstage/TFLOP,upstage,new,Unigram,58337,4.1427,1.0489,0.9724,350258,0,False,0.4932,43.07,,1
+,alphaedge-ai__metaclip-2-worldwide-s16-384-pus-16384,alphaedge-ai/metaclip-2-worldwide-s16-384-pus-16384,alphaedge-ai,new,Unigram,16384,4.1383,2.0042,0.4278,350636,38,False,0.0074,43.13,,6
+,Geotrend__bert-base-15lang-cased,Geotrend/bert-base-15lang-cased,Geotrend,new,WordPiece,71577,4.1132,2.0538,0.4048,352772,0,False,0.0106,43.48,,3
+,Geotrend__bert-base-ur-cased,Geotrend/bert-base-ur-cased,Geotrend,new,WordPiece,8669,4.1116,2.0545,0.4044,352907,0,False,0.0106,43.5,,2
+,goldfish-models__pus_arab_1000mb,goldfish-models/pus_arab_1000mb,goldfish-models,new,Unigram,51200,4.1103,2.0186,0.4222,353026,0,False,0.1204,43.52,,2
+,goldfish-models__pbt_arab_10mb,goldfish-models/pbt_arab_10mb,goldfish-models,new,Unigram,51200,4.1048,2.0308,0.3903,353493,0,False,0.1259,43.59,,1
+,alphaedge-ai__embeddinggemma-urd-32768,alphaedge-ai/embeddinggemma-urd-32768,alphaedge-ai,new,BPE,32768,4.104,2.0332,0.3787,353565,445,False,0.0003,43.61,,1
+,aakashMeghwar01__SindhiLM-Qwen-0.5B__last-checkpoint,aakashMeghwar01/SindhiLM-Qwen-0.5B/last-checkpoint,aakashMeghwar01,new,BPE,13032,4.0999,2.0337,0.3846,353921,162,False,0.0,43.66,,1
+,aakashMeghwar01__SindhiLM-Qwen-0.5B-v2__checkpoint-1000,aakashMeghwar01/SindhiLM-Qwen-0.5B-v2/checkpoint-1000,aakashMeghwar01,new,BPE,13038,4.0908,2.0362,0.3839,354707,162,False,0.0,43.79,,27
+,goldfish-models__pbt_arab_5mb,goldfish-models/pbt_arab_5mb,goldfish-models,new,Unigram,43008,4.0615,2.0534,0.374,357267,0,False,0.1246,44.19,,1
+,rabbiaidrees__donut-urdu-vqa,rabbiaidrees/donut-urdu-vqa,rabbiaidrees,new,Unigram,57536,4.0456,1.1178,0.9153,358671,0,False,0.4581,44.41,,4
+,goldfish-models__pus_arab_10mb,goldfish-models/pus_arab_10mb,goldfish-models,new,Unigram,51200,4.035,2.0685,0.3847,359606,0,False,0.1238,44.55,,1
+,APARSIN__PashtoEnglishBilingualSentimentCorpus_fabert,APARSIN/PashtoEnglishBilingualSentimentCorpus_fabert,APARSIN,new,WordPiece,50000,4.0121,2.1055,0.3653,361662,0,False,0.0,44.87,,2
+,Abdullah104__my_awesome_pashto_clm-model,Abdullah104/my_awesome_pashto_clm-model,Abdullah104,new,BPE,12049,4.0065,2.1084,0.3274,362164,0,False,0.0,44.94,,1
+,alphaedge-ai__metaclip-2-worldwide-l14-pan-16384,alphaedge-ai/metaclip-2-worldwide-l14-pan-16384,alphaedge-ai,new,Unigram,16384,3.9646,1.1995,0.8506,365992,0,False,0.4805,45.52,,6
+,alphaedge-ai__metaclip-2-worldwide-l14-urd-32768,alphaedge-ai/metaclip-2-worldwide-l14-urd-32768,alphaedge-ai,new,Unigram,32768,3.956,2.0648,0.4267,366787,155,False,0.0009,45.64,,6
+,jamalimubashirali__sindhi-tinyllama-tokenizer,jamalimubashirali/sindhi-tinyllama-tokenizer,jamalimubashirali,new,BPE,42681,3.9494,2.1389,0.3072,367402,0,False,0.0,45.73,,1
+,Helsinki-NLP__opus-mt-ine-en,Helsinki-NLP/opus-mt-ine-en,Helsinki-NLP,new,Unigram,32000,3.922,2.0514,0.4085,369970,169,False,0.0,46.11,,1
+,CohereLabs__aya-101,CohereLabs/aya-101,CohereLabs,new,Unigram,250100,3.9125,1.8745,0.413,370866,174,False,0.0,46.24,,17
+,mt5,google/mt5-small,Google,baseline,Unigram,250100,3.9125,1.8745,0.413,370870,175,False,0.0,46.24,,3
+,areebas465__urt5-lora-urdu-summarization,areebas465/urt5-lora-urdu-summarization,areebas465,new,Unigram,40750,3.912,1.8748,0.413,370912,172,False,0.0,46.24,,1
+,mbshr__urt5-base-finetuned,mbshr/urt5-base-finetuned,mbshr,new,Unigram,40750,3.9119,1.8748,0.413,370924,175,False,0.0,46.25,,1
+,alifaheem94__distilled-mt5-base,alifaheem94/distilled-mt5-base,alifaheem94,new,Unigram,8068,3.9037,1.8795,0.4115,371705,175,False,0.0,46.36,,1
+,goldfish-models__pus_arab_5mb,goldfish-models/pus_arab_5mb,goldfish-models,new,Unigram,45056,3.8989,2.0851,0.367,372159,0,False,0.1932,46.42,,1
+,alphaedge-ai__mt5-small-snd-32768,alphaedge-ai/mt5-small-snd-32768,alphaedge-ai,new,Unigram,32668,3.898,1.8832,0.4087,372249,175,False,0.0,46.44,,2
+,alphaedge-ai__mt5-small-pus-32768,alphaedge-ai/mt5-small-pus-32768,alphaedge-ai,new,Unigram,32668,3.8764,1.8905,0.4103,374322,175,False,0.0,46.73,,2
+,alphaedge-ai__mt5-small-snd-16384,alphaedge-ai/mt5-small-snd-16384,alphaedge-ai,new,Unigram,16284,3.8711,1.906,0.3999,374834,175,False,0.0,46.81,,2
+,Helsinki-NLP__opus-mt-ine-ine,Helsinki-NLP/opus-mt-ine-ine,Helsinki-NLP,new,Unigram,32000,3.8334,2.1056,0.3839,378526,169,False,0.0001,47.32,,2
+,goldfish-models__bgp_latn_full,goldfish-models/bgp_latn_full,goldfish-models,new,Unigram,51200,3.8289,2.0435,0.3549,378969,2,False,0.0271,47.39,,1
+,alphaedge-ai__mt5-small-pus-16384,alphaedge-ai/mt5-small-pus-16384,alphaedge-ai,new,Unigram,16284,3.8116,1.9322,0.4017,380687,175,False,0.0,47.62,,2
+,goldfish-models__bgp_latn_5mb,goldfish-models/bgp_latn_5mb,goldfish-models,new,Unigram,45056,3.8088,2.0627,0.3514,380970,2,False,0.027,47.66,,1
+,alphaedge-ai__mt5-base-urd-32768,alphaedge-ai/mt5-base-urd-32768,alphaedge-ai,new,Unigram,32668,3.8039,1.9612,0.3839,381453,175,False,0.0,47.73,,2
+,jhu-clsp__mmBERT-base,jhu-clsp/mmBERT-base,jhu-clsp,new,BPE,256000,3.791,2.2029,0.305,382760,0,False,0.0,47.91,,3
+,durgaamma2005__indic-transformers-te-distilbert,durgaamma2005/indic-transformers-te-distilbert,durgaamma2005,new,WordPiece,30522,3.7858,2.2314,0.4802,383283,0,False,0.2508,47.98,,1
+,alphaedge-ai__mmBERT-small-pus-32768,alphaedge-ai/mmBERT-small-pus-32768,alphaedge-ai,new,BPE,32768,3.7812,2.2086,0.3019,383743,0,False,0.0,48.04,,2
+,alphaedge-ai__mmBERT-base-snd-32768,alphaedge-ai/mmBERT-base-snd-32768,alphaedge-ai,new,BPE,32768,3.7759,2.2118,0.3013,384290,0,False,0.0,48.11,,2
+,alphaedge-ai__mmBERT-base-pus-16384,alphaedge-ai/mmBERT-base-pus-16384,alphaedge-ai,new,BPE,16384,3.7692,2.2157,0.301,384965,0,False,0.0003,48.21,,2
+,hellosindh__sindhi-bert-base,hellosindh/sindhi-bert-base,hellosindh,new,Unigram,32001,3.7602,2.2332,0.3079,385887,1,False,0.0442,48.33,,1
+,alphaedge-ai__mmBERT-base-snd-16384,alphaedge-ai/mmBERT-base-snd-16384,alphaedge-ai,new,BPE,16384,3.754,2.2248,0.2962,386530,0,False,0.0002,48.42,,2
+,Helsinki-NLP__opus-mt-tc-bible-big-mul-mul,Helsinki-NLP/opus-mt-tc-bible-big-mul-mul,Helsinki-NLP,new,Unigram,64000,3.7268,2.1428,0.3472,389348,175,False,0.0,48.79,,2
+,alphaedge-ai__bge-m3-urd-16384,alphaedge-ai/bge-m3-urd-16384,alphaedge-ai,new,Unigram,16384,3.697,2.2145,0.3661,392487,151,False,0.0011,49.2,,1
+,almanach__moderncamembert-base,almanach/moderncamembert-base,almanach,new,WordPiece,32768,3.6891,2.2688,0.4832,393332,0,False,0.2583,49.31,,6
+,alphaedge-ai__multilingual-e5-large-instruct-urd-16384,alphaedge-ai/multilingual-e5-large-instruct-urd-16384,alphaedge-ai,new,Unigram,16384,3.6725,2.2234,0.3614,395102,151,False,0.0011,49.53,,1
+,alphaedge-ai__multilingual-e5-large-urd-16384,alphaedge-ai/multilingual-e5-large-urd-16384,alphaedge-ai,new,Unigram,16384,3.6725,2.2234,0.3611,395110,151,False,0.0007,49.54,,3
+,alphaedge-ai__mmBERT-base-urd-32768,alphaedge-ai/mmBERT-base-urd-32768,alphaedge-ai,new,BPE,32768,3.6549,2.2857,0.2806,397012,0,False,0.0003,49.78,,3
+,HPLT__hplt_bert_base_ps,HPLT/hplt_bert_base_ps,HPLT,new,WordPiece,16384,3.6542,2.2906,0.3216,397080,467,False,0.0,49.79,,1
+,alphaedge-ai__mbart-large-50-urd-16384,alphaedge-ai/mbart-large-50-urd-16384,alphaedge-ai,new,Unigram,16384,3.6506,2.2237,0.3607,397478,151,False,0.0101,49.84,,1
+,aariciah__gpt2-persian-dutch-first,aariciah/gpt2-persian-dutch-first,aariciah,new,BPE,38488,3.6416,2.2353,0.2982,398456,0,False,0.0951,49.96,,1
+,aariciah__gpt2-persian-dutch-merge,aariciah/gpt2-persian-dutch-merge,aariciah,new,BPE,38488,3.6413,2.2355,0.2981,398496,0,False,0.0951,49.96,,1
+,aariciah__gpt2-persian-20k-lc,aariciah/gpt2-persian-20k-lc,aariciah,new,BPE,20000,3.6395,2.2362,0.298,398691,1,False,0.0503,49.99,,3
+,aariciah__gpt2-persian-configC-20k,aariciah/gpt2-persian-configC-20k,aariciah,new,BPE,20000,3.639,2.2365,0.2979,398742,0,False,0.0951,50.0,,4
+,Kashif786__sindhi_wordpiece_20k_3M,Kashif786/sindhi_wordpiece_20k_3M,Kashif786,new,WordPiece,20000,3.6274,2.3288,0.2715,400015,0,False,0.0,50.15,,1
+,tencent__WeMM-Embedding-2B,tencent/WeMM-Embedding-2B,tencent,new,BPE,248078,3.5982,2.3227,0.2575,403264,482,False,0.0,50.56,,3
+,Theoistic__Bantam-285m,Theoistic/Bantam-285m,Theoistic,new,BPE,49152,3.5545,2.3543,0.2692,408218,467,False,0.0,51.16,,1
+,alphaedge-ai__embeddinggemma-urd-16384,alphaedge-ai/embeddinggemma-urd-16384,alphaedge-ai,new,BPE,16384,3.537,2.3628,0.2711,410241,430,False,0.0003,51.4,,1
+,Helsinki-NLP__opus-mt-tc-bible-big-ine-deu_eng_fra_por_spa,Helsinki-NLP/opus-mt-tc-bible-big-ine-deu_eng_fra_por_spa,Helsinki-NLP,new,Unigram,32000,3.5024,2.2991,0.2935,414298,175,False,0.0,51.87,,2
+,Helsinki-NLP__opus-mt-mul-en,Helsinki-NLP/opus-mt-mul-en,Helsinki-NLP,new,Unigram,32000,3.4899,2.2612,0.3246,415776,169,False,0.0,52.04,,5
+,alphaedge-ai__mt5-base-urd-16384,alphaedge-ai/mt5-base-urd-16384,alphaedge-ai,new,Unigram,16284,3.4462,2.203,0.3351,421047,175,False,0.0,52.64,,2
+,alphaedge-ai__mmBERT-small-urd-16384,alphaedge-ai/mmBERT-small-urd-16384,alphaedge-ai,new,BPE,16384,3.4166,2.4465,0.2385,424698,0,False,0.0006,53.05,,1
+,Helsinki-NLP__opus-mt-tc-bible-big-ine-deu_eng_nld,Helsinki-NLP/opus-mt-tc-bible-big-ine-deu_eng_nld,Helsinki-NLP,new,Unigram,32000,3.413,2.3591,0.3047,425147,175,False,0.0,53.1,,2
+,aariciah__gpt2-persian-configC-6k,aariciah/gpt2-persian-configC-6k,aariciah,new,BPE,6600,3.3504,2.4344,0.2283,433091,0,False,0.0875,53.96,,2
+,Helsinki-NLP__opus-mt-tc-bible-big-mul-deu_eng_fra_por_spa,Helsinki-NLP/opus-mt-tc-bible-big-mul-deu_eng_fra_por_spa,Helsinki-NLP,new,Unigram,32000,3.2584,2.4389,0.2482,445315,175,False,0.0,55.22,,2
+,shaheen6__ArabicOCR,shaheen6/ArabicOCR,shaheen6,new,WordPiece,32000,3.2484,2.6005,0.1529,446690,0,False,0.0,55.36,,1
+,vonjack__opus-mt-mul-en-big,vonjack/opus-mt-mul-en-big,vonjack,new,Unigram,32000,3.2343,2.4163,0.2517,448630,175,False,0.0,55.56,,1
+,Helsinki-NLP__opus-mt-tc-bible-big-mul-deu_eng_nld,Helsinki-NLP/opus-mt-tc-bible-big-mul-deu_eng_nld,Helsinki-NLP,new,Unigram,32000,3.2055,2.455,0.2394,452672,175,False,0.0,55.95,,2
+,ibraheemmoosa__xlmindic-base-multiscript-soham,ibraheemmoosa/xlmindic-base-multiscript-soham,ibraheemmoosa,new,Unigram,50000,3.1922,1.786,0.5704,454552,0,False,0.4794,56.13,,2
+,alphaedge-ai__metaclip-2-worldwide-l14-urd-16384,alphaedge-ai/metaclip-2-worldwide-l14-urd-16384,alphaedge-ai,new,Unigram,16384,3.1004,2.6309,0.299,468008,0,False,0.0492,57.4,,6
+,KooAI__KooBERT,KooAI/KooBERT,KooAI,new,WordPiece,128000,3.0876,2.736,0.1497,469952,0,False,0.0,57.57,,1
+,cservan__multilingual-albert-base-cased-128k,cservan/multilingual-albert-base-cased-128k,cservan,new,Unigram,128000,3.0869,2.5386,0.1608,470066,169,False,0.0003,57.58,,1
+,aakashMeghwar01__sindhi-gpt-bpe-final,aakashMeghwar01/sindhi-gpt-bpe-final,aakashMeghwar01,new,BPE,5000,3.0858,2.7162,0.1839,470221,467,False,0.0,57.6,,1
+,zuhri025__TTS-urdu-v1,zuhri025/TTS-urdu-v1,zuhri025,new,BPE,23934,3.0439,1.7688,0.5336,476692,0,False,0.0004,58.17,,11
+,alphaedge-ai__bge-m3-pan-32768,alphaedge-ai/bge-m3-pan-32768,alphaedge-ai,new,Unigram,32768,2.9849,2.657,0.288,486129,0,False,0.0851,58.98,,1
+,Helsinki-NLP__opus-mt_tiny_ara-eng,Helsinki-NLP/opus-mt_tiny_ara-eng,Helsinki-NLP,new,Unigram,32000,2.9587,2.5648,0.1532,490424,0,False,0.3645,59.34,,1
+,alphaedge-ai__multilingual-e5-base-pan-32768,alphaedge-ai/multilingual-e5-base-pan-32768,alphaedge-ai,new,Unigram,32768,2.9217,2.7008,0.2876,496632,0,False,0.0971,59.85,,3
+,alphaedge-ai__multilingual-e5-large-instruct-pan-32768,alphaedge-ai/multilingual-e5-large-instruct-pan-32768,alphaedge-ai,new,Unigram,32768,2.9216,2.701,0.2873,496661,0,False,0.0974,59.85,,1
+,sgugger__push-to-hub-test-2,sgugger/push-to-hub-test-2,sgugger,new,WordPiece,28996,2.8884,2.9246,0.1999,502358,0,False,0.0858,60.31,,3
+,aariciah__gpt2-arabic-dutch-first,aariciah/gpt2-arabic-dutch-first,aariciah,new,BPE,38699,2.8482,2.8058,0.1223,509459,3,False,0.0304,60.86,,1
+,aariciah__gpt2-arabic-dutch-merge,aariciah/gpt2-arabic-dutch-merge,aariciah,new,BPE,38699,2.8481,2.806,0.1223,509480,3,False,0.0304,60.86,,1
+,aariciah__gpt2-arabic-20k-lc,aariciah/gpt2-arabic-20k-lc,aariciah,new,BPE,20000,2.8479,2.8061,0.1222,509512,162,False,0.0074,60.87,,2
+,aariciah__gpt2-arabic-20k-lc__gpt2-arabic-20k-lc-GPT2TokenizerFast,aariciah/gpt2-arabic-20k-lc/gpt2-arabic-20k-lc-GPT2TokenizerFast,aariciah,new,BPE,20000,2.8479,2.8061,0.1222,509512,19,False,0.0237,60.87,,1
+,aariciah__gpt2-arabic-20k,aariciah/gpt2-arabic-20k,aariciah,new,BPE,20000,2.8467,2.8074,0.1221,509730,3,False,0.0304,60.88,,2
+,zuhri025__urdu-finetune-clean__checkpoint-10000,zuhri025/urdu-finetune-clean/checkpoint-10000,zuhri025,new,BPE,17942,2.8362,1.9721,0.416,511611,0,False,0.0003,61.03,,10
+,Helsinki-NLP__opus-mt-tc-bible-big-ira-deu_eng_fra_por_spa,Helsinki-NLP/opus-mt-tc-bible-big-ira-deu_eng_fra_por_spa,Helsinki-NLP,new,Unigram,32000,2.8095,2.927,0.2936,516472,175,False,0.0,61.39,,2
+,aakashMeghwar01__SindhiLM-Tokenizer-v3,aakashMeghwar01/SindhiLM-Tokenizer-v3,aakashMeghwar01,new,Unigram,32000,2.7894,3.0285,0.1286,520199,0,False,0.1255,61.67,,1
+,alphaedge-ai__embeddinggemma-pan-16384,alphaedge-ai/embeddinggemma-pan-16384,alphaedge-ai,new,BPE,16384,2.7771,2.0494,0.3723,522494,0,False,0.3772,61.84,,1
+,alphaedge-ai__mmBERT-small-pan-32768,alphaedge-ai/mmBERT-small-pan-32768,alphaedge-ai,new,BPE,32768,2.6635,2.963,0.1469,544783,0,False,0.1094,63.4,,2
+,zuhri025__urdu-finetune-clean,zuhri025/urdu-finetune-clean,zuhri025,new,BPE,17942,2.6148,2.2334,0.2962,554919,463,False,0.0003,64.07,,4
+,neuralspace__indic-transformers-hi-distilbert,neuralspace/indic-transformers-hi-distilbert,neuralspace,new,WordPiece,30522,2.6143,3.2313,0.093,555042,0,False,0.0406,64.08,,1
+,cservan__multilingual-albert-base-cased-32k,cservan/multilingual-albert-base-cased-32k,cservan,new,Unigram,32000,2.6095,2.8863,0.1105,556050,169,False,0.0002,64.14,,1
+,alphaedge-ai__embeddinggemma-pan-32768,alphaedge-ai/embeddinggemma-pan-32768,alphaedge-ai,new,BPE,32768,2.5965,3.0076,0.1677,558831,0,False,0.1839,64.32,,1
+,unsloth__bge-small-en-v1.5,unsloth/bge-small-en-v1.5,unsloth,new,WordPiece,30522,2.5069,3.3697,0.0727,578806,0,False,0.0324,65.55,,61
+,alphaedge-ai__metaclip-2-worldwide-huge-378-pan-32768,alphaedge-ai/metaclip-2-worldwide-huge-378-pan-32768,alphaedge-ai,new,Unigram,32768,2.4408,2.9426,0.1722,594497,0,False,0.293,66.46,,6
+,Kashif786__sindhi-tokenizer-20k,Kashif786/sindhi-tokenizer-20k,Kashif786,new,Unigram,20000,2.4295,2.4855,0.185,597250,0,False,0.3082,66.62,,1
+,Kashif786__sindhi-tokenizer-20k-3Mdata,Kashif786/sindhi-tokenizer-20k-3Mdata,Kashif786,new,Unigram,20000,2.4239,2.4935,0.1853,598629,0,False,0.3075,66.69,,1
+,alphaedge-ai__mmBERT-small-pan-16384,alphaedge-ai/mmBERT-small-pan-16384,alphaedge-ai,new,BPE,16384,2.3697,2.744,0.1725,612320,0,False,0.343,67.44,,2
+,alphaedge-ai__bge-m3-pan-16384,alphaedge-ai/bge-m3-pan-16384,alphaedge-ai,new,Unigram,16384,2.3431,2.9924,0.1107,619285,0,False,0.3325,67.8,,1
+,kevinscaria__IndicBERT-hi-ml,kevinscaria/IndicBERT-hi-ml,kevinscaria,new,WordPiece,100000,2.3429,3.6057,0.0358,619341,0,False,0.0074,67.81,,1
+,iutsslir__doc2query-banglat5-indicmarco,iutsslir/doc2query-banglat5-indicmarco,iutsslir,new,Unigram,32100,2.3056,3.2517,0.0577,629344,0,False,0.2246,68.32,,4
+,alphaedge-ai__multilingual-e5-large-instruct-pan-16384,alphaedge-ai/multilingual-e5-large-instruct-pan-16384,alphaedge-ai,new,Unigram,16384,2.2847,3.0086,0.0925,635119,0,False,0.3431,68.61,,1
+,alphaedge-ai__multilingual-e5-base-pan-16384,alphaedge-ai/multilingual-e5-base-pan-16384,alphaedge-ai,new,Unigram,16384,2.2843,3.0091,0.0924,635225,0,False,0.3429,68.61,,3
+,rpii2023__Je_baat,rpii2023/Je_baat,rpii2023,new,BPE,36749,2.2721,3.7179,0.0089,638618,0,False,0.9963,68.78,,1
+,mbilalcr07__bol-urdu-tts__pretrained_models,mbilalcr07/bol-urdu-tts/pretrained_models,mbilalcr07,new,BPE,2454,2.2688,3.7234,0.0071,639565,0,False,0.0,68.82,,1
+,LGAI-EXAONE__EXAONE-Deep-32B,LGAI-EXAONE/EXAONE-Deep-32B,LGAI-EXAONE,new,BPE,102400,2.2372,3.7317,0.0074,648588,467,False,0.0,69.26,,6
+,alphaedge-ai__mt5-base-pan-32768,alphaedge-ai/mt5-base-pan-32768,alphaedge-ai,new,Unigram,32668,2.2132,3.4955,0.2297,655624,175,False,0.0,69.59,,2
+,HPLT__hplt_t5_base_3_0_ckb_Arab,HPLT/hplt_t5_base_3_0_ckb_Arab,HPLT,new,WordPiece,16384,2.0835,3.036,0.1939,696431,467,False,0.0,71.37,,1
+,Helsinki-NLP__opus-mt_tiny_zho-eng,Helsinki-NLP/opus-mt_tiny_zho-eng,Helsinki-NLP,new,Unigram,32000,2.0079,3.2167,0.0834,722652,0,False,0.2858,72.41,,1
+,aakashMeghwar01__SindhiLM-Tokenizer-v1,aakashMeghwar01/SindhiLM-Tokenizer-v1,aakashMeghwar01,new,Unigram,43,2.0028,3.3454,0.0286,724485,0,False,0.2582,72.48,,1
+,vaishnavi__indic-bert-512,vaishnavi/indic-bert-512,vaishnavi,new,Unigram,200000,1.9782,3.4125,0.0623,733494,0,False,0.2477,72.82,,4
+,HPLT__hplt_gpt_bert_base_3_0_ara_Arab,HPLT/hplt_gpt_bert_base_3_0_ara_Arab,HPLT,new,WordPiece,32768,1.9707,3.2681,0.1441,736296,467,False,0.0,72.92,,2
+,Helsinki-NLP__opus-mt-en-ur,Helsinki-NLP/opus-mt-en-ur,Helsinki-NLP,new,Unigram,32000,1.9041,3.4467,0.0586,762043,0,False,0.2262,73.83,,6
+,BAAI__BGE-VL-large,BAAI/BGE-VL-large,BAAI,new,BPE,49408,1.8499,4.5665,0.0047,784390,0,False,0.0,74.58,,4
+,Helsinki-NLP__opus-mt-en-iir,Helsinki-NLP/opus-mt-en-iir,Helsinki-NLP,new,Unigram,32000,1.8258,3.637,0.0116,794729,0,False,0.0236,74.91,,2
+,Helsinki-NLP__opus-mt-en-inc,Helsinki-NLP/opus-mt-en-inc,Helsinki-NLP,new,Unigram,32000,1.8223,3.6461,0.0116,796279,0,False,0.0287,74.96,,1
+,Michelle1606__konkani-r2d-indicbart,Michelle1606/konkani-r2d-indicbart,Michelle1606,new,Unigram,64014,1.8201,3.6554,0.0116,797236,0,False,0.0,74.99,,2
+,AvinabhDutta-Dev__bangla-paraphrase-indicbartss,AvinabhDutta-Dev/bangla-paraphrase-indicbartss,AvinabhDutta-Dev,new,Unigram,64015,1.8199,3.6557,0.0115,797291,0,False,0.0,74.99,,4
+,prthmgoyl__slm-indic-assamese-santali-pretrained,prthmgoyl/slm-indic-assamese-santali-pretrained,prthmgoyl,new,BPE,15000,1.8076,3.6906,0.0081,802722,0,False,0.0766,75.16,,1
+,Shoaibbaluch__balochi-translation-model,Shoaibbaluch/balochi-translation-model,Shoaibbaluch,new,Unigram,32000,1.8007,3.7012,0.0092,805831,0,False,0.0377,75.26,,1
+,ai4bharat__IndicBART,ai4bharat/IndicBART,ai4bharat,new,Unigram,64000,1.8004,3.7056,0.0116,805955,168,False,0.0002,75.26,,20
+,ai4bharat__IndicBARTSS,ai4bharat/IndicBARTSS,ai4bharat,new,Unigram,64000,1.8003,3.7059,0.0115,806013,169,False,0.0002,75.26,,33
+,Helsinki-NLP__opus-mt-en-ine,Helsinki-NLP/opus-mt-en-ine,Helsinki-NLP,new,Unigram,32000,1.7965,3.7126,0.0115,807701,0,False,0.0233,75.31,,2
+,Helsinki-NLP__opus-mt-en-mul,Helsinki-NLP/opus-mt-en-mul,Helsinki-NLP,new,Unigram,32000,1.7951,3.7161,0.0089,808313,0,False,0.0326,75.33,,6
+,aariciah__gpt2-english-dutch-merge,aariciah/gpt2-english-dutch-merge,aariciah,new,BPE,43790,1.7946,3.7142,0.0086,808545,0,False,0.1283,75.34,,1
+,aariciah__gpt2-english-baseline-dutch-configC-6k,aariciah/gpt2-english-baseline-dutch-configC-6k,aariciah,new,BPE,30000,1.7946,3.7142,0.0086,808552,0,False,0.1693,75.34,,1
+,aariciah__gpt2-english-20k-lc,aariciah/gpt2-english-20k-lc,aariciah,new,BPE,20000,1.7946,3.7143,0.0086,808566,0,False,0.1311,75.34,,3
+,aariciah__gpt2-english-dutch-first,aariciah/gpt2-english-dutch-first,aariciah,new,BPE,43790,1.7945,3.7144,0.0085,808577,0,False,0.1283,75.34,,1
+,aariciah__gpt2-turkish-20k-lc,aariciah/gpt2-turkish-20k-lc,aariciah,new,BPE,20000,1.7944,3.7143,0.0087,808636,0,False,0.1784,75.34,,3
+,aariciah__gpt2-turkish-dutch-first,aariciah/gpt2-turkish-dutch-first,aariciah,new,BPE,36484,1.7944,3.7147,0.0084,808636,0,False,0.1784,75.34,,1
+,aariciah__gpt2-turkish-dutch-merge,aariciah/gpt2-turkish-dutch-merge,aariciah,new,BPE,36484,1.7944,3.7144,0.0087,808656,0,False,0.1784,75.34,,1
+,aariciah__gpt2-turkish-20k,aariciah/gpt2-turkish-20k,aariciah,new,BPE,20000,1.7943,3.7147,0.0086,808696,0,False,0.1784,75.34,,4
+,aariciah__gpt2-arabic-dutch-routed,aariciah/gpt2-arabic-dutch-routed,aariciah,new,BPE,38724,1.7929,3.7187,0.0084,809324,162,False,0.0046,75.36,,1
+,aariciah__gpt2-dutch-20k-lc,aariciah/gpt2-dutch-20k-lc,aariciah,new,BPE,20000,1.7929,3.7187,0.0084,809324,0,False,0.2343,75.36,,16
+,aariciah__gpt2-persian-dutch-routed,aariciah/gpt2-persian-dutch-routed,aariciah,new,BPE,38466,1.7929,3.7187,0.0084,809324,1,False,0.0248,75.36,,1
+,aariciah__gpt2-portuguese-dutch-first,aariciah/gpt2-portuguese-dutch-first,aariciah,new,BPE,35797,1.7929,3.7188,0.0084,809339,0,False,0.1922,75.36,,1
+,aariciah__gpt2-chinese-dutch-first,aariciah/gpt2-chinese-dutch-first,aariciah,new,BPE,39293,1.7928,3.7188,0.0084,809341,0,False,0.2349,75.36,,1
+,aariciah__gpt2-spanish-dutch-first,aariciah/gpt2-spanish-dutch-first,aariciah,new,BPE,36363,1.7928,3.7188,0.0084,809342,0,False,0.2349,75.36,,1
+,aariciah__gpt2-japanese-dutch-first,aariciah/gpt2-japanese-dutch-first,aariciah,new,BPE,39304,1.7928,3.7188,0.0084,809346,0,False,0.2349,75.36,,1
+,aariciah__gpt2-german-dutch-first,aariciah/gpt2-german-dutch-first,aariciah,new,BPE,34890,1.7928,3.7189,0.0084,809349,0,False,0.2349,75.36,,1
+,aariciah__gpt2-portuguese-dutch-merge,aariciah/gpt2-portuguese-dutch-merge,aariciah,new,BPE,35797,1.7928,3.7189,0.0084,809352,0,False,0.1922,75.36,,1
+,aariciah__gpt2-russian-dutch-first,aariciah/gpt2-russian-dutch-first,aariciah,new,BPE,38574,1.7928,3.7189,0.0084,809352,0,False,0.2248,75.36,,1
+,aariciah__gpt2-spanish-dutch-merge,aariciah/gpt2-spanish-dutch-merge,aariciah,new,BPE,36363,1.7928,3.7189,0.0083,809353,0,False,0.2349,75.36,,1
+,aariciah__gpt2-dutch-20k,aariciah/gpt2-dutch-20k,aariciah,new,BPE,20000,1.7928,3.7189,0.0084,809357,0,False,0.2349,75.36,,2
+,aariciah__gpt2-german-dutch-merge,aariciah/gpt2-german-dutch-merge,aariciah,new,BPE,34890,1.7928,3.7189,0.0084,809360,0,False,0.2349,75.36,,1
+,aariciah__gpt2-english-20k,aariciah/gpt2-english-20k,aariciah,new,BPE,20000,1.7928,3.719,0.0084,809377,0,False,0.1746,75.37,,2
+,aariciah__gpt2-russian-dutch-merge,aariciah/gpt2-russian-dutch-merge,aariciah,new,BPE,38574,1.7928,3.719,0.0084,809384,0,False,0.2248,75.37,,1
+,aariciah__gpt2-japanese-dutch-merge,aariciah/gpt2-japanese-dutch-merge,aariciah,new,BPE,39304,1.7927,3.719,0.0083,809400,0,False,0.2348,75.37,,1
+,aariciah__gpt2-chinese-dutch-merge,aariciah/gpt2-chinese-dutch-merge,aariciah,new,BPE,39293,1.7927,3.7191,0.0083,809408,0,False,0.2348,75.37,,1
+,aariciah__gpt2-german-20k-lc,aariciah/gpt2-german-20k-lc,aariciah,new,BPE,20000,1.7926,3.7192,0.0084,809472,0,False,0.2684,75.37,,3
+,aariciah__gpt2-portuguese-20k-lc,aariciah/gpt2-portuguese-20k-lc,aariciah,new,BPE,20000,1.7925,3.7192,0.0083,809486,0,False,0.2526,75.37,,3
+,aariciah__gpt2-spanish-20k-lc,aariciah/gpt2-spanish-20k-lc,aariciah,new,BPE,20000,1.7925,3.7193,0.0083,809490,0,False,0.3548,75.37,,3
+,aariciah__gpt2-portuguese-20k,aariciah/gpt2-portuguese-20k,aariciah,new,BPE,20000,1.7924,3.7195,0.0082,809526,0,False,0.2928,75.37,,2
+,aariciah__gpt2-german-20k,aariciah/gpt2-german-20k,aariciah,new,BPE,20000,1.7924,3.7195,0.0083,809532,0,False,0.2684,75.37,,2
+,aariciah__gpt2-spanish-20k,aariciah/gpt2-spanish-20k,aariciah,new,BPE,20000,1.7924,3.7196,0.0082,809552,0,False,0.3594,75.37,,2
+,Helsinki-NLP__opus-mt-hi-ur,Helsinki-NLP/opus-mt-hi-ur,Helsinki-NLP,new,Unigram,4000,1.7923,3.7227,0.0077,809589,0,False,0.0195,75.37,,1
+,aariciah__gpt2-japanese-dutch-synsem,aariciah/gpt2-japanese-dutch-synsem,aariciah,new,BPE,6600,1.7922,3.7206,0.0079,809649,0,False,0.2348,75.37,,47
+,aariciah__gpt2-english-dutch-configC-6k__gpt2-english-configC-6k-GPT2TokenizerFast,aariciah/gpt2-english-dutch-configC-6k/gpt2-english-configC-6k-GPT2TokenizerFast,aariciah,new,BPE,6600,1.792,3.7207,0.0079,809736,0,False,0.1674,75.38,,1
+,aariciah__gpt2-turkish-configC-6k,aariciah/gpt2-turkish-configC-6k,aariciah,new,BPE,6600,1.7919,3.7208,0.0078,809753,0,False,0.1781,75.38,,2
+,prthmgoyl__slm-indic-bengali-pretrained,prthmgoyl/slm-indic-bengali-pretrained,prthmgoyl,new,BPE,10000,1.7919,3.721,0.0076,809763,162,False,0.0048,75.38,,1
+,aariciah__gpt2-russian-20k-lc,aariciah/gpt2-russian-20k-lc,aariciah,new,BPE,20000,1.7915,3.7202,0.0082,809957,0,False,0.227,75.38,,3
+,aariciah__gpt2-russian-configC-20k,aariciah/gpt2-russian-configC-20k,aariciah,new,BPE,20000,1.7914,3.7205,0.0081,810015,0,False,0.2278,75.38,,4
+,LingoIITGN__Ganga-2-1B,LingoIITGN/Ganga-2-1B,LingoIITGN,new,BPE,32768,1.7913,3.7223,0.0073,810029,0,False,0.4192,75.38,,2
+,aariciah__gpt2-chinese-20k-lc,aariciah/gpt2-chinese-20k-lc,aariciah,new,BPE,20000,1.7912,3.7206,0.0076,810066,0,False,0.79,75.39,,3
+,aariciah__gpt2-japanese-20k-lc,aariciah/gpt2-japanese-20k-lc,aariciah,new,BPE,20000,1.7912,3.7208,0.0076,810090,0,False,0.7899,75.39,,3
+,aariciah__gpt2-russian-configC-6k,aariciah/gpt2-russian-configC-6k,aariciah,new,BPE,6600,1.7907,3.7219,0.0077,810302,0,False,0.2269,75.39,,2
+,Omarrran__Kashmiri_Char_Tokenizer,Omarrran/Kashmiri_Char_Tokenizer,Omarrran,new,WordLevel,133,1.7889,3.7249,0.007,811115,0,False,0.0082,75.42,,2
+,HPLT__hplt_gpt_bert_base_4_0_lat_Latn,HPLT/hplt_gpt_bert_base_4_0_lat_Latn,HPLT,new,WordPiece,32768,1.7777,3.7335,0.0585,816224,467,False,0.0,75.57,,1
+,HPLT__hplt_gpt_bert_base_3_0_kmr_Latn,HPLT/hplt_gpt_bert_base_3_0_kmr_Latn,HPLT,new,WordPiece,32768,1.7595,3.7827,0.0804,824691,467,False,0.0,75.82,,2
+,dheeyantra__dhee-vagmi-core,dheeyantra/dhee-vagmi-core,dheeyantra,new,Unigram,64000,1.7469,4.3468,0.0783,830642,175,False,0.0,76.0,,1
+,roshanali0408__sindhi-transliterator,roshanali0408/sindhi-transliterator,roshanali0408,new,Unigram,250055,1.7232,3.6303,0.0155,842037,0,False,0.0,76.32,,1
+,HPLT__hplt_gpt_bert_base_3_0_swh_Latn,HPLT/hplt_gpt_bert_base_3_0_swh_Latn,HPLT,new,WordPiece,32768,1.6204,4.1948,0.0219,895474,467,False,0.0,77.73,,2
+,HPLT__hplt_gpt_bert_base_3_0_ind_Latn,HPLT/hplt_gpt_bert_base_3_0_ind_Latn,HPLT,new,WordPiece,32768,1.6163,4.208,0.0207,897741,467,False,0.0,77.79,,2
+,HPLT__hplt_bert_base_2_0_swh-Latn,HPLT/hplt_bert_base_2_0_swh-Latn,HPLT,new,WordPiece,32768,1.6111,4.2248,0.0205,900621,467,False,0.0,77.86,,1
+,Shyamkant__indic-gpt,Shyamkant/indic-gpt,Shyamkant,new,BPE,50000,1.6008,4.8547,0.0219,906441,176,False,0.0,78.0,,1
+,aashay96__indic-gpt,aashay96/indic-gpt,aashay96,new,BPE,60000,1.5867,5.0122,0.0087,914468,176,False,0.0,78.2,,1
+,HPLT__hplt_t5_base_3_0_fra_Latn,HPLT/hplt_t5_base_3_0_fra_Latn,HPLT,new,WordPiece,32768,1.5634,4.3848,0.0193,928102,467,False,0.0,78.52,,2
+,HPLT__hplt_t5_base_3_0_als_Latn,HPLT/hplt_t5_base_3_0_als_Latn,HPLT,new,WordPiece,32768,1.5592,4.3995,0.0052,930638,467,False,0.0,78.57,,2
+,claude-legacy,Xenova/claude-tokenizer,Anthropic,baseline,BPE,65000,1.5542,4.945,0.0063,933622,467,False,0.0,78.64,,1
+,HPLT__hplt_gpt_bert_base_3_0_tha_Thai,HPLT/hplt_gpt_bert_base_3_0_tha_Thai,HPLT,new,WordPiece,32768,1.5492,4.4344,0.0054,936633,467,False,0.0,78.71,,2
+,HPLT__hplt_gpt_bert_base_3_0_eng_Latn,HPLT/hplt_gpt_bert_base_3_0_eng_Latn,HPLT,new,WordPiece,32768,1.486,4.6662,0.0049,976441,467,False,0.0,79.58,,2
+,HPLT__hplt_gpt_bert_base_3_0_gle_Latn,HPLT/hplt_gpt_bert_base_3_0_gle_Latn,HPLT,new,WordPiece,32768,1.4857,4.6673,0.0057,976637,467,False,0.0,79.58,,2
+,HPLT__hplt_gpt_bert_base_3_0_heb_Hebr,HPLT/hplt_gpt_bert_base_3_0_heb_Hebr,HPLT,new,WordPiece,32768,1.4802,4.6884,0.005,980259,467,False,0.0,79.66,,2
+,HPLT__hplt_gpt_bert_base_4_0_bos_Latn,HPLT/hplt_gpt_bert_base_4_0_bos_Latn,HPLT,new,WordPiece,32768,1.4695,4.7301,0.005,987425,467,False,0.0,79.81,,1
+,HPLT__hplt_t5_base_3_0_bos_Latn,HPLT/hplt_t5_base_3_0_bos_Latn,HPLT,new,WordPiece,32768,1.466,4.744,0.0049,989807,467,False,0.0,79.86,,1
+,HPLT__hplt_gpt_bert_base_3_0_tur_Latn,HPLT/hplt_gpt_bert_base_3_0_tur_Latn,HPLT,new,WordPiece,32768,1.4615,4.7616,0.0049,992826,467,False,0.0,79.92,,2
+,HPLT__hplt_gpt_bert_base_3_0_hye_Armn,HPLT/hplt_gpt_bert_base_3_0_hye_Armn,HPLT,new,WordPiece,32768,1.4463,4.8222,0.0047,1003236,467,False,0.0,80.13,,2
+,HPLT__hplt_gpt_bert_base_3_0_kor_Hang,HPLT/hplt_gpt_bert_base_3_0_kor_Hang,HPLT,new,WordPiece,32768,1.4453,4.8262,0.005,1003929,467,False,0.0,80.14,,2
+,HPLT__hplt_gpt_bert_base_3_0_vie_Latn,HPLT/hplt_gpt_bert_base_3_0_vie_Latn,HPLT,new,WordPiece,32768,1.4436,4.8332,0.0065,1005128,467,False,0.0,80.16,,2
+,HPLT__hplt_gpt_bert_base_3_0_tam_Taml,HPLT/hplt_gpt_bert_base_3_0_tam_Taml,HPLT,new,WordPiece,32768,1.4388,4.8528,0.0074,1008499,467,False,0.0,80.23,,2
+,nvidia__NVIDIA-Nemotron-Parse-v1.2,nvidia/NVIDIA-Nemotron-Parse-v1.2,nvidia,new,BPE,52329,1.4277,5.1183,0.0047,1016370,467,False,0.0,80.38,,3
+,HPLT__hplt_gpt_bert_base_3_0_mlt_Latn,HPLT/hplt_gpt_bert_base_3_0_mlt_Latn,HPLT,new,WordPiece,32768,1.427,4.9014,0.0046,1016832,467,False,0.0,80.39,,2
+,HPLT__hplt_gpt_bert_base_3_0_spa_Latn,HPLT/hplt_gpt_bert_base_3_0_spa_Latn,HPLT,new,WordPiece,32768,1.4242,4.9129,0.0046,1018816,467,False,0.0,80.43,,2
+,HPLT__hplt_gpt_bert_base_3_0_bel_Cyrl,HPLT/hplt_gpt_bert_base_3_0_bel_Cyrl,HPLT,new,WordPiece,32768,1.4118,4.9651,0.0045,1027782,467,False,0.0,80.6,,2
+,HPLT__hplt_gpt_bert_base_3_0_ita_Latn,HPLT/hplt_gpt_bert_base_3_0_ita_Latn,HPLT,new,WordPiece,32768,1.399,5.0198,0.0046,1037181,467,False,0.0,80.78,,2
+,HPLT__hplt_gpt_bert_base_3_0_deu_Latn,HPLT/hplt_gpt_bert_base_3_0_deu_Latn,HPLT,new,WordPiece,32768,1.3894,5.0616,0.0046,1044357,467,False,0.0,80.91,,2
+,HPLT__hplt_gpt_bert_base_3_0_kat_Geor,HPLT/hplt_gpt_bert_base_3_0_kat_Geor,HPLT,new,WordPiece,32768,1.3853,5.0797,0.0044,1047469,467,False,0.0,80.96,,2
+,SIRIS-Lab__citation-parser-ENTITY,SIRIS-Lab/citation-parser-ENTITY,SIRIS-Lab,new,BPE,50265,1.3844,5.749,0.0051,1048149,0,False,0.0,80.98,,9
+,HPLT__hplt_gpt_bert_base_3_0_tat_Cyrl,HPLT/hplt_gpt_bert_base_3_0_tat_Cyrl,HPLT,new,WordPiece,32768,1.3833,5.0886,0.0045,1048994,467,False,0.0,80.99,,1
+,HPLT__hplt_gpt_bert_base_3_0_ron_Latn,HPLT/hplt_gpt_bert_base_3_0_ron_Latn,HPLT,new,WordPiece,32768,1.3732,5.1332,0.0044,1056660,467,False,0.0,81.13,,2
+,HPLT__hplt_gpt_bert_base_3_0_ltz_Latn,HPLT/hplt_gpt_bert_base_3_0_ltz_Latn,HPLT,new,WordPiece,32768,1.3362,5.3036,0.0043,1085928,467,False,0.0,81.64,,2
+,alphaedge-ai__mt5-base-pan-16384,alphaedge-ai/mt5-base-pan-16384,alphaedge-ai,new,Unigram,16284,1.3193,5.6465,0.0363,1099838,175,False,0.0,81.87,,2
+,HPLT__hplt_t5_base_3_0_cat_Latn,HPLT/hplt_t5_base_3_0_cat_Latn,HPLT,new,WordPiece,32768,1.3115,5.4228,0.0043,1106402,467,False,0.0,81.98,,2
+,HPLT__hplt_t5_base_3_0_hrv_Latn,HPLT/hplt_t5_base_3_0_hrv_Latn,HPLT,new,WordPiece,32768,1.3114,5.423,0.0043,1106433,467,False,0.0,81.98,,1
+,HPLT__hplt_gpt_bert_base_3_0_por_Latn,HPLT/hplt_gpt_bert_base_3_0_por_Latn,HPLT,new,WordPiece,32768,1.3058,5.451,0.0042,1111241,467,False,0.0,82.06,,2
+,HPLT__hplt_gpt_bert_base_3_0_cym_Latn,HPLT/hplt_gpt_bert_base_3_0_cym_Latn,HPLT,new,WordPiece,32768,1.3058,5.451,0.0043,1111245,467,False,0.0,82.06,,2
+,HPLT__hplt_gpt_bert_base_3_0_mkd_Cyrl,HPLT/hplt_gpt_bert_base_3_0_mkd_Cyrl,HPLT,new,WordPiece,32768,1.276,5.602,0.004,1137184,467,False,0.0,82.47,,2
+,HPLT__hplt_t5_base_3_0_ast_Latn,HPLT/hplt_t5_base_3_0_ast_Latn,HPLT,new,WordPiece,16384,1.2501,5.7392,0.004,1160745,467,False,0.0,82.82,,1
+,HPLT__hplt_gpt_bert_base_3_0_bul_Cyrl,HPLT/hplt_gpt_bert_base_3_0_bul_Cyrl,HPLT,new,WordPiece,32768,1.2492,5.7438,0.0038,1161534,467,False,0.0,82.83,,2
+,HPLT__hplt_gpt_bert_base_3_0_nld_Latn,HPLT/hplt_gpt_bert_base_3_0_nld_Latn,HPLT,new,WordPiece,32768,1.244,5.7723,0.0039,1166436,467,False,0.0,82.91,,2
+,HPLT__hplt_gpt_bert_base_3_0_swe_Latn,HPLT/hplt_gpt_bert_base_3_0_swe_Latn,HPLT,new,WordPiece,32768,1.2332,5.8315,0.0038,1176607,467,False,0.0,83.05,,2
+,HPLT__hplt_t5_base_3_0_lim_Latn,HPLT/hplt_t5_base_3_0_lim_Latn,HPLT,new,WordPiece,16384,1.2254,5.8753,0.0038,1184122,467,False,0.0,83.16,,1
+,NareshSandrugu__indic-gpt,NareshSandrugu/indic-gpt,NareshSandrugu,new,BPE,60000,1.2202,5.9515,0.0045,1189182,0,False,0.0,83.23,,2
+,HPLT__hplt_t5_base_3_0_srp_Cyrl,HPLT/hplt_t5_base_3_0_srp_Cyrl,HPLT,new,WordPiece,32768,1.1965,6.0419,0.0036,1212748,467,False,0.0,83.56,,2
+,HPLT__hplt_gpt_bert_base_3_0_lit_Latn,HPLT/hplt_gpt_bert_base_3_0_lit_Latn,HPLT,new,WordPiece,32768,1.1649,6.2333,0.0036,1245618,467,False,0.0,83.99,,2
+,HPLT__hplt_gpt_bert_base_3_0_nno_Latn,HPLT/hplt_gpt_bert_base_3_0_nno_Latn,HPLT,new,WordPiece,32768,1.1597,6.2661,0.0021,1251249,467,False,0.0,84.06,,2
+,indicnode__siglip-base-patch16-224,indicnode/siglip-base-patch16-224,indicnode,new,Unigram,32000,1.1567,6.3694,0.0034,1254507,0,False,0.0,84.11,,1
+,HPLT__hplt_gpt_bert_base_3_0_nob_Latn,HPLT/hplt_gpt_bert_base_3_0_nob_Latn,HPLT,new,WordPiece,32768,1.1255,6.4869,0.002,1289178,467,False,0.0,84.53,,2
+,HPLT__hplt_gpt_bert_base_3_0_jpn_Jpan,HPLT/hplt_gpt_bert_base_3_0_jpn_Jpan,HPLT,new,WordPiece,32768,1.1132,6.5703,0.0034,1303511,467,False,0.0,84.7,,2
+,HPLT__hplt_gpt_bert_base_3_0_ukr_Cyrl,HPLT/hplt_gpt_bert_base_3_0_ukr_Cyrl,HPLT,new,WordPiece,32768,1.0993,6.6658,0.0017,1319912,467,False,0.0,84.89,,2
+,HPLT__hplt_gpt_bert_base_3_0_rus_Cyrl,HPLT/hplt_gpt_bert_base_3_0_rus_Cyrl,HPLT,new,WordPiece,32768,1.0818,6.7906,0.0017,1341350,467,False,0.0,85.14,,2
+,HPLT__hplt_t5_base_3_0_dan_Latn,HPLT/hplt_t5_base_3_0_dan_Latn,HPLT,new,WordPiece,32768,1.0812,6.7948,0.0018,1342065,467,False,0.0,85.14,,2
+,HPLT__hplt_gpt_bert_base_3_0_ekk_Latn,HPLT/hplt_gpt_bert_base_3_0_ekk_Latn,HPLT,new,WordPiece,32768,1.0812,6.7949,0.0017,1342084,467,False,0.0,85.14,,2
+,HPLT__hplt_gpt_bert_base_3_0_isl_Latn,HPLT/hplt_gpt_bert_base_3_0_isl_Latn,HPLT,new,WordPiece,32768,1.0812,6.7949,0.0018,1342084,467,False,0.0,85.14,,2
+,HPLT__hplt_t5_base_3_0_glg_Latn,HPLT/hplt_t5_base_3_0_glg_Latn,HPLT,new,WordPiece,32768,1.0812,6.7949,0.0018,1342089,467,False,0.0,85.14,,2
+,HPLT__hplt_gpt_bert_base_3_0_eus_Latn,HPLT/hplt_gpt_bert_base_3_0_eus_Latn,HPLT,new,WordPiece,32768,1.0812,6.795,0.0017,1342094,467,False,0.0,85.14,,2
+,HPLT__hplt_gpt_bert_base_3_0_ces_Latn,HPLT/hplt_gpt_bert_base_3_0_ces_Latn,HPLT,new,WordPiece,32768,1.0812,6.795,0.0017,1342103,467,False,0.0,85.14,,2
+,HPLT__hplt_gpt_bert_base_3_0_ell_Grek,HPLT/hplt_gpt_bert_base_3_0_ell_Grek,HPLT,new,WordPiece,32768,1.0812,6.795,0.0017,1342105,467,False,0.0,85.14,,2
+,HPLT__hplt_gpt_bert_base_3_0_pol_Latn,HPLT/hplt_gpt_bert_base_3_0_pol_Latn,HPLT,new,WordPiece,32768,1.0812,6.795,0.0017,1342105,467,False,0.0,85.14,,2
+,HPLT__hplt_gpt_bert_base_3_0_slv_Latn,HPLT/hplt_gpt_bert_base_3_0_slv_Latn,HPLT,new,WordPiece,32768,1.0812,6.795,0.0017,1342106,467,False,0.0,85.14,,2
+,HPLT__hplt_gpt_bert_base_3_0_hun_Latn,HPLT/hplt_gpt_bert_base_3_0_hun_Latn,HPLT,new,WordPiece,32768,1.0641,6.9202,0.0016,1363601,467,False,0.0,85.38,,2
+,HPLT__hplt_t5_base_3_0_slk_Latn,HPLT/hplt_t5_base_3_0_slk_Latn,HPLT,new,WordPiece,32768,1.0641,6.9202,0.0016,1363606,467,False,0.0,85.38,,2
+,HPLT__hplt_gpt_bert_base_3_0_lvs_Latn,HPLT/hplt_gpt_bert_base_3_0_lvs_Latn,HPLT,new,WordPiece,32768,1.0641,6.9203,0.0016,1363617,467,False,0.0,85.38,,2
+,HPLT__hplt_gpt_bert_small_3_0_ltg_Latn,HPLT/hplt_gpt_bert_small_3_0_ltg_Latn,HPLT,new,WordPiece,16384,1.064,6.9207,0.0015,1363687,467,False,0.0,85.38,,1
+,Helsinki-NLP__opus-mt-tc-bible-big-deu_eng_fra_por_spa-inc,Helsinki-NLP/opus-mt-tc-bible-big-deu_eng_fra_por_spa-inc,Helsinki-NLP,new,Unigram,32000,1.0066,7.4025,0.0027,1441521,175,False,0.0,86.17,,2
+,Helsinki-NLP__opus-mt-tc-bible-big-deu_eng_fra_por_spa-iir,Helsinki-NLP/opus-mt-tc-bible-big-deu_eng_fra_por_spa-iir,Helsinki-NLP,new,Unigram,32000,1.0066,7.4025,0.0027,1441526,175,False,0.0,86.17,,2
+,Helsinki-NLP__opus-mt-tc-bible-big-deu_eng_fra_por_spa-mul,Helsinki-NLP/opus-mt-tc-bible-big-deu_eng_fra_por_spa-mul,Helsinki-NLP,new,Unigram,32000,1.0065,7.4029,0.0026,1441588,175,False,0.0,86.17,,2
+,Helsinki-NLP__opus-mt-tc-bible-big-deu_eng_fra_por_spa-ine,Helsinki-NLP/opus-mt-tc-bible-big-deu_eng_fra_por_spa-ine,Helsinki-NLP,new,Unigram,32000,1.0065,7.4026,0.0027,1441614,175,False,0.0,86.17,,2
+,BSC-LT__MrBERT-ca,BSC-LT/MrBERT-ca,BSC-LT,new,BPE,50304,1.0062,7.4004,0.0012,1442111,482,False,0.0,86.17,,1
+,Helsinki-NLP__opus-mt-eo-caenes,Helsinki-NLP/opus-mt-eo-caenes,Helsinki-NLP,new,Unigram,32000,1.0035,7.4233,0.0026,1445949,175,False,0.0,86.21,,2
+,HPLT__hplt_gpt_bert_base_3_0_fin_Latn,HPLT/hplt_gpt_bert_base_3_0_fin_Latn,HPLT,new,WordPiece,32768,1.0024,7.4086,0.0012,1447507,467,False,0.0,86.23,,2
+,HPLT__hplt_gpt_bert_base_3_0_fao_Latn,HPLT/hplt_gpt_bert_base_3_0_fao_Latn,HPLT,new,WordPiece,16384,1.0024,7.4089,0.0011,1447543,467,False,0.0,86.23,,2
+,HPLT__hplt_gpt_bert_base_4_0_hrv_Latn,HPLT/hplt_gpt_bert_base_4_0_hrv_Latn,HPLT,new,WordPiece,32768,1.0024,7.4089,0.0011,1447544,467,False,0.0,86.23,,1
+,HPLT__hplt_gpt_bert_base_4_0_srp_Latn,HPLT/hplt_gpt_bert_base_4_0_srp_Latn,HPLT,new,WordPiece,32768,1.0024,7.4089,0.0011,1447553,467,False,0.0,86.23,,1
+,HPLT__hplt_gpt_bert_base_3_0_cmn_Hans,HPLT/hplt_gpt_bert_base_3_0_cmn_Hans,HPLT,new,WordPiece,32768,0.9974,7.4088,0.0011,1454781,467,False,0.0,86.29,,2

eval/dev_lm_confirm_ranking.csv

@@ -0,0 +1,19 @@
+rank,id,origin,algorithm,pretokenizer,data_mix,vocab,encoder_kind,params_total,params_non_embedding,ctx_tokens,n_seeds,mean_bpb,seed_sd_rel_pct,delta_vs_baseline_pct,delta_vs_baseline_ci95_lo,delta_vs_baseline_ci95_hi,p_vs_baseline,delta_vs_best_pct,delta_vs_best_ci95_lo,delta_vs_best_ci95_hi,holm_p_vs_best,in_top_set,stage3_mean_bpb,hf_native_exact,dev_bytes_per_token,learned_tokens,train_D1_freq_lt20,train_D1_freq_lt20_pct,train_D1_freq_eq0,robustness_seg_change_affected_mean,gates_pass
+1,R2-A4-SPnat-D2-32k,round2_exploratory,SentencePiece Unigram (native),sp-native,D2,32768,sentencepiece,8112192,1772928,249,5,1.225836,0.226655,-1.623655,-2.107054,-1.236059,0.0,,,,,True,1.409353,False,7.076413,32441,13182,40.633766,2,0.589154,True
+2,R2-A10-MinGram-P1r3-D2-48k,round2_exploratory,"MinGram (reimpl., HF Unigram)",P1r3,D2,49152,hf,11256960,1772928,244,5,1.226358,0.150669,-1.58176,-1.867521,-1.355869,0.0,0.042586,-0.400383,0.567773,0.8366,True,1.390329,True,7.223664,48831,27246,55.796523,1,0.294551,True
+3,R2-A10-MinGram-P1r3-D2-32k,round2_exploratory,"MinGram (reimpl., HF Unigram)",P1r3,D2,32768,hf,8112384,1772928,250,5,1.228443,0.206635,-1.414409,-1.66217,-1.19142,0.0,0.212699,-0.17228,0.645863,0.58,True,1.395279,True,7.125844,32447,11329,34.9154,0,0.316108,True
+4,R2-A4-SPnat-D2-48k,round2_exploratory,SentencePiece Unigram (native),sp-native,D2,49152,sentencepiece,11257152,1772928,245,5,1.232045,0.377674,-1.125362,-1.686493,-0.639227,0.0,0.506517,0.136094,0.865183,0.0234,False,1.407112,False,7.134636,48792,28858,59.144942,58,0.597896,True
+5,R2-A1-P1r3-D2-48k,round2_exploratory,byte-level BPE (HF),P1r3,D2,49152,hf,11257152,1772928,245,5,1.233741,0.167479,-0.98924,-1.211704,-0.778742,0.0,0.644886,0.252925,1.118305,0.008,False,1.397584,True,7.201747,48832,27273,55.850672,386,0.382379,True
+6,A1-P1r3-D2-32k,round1_preregistered_rank,byte-level BPE (HF),P1r3,D2,32768,hf,8112576,1772928,251,5,1.235969,0.073886,-0.810497,-1.000743,-0.657493,0.0,0.826579,0.466083,1.259031,0.0,False,1.400393,True,7.094891,32448,11512,35.478304,172,0.426602,True
+7,A4-SPnat-D1-16k,round1_preregistered_rank,SentencePiece Unigram (native),sp-native,D1,16384,sentencepiece,4966080,1772928,247,5,1.239248,0.807133,-0.547358,-1.300316,0.229794,0.176,1.094061,0.427119,1.82523,0.004,False,1.421296,False,6.866849,16061,68,0.423386,0,0.50312,True
+8,A10-MinGram-P1-D1-16k,round1_preregistered_rank,"MinGram (reimpl., HF Unigram)",P1,D1,16384,hf,4967808,1772928,256,5,1.241801,0.203434,-0.342444,-0.567013,-0.10292,0.0046,1.302356,0.912046,1.790367,0.0,False,1.418708,True,6.770172,16063,66,0.410882,0,0.204898,True
+9,A1-P1r3-D2-16k,round1_preregistered_rank,byte-level BPE (HF),P1r3,D2,16384,hf,4969536,1772928,265,5,1.246068,0.09998,,,,,1.650452,1.251529,2.152406,0.0,False,1.422536,True,6.788263,16064,577,3.591882,33,0.425791,True
+10,A1-P1-D1-16k,round1_added_reference_arm,byte-level BPE (HF),P1,D1,16384,hf,4968192,1772928,258,5,1.254297,0.164791,0.660407,0.459539,0.859504,0.0,2.321759,1.942494,2.804053,0.0,False,1.432781,True,6.711735,16064,447,2.78262,25,0.291955,True
+11,A7-PickyBPE-P1-D1-16k-tau0.9,round1_preregistered_rank,"PickyBPE (reimpl., custom encoder)",P1,D1,16384,custom,4968192,1772928,258,5,1.254461,0.102701,0.673542,0.506137,0.843774,0.0,2.335111,1.973769,2.771723,0.0,False,1.435245,False,6.718754,16064,249,1.55005,0,0.297051,True
+12,A1-P1r3-D1-16k,round1_added_reference_arm,byte-level BPE (HF),P1r3,D1,16384,hf,4968000,1772928,257,5,1.255319,0.176777,0.742422,0.514271,0.97213,0.0,2.405127,1.983783,2.958974,0.0,False,1.437665,True,6.722691,16064,434,2.701693,24,0.409736,True
+13,A1-P1r3-D2-8k,round1_preregistered_rank,byte-level BPE (HF),P1r3,D2,8192,hf,3400704,1772928,286,5,1.256723,0.175271,0.855065,0.371836,1.288044,0.001,2.51963,1.87704,3.244172,0.0,False,1.462345,True,6.307629,7872,125,1.587907,15,0.35145,True
+14,A3-SPnat-D1-16k,round1_preregistered_rank,SentencePiece BPE (native),sp-native,D1,16384,sentencepiece,4966080,1772928,247,5,1.266513,1.463571,1.640794,0.43226,2.962861,0.005,3.318327,2.150559,4.614722,0.0,False,1.446105,True,6.876486,15812,405,2.561346,9,0.488926,True
+15,R2-A6-SBPE-P1r3-D2-32k-t080,round2_exploratory,SuperBPE (reimpl.),P1r3,D2,32768,hf,8108544,1772928,230,5,1.273754,0.124415,2.221863,1.820238,2.614962,0.0,3.908986,3.5254,4.321494,0.0,False,1.392271,True,8.399018,32448,6057,18.66679,119,0.482695,True
+16,A6-SBPE-P1-D1-16k-t090,round1_preregistered_rank,SuperBPE (reimpl.),P1,D1,16384,hf,4965120,1772928,242,5,1.274849,0.196683,2.30974,2.023357,2.629583,0.0,3.998313,3.618359,4.472349,0.0,False,1.438367,True,7.358997,16064,369,2.297062,22,0.330562,True
+,A6-SBPE-P1-D1-32k-t080,round1_preregistered_rank,SuperBPE (reimpl.),P1,D1,32768,hf,8106624,1772928,220,5,1.2796,0.127255,2.691031,2.251724,3.106462,0.0,4.385897,4.038769,4.776852,,False,1.405332,True,8.189968,32448,1838,5.664448,68,0.351923,True
+17,A6-SBPE-P1-D1-16k-t080,round1_preregistered_rank,SuperBPE (reimpl.),P1,D1,16384,hf,4964736,1772928,240,5,1.286919,0.320969,3.278394,2.875843,3.712525,0.0,4.982954,4.544152,5.522518,0.0,False,1.438769,True,7.465105,16064,326,2.029382,19,0.29576,True

eval/dev_lm_large.csv

@@ -0,0 +1,6 @@
+rank_large,id,mean_bpb,seed1_bpb,seed2_bpb,delta_vs_baseline_pct,delta_vs_baseline_ci95_pct_lo,delta_vs_baseline_ci95_pct_hi,p_vs_baseline,delta_vs_released_pct,delta_vs_released_ci95_pct_lo,delta_vs_released_ci95_pct_hi,p_vs_released,confirm_mean_bpb,confirm_rank,in_confirm_top_set,n_vocab,ctx_tokens,params_total,params_non_embedding,steps,epochs_seen,lr,device,amp_dtype
+1,R2-A4-SPnat-D2-32k,1.133735,1.133186,1.134283,-1.335532,-1.775676,-1.026276,0.0,0.0,,,,1.225836,1,True,32768,249,23305344,10626816,7325,1.988243,0.0005,Tesla T4,fp16
+2,R2-A10-MinGram-P1r3-D2-32k,1.140865,1.140158,1.141572,-0.715001,-0.910685,-0.524984,0.0,0.628931,0.306563,1.05415,0.0004,1.228443,3,True,32768,250,23305728,10626816,7325,1.993469,0.0005,Tesla T4,fp16
+3,R2-A10-MinGram-P1r3-D2-48k,1.144066,1.143357,1.144775,-0.436436,-0.622803,-0.286684,0.0,0.911266,0.560733,1.365554,0.0,1.226358,2,True,49152,244,29594880,10626816,7325,1.988087,0.0005,Tesla T4,fp16
+4,A1-P1r3-D2-16k,1.149081,1.148993,1.149168,0.0,,,,1.35361,1.036917,1.807776,0.0,1.246068,9,False,16384,265,17020032,10626816,7325,1.992175,0.0005,Tesla T4,fp16
+5,A1-P1r3-D2-32k,1.149395,1.15108,1.147711,0.027361,-0.296706,0.317218,0.908,1.381342,1.023198,1.820087,0.0,1.235969,6,False,32768,251,23306112,10626816,7325,1.989579,0.0005,Tesla T4,fp16

eval/examples_segmentations.json

@@ -0,0 +1,1354 @@
+{
+ "what": "Three natural sentences from the strict test split (held out from tokenizer training), segmented by each tokenizer's own encoder, no special tokens. '▁' = space carried by the token; '▯' = one byte of a letter split across tokens (byte-level BPE). Source: sota/robustness/robustness.json -> examples.",
+ "sentences": [
+  "ایہہ بہت کہٹ لوک جانڑدین کہ ڈرامے دی پیدائش کسراں تے کتھے ہوئی آئی۔",
+  "اج اساں اگر آپڑیں بزرگاں دی انہاں گلاں تے غور نہ کیتا تے پچھتاوے دے سوا کج ہتھ نہ آسی۔",
+  "اسی طراں خیبر پختونخوا اچ بولی جانڑیں ولی دیگر زباناں دے ادب و ثقافت دی بی نمائندگی کیتی گئی اَئی۔"
+ ],
+ "tokenizers": {
+  "released": "Hindko SP-32k (released)",
+  "mingram48k": "MinGram-48k (runner-up)",
+  "gpt-4o": "GPT-4o (o200k)",
+  "gemma-3": "Gemma 3",
+  "llama-3": "Llama 3",
+  "llama-4": "Llama 4",
+  "qwen-3.5": "Qwen 3.5",
+  "deepseek-v3": "DeepSeek-V3",
+  "bloom": "BLOOM",
+  "roberta-urdu": "RoBERTa-Urdu",
+  "mt5": "mT5"
+ },
+ "segmentations": {
+  "released": [
+   {
+    "n": 14,
+    "tokens": [
+     "▁ایہہ",
+     "▁بہت",
+     "▁کہٹ",
+     "▁لوک",
+     "▁جانڑدین",
+     "▁کہ",
+     "▁ڈرامے",
+     "▁دی",
+     "▁پیدائش",
+     "▁کسراں",
+     "▁تے",
+     "▁کتھے",
+     "▁ہوئی",
+     "▁آئی۔"
+    ]
+   },
+   {
+    "n": 20,
+    "tokens": [
+     "▁اج",
+     "▁اساں",
+     "▁اگر",
+     "▁آپڑیں",
+     "▁بزرگاں",
+     "▁دی",
+     "▁انہاں",
+     "▁گلاں",
+     "▁تے",
+     "▁غور",
+     "▁نہ",
+     "▁کیتا",
+     "▁تے",
+     "▁پچھتاوے",
+     "▁دے",
+     "▁سوا",
+     "▁کج",
+     "▁ہتھ",
+     "▁نہ",
+     "▁آسی۔"
+    ]
+   },
+   {
+    "n": 20,
+    "tokens": [
+     "▁اسی",
+     "▁طراں",
+     "▁خیبر",
+     "▁پختونخوا",
+     "▁اچ",
+     "▁بولی",
+     "▁جانڑیں",
+     "▁ولی",
+     "▁دیگر",
+     "▁زباناں",
+     "▁دے",
+     "▁ادب",
+     "▁و",
+     "▁ثقافت",
+     "▁دی",
+     "▁بی",
+     "▁نمائندگی",
+     "▁کیتی",
+     "▁گئی",
+     "▁اَئی۔"
+    ]
+   }
+  ],
+  "mingram48k": [
+   {
+    "n": 15,
+    "tokens": [
+     "ایہہ",
+     "▁بہت",
+     "▁کہٹ",
+     "▁لوک",
+     "▁جانڑدین",
+     "▁کہ",
+     "▁ڈرامے",
+     "▁دی",
+     "▁پیدائش",
+     "▁کسراں",
+     "▁تے",
+     "▁کتھے",
+     "▁ہوئی",
+     "▁آئی",
+     "۔"
+    ]
+   },
+   {
+    "n": 21,
+    "tokens": [
+     "اج",
+     "▁اساں",
+     "▁اگر",
+     "▁آپڑیں",
+     "▁بزرگاں",
+     "▁دی",
+     "▁انہاں",
+     "▁گلاں",
+     "▁تے",
+     "▁غور",
+     "▁نہ",
+     "▁کیتا",
+     "▁تے",
+     "▁پچھتاوے",
+     "▁دے",
+     "▁سوا",
+     "▁کج",
+     "▁ہتھ",
+     "▁نہ",
+     "▁آسی",
+     "۔"
+    ]
+   },
+   {
+    "n": 21,
+    "tokens": [
+     "اسی",
+     "▁طراں",
+     "▁خیبر",
+     "▁پختونخوا",
+     "▁اچ",
+     "▁بولی",
+     "▁جانڑیں",
+     "▁ولی",
+     "▁دیگر",
+     "▁زباناں",
+     "▁دے",
+     "▁ادب",
+     "▁و",
+     "▁ثقافت",
+     "▁دی",
+     "▁بی",
+     "▁نمائندگی",
+     "▁کیتی",
+     "▁گئی",
+     "▁اَئی",
+     "۔"
+    ]
+   }
+  ],
+  "gpt-4o": [
+   {
+    "n": 29,
+    "tokens": [
+     "ای",
+     "ہ",
+     "ہ",
+     "▁بہت",
+     "▁کہ",
+     "ٹ",
+     "▁لو",
+     "ک",
+     "▁جان",
+     "ڑ",
+     "د",
+     "ین",
+     "▁کہ",
+     "▁ڈ",
+     "رام",
+     "ے",
+     "▁دی",
+     "▁پید",
+     "ائش",
+     "▁کس",
+     "را",
+     "ں",
+     "▁ت",
+     "ے",
+     "▁کت",
+     "ھے",
+     "▁ہوئی",
+     "▁آئی",
+     "۔"
+    ]
+   },
+   {
+    "n": 39,
+    "tokens": [
+     "اج",
+     "▁اس",
+     "اں",
+     "▁اگر",
+     "▁آپ",
+     "ڑ",
+     "یں",
+     "▁بزرگ",
+     "اں",
+     "▁دی",
+     "▁انہ",
+     "اں",
+     "▁گ",
+     "لا",
+     "ں",
+     "▁ت",
+     "ے",
+     "▁غور",
+     "▁نہ",
+     "▁کی",
+     "تا",
+     "▁ت",
+     "ے",
+     "▁پ",
+     "چھ",
+     "تا",
+     "و",
+     "ے",
+     "▁دے",
+     "▁س",
+     "وا",
+     "▁ک",
+     "ج",
+     "▁ہ",
+     "ت",
+     "ھ",
+     "▁نہ",
+     "▁آسی",
+     "۔"
+    ]
+   },
+   {
+    "n": 38,
+    "tokens": [
+     "اسی",
+     "▁ط",
+     "را",
+     "ں",
+     "▁خی",
+     "بر",
+     "▁پ",
+     "خت",
+     "ون",
+     "خوا",
+     "▁اچ",
+     "▁ب��ل",
+     "ی",
+     "▁جان",
+     "ڑ",
+     "یں",
+     "▁ولی",
+     "▁دیگر",
+     "▁ز",
+     "ب",
+     "انا",
+     "ں",
+     "▁دے",
+     "▁ادب",
+     "▁و",
+     "▁ثق",
+     "افت",
+     "▁دی",
+     "▁بی",
+     "▁نمائ",
+     "ندگی",
+     "▁کی",
+     "تی",
+     "▁گئی",
+     "▁ا",
+     "َ",
+     "ئی",
+     "۔"
+    ]
+   }
+  ],
+  "gemma-3": [
+   {
+    "n": 27,
+    "tokens": [
+     "ایہ",
+     "ہ",
+     "▁بہت",
+     "▁کہ",
+     "ٹ",
+     "▁لو",
+     "ک",
+     "▁جان",
+     "ڑ",
+     "د",
+     "ین",
+     "▁کہ",
+     "▁ڈ",
+     "رام",
+     "ے",
+     "▁دی",
+     "▁پید",
+     "ائش",
+     "▁کس",
+     "را",
+     "ں",
+     "▁تے",
+     "▁کت",
+     "ھے",
+     "▁ہوئی",
+     "▁آئی",
+     "۔"
+    ]
+   },
+   {
+    "n": 36,
+    "tokens": [
+     "اج",
+     "▁اس",
+     "اں",
+     "▁اگر",
+     "▁آپ",
+     "ڑ",
+     "یں",
+     "▁بزرگ",
+     "اں",
+     "▁دی",
+     "▁انہ",
+     "اں",
+     "▁گ",
+     "لا",
+     "ں",
+     "▁تے",
+     "▁غور",
+     "▁نہ",
+     "▁کی",
+     "تا",
+     "▁تے",
+     "▁پ",
+     "چھ",
+     "تا",
+     "و",
+     "ے",
+     "▁دے",
+     "▁سوا",
+     "▁ک",
+     "ج",
+     "▁ہ",
+     "ت",
+     "ھ",
+     "▁نہ",
+     "▁آسی",
+     "۔"
+    ]
+   },
+   {
+    "n": 35,
+    "tokens": [
+     "اسی",
+     "▁ط",
+     "را",
+     "ں",
+     "▁خی",
+     "بر",
+     "▁پ",
+     "خت",
+     "ون",
+     "خوا",
+     "▁اچ",
+     "▁بولی",
+     "▁جان",
+     "ڑ",
+     "یں",
+     "▁ولی",
+     "▁دیگر",
+     "▁زبان",
+     "اں",
+     "▁دے",
+     "▁ادب",
+     "▁و",
+     "▁ثقاف",
+     "ت",
+     "▁دی",
+     "▁بی",
+     "▁نمائ",
+     "ندگی",
+     "▁کی",
+     "تی",
+     "▁گئی",
+     "▁ا",
+     "َ",
+     "ئی",
+     "۔"
+    ]
+   }
+  ],
+  "llama-3": [
+   {
+    "n": 58,
+    "tokens": [
+     "ای",
+     "▯",
+     "▯",
+     "▯",
+     "▯",
+     "▁ب",
+     "▯",
+     "▯",
+     "ت",
+     "▁ک",
+     "▯",
+     "▯",
+     "ٹ",
+     "▁لو",
+     "ک",
+     "▁جان",
+     "▯",
+     "▯",
+     "د",
+     "ین",
+     "▁ک",
+     "▯",
+     "▯",
+     "▁",
+     "▯",
+     "▯",
+     "ر",
+     "ام",
+     "▯",
+     "▯",
+     "▁دی",
+     "▁پ",
+     "ید",
+     "ائ",
+     "ش",
+     "▁ک",
+     "سر",
+     "ا",
+     "▯",
+     "▯",
+     "▁ت",
+     "▯",
+     "▯",
+     "▁ک",
+     "ت",
+     "ھ",
+     "▯",
+     "▯",
+     "▁",
+     "▯",
+     "▯",
+     "وئ",
+     "ی",
+     "▁آ",
+     "ئ",
+     "ی",
+     "▯",
+     "▯"
+    ]
+   },
+   {
+    "n": 66,
+    "tokens": [
+     "اج",
+     "▁اس",
+     "ا",
+     "▯",
+     "▯",
+     "▁اگر",
+     "▁آپ",
+     "▯",
+     "▯",
+     "ی",
+     "▯",
+     "▯",
+     "▁بزرگ",
+     "ا",
+     "▯",
+     "▯",
+     "▁دی",
+     "▁ان",
+     "▯",
+     "▯",
+     "ا",
+     "▯",
+     "▯",
+     "▁گ",
+     "لا",
+     "▯",
+     "▯",
+     "▁ت",
+     "▯",
+     "▯",
+     "▁غ",
+     "ور",
+     "▁ن",
+     "▯",
+     "▯",
+     "▁ک",
+     "یت",
+     "ا",
+     "▁ت",
+     "▯",
+     "▯",
+     "▁پ",
+     "چ",
+     "ھ",
+     "ت",
+     "او",
+     "▯",
+     "▯",
+     "▁د",
+     "▯",
+     "▯",
+     "▁س",
+     "وا",
+     "▁ک",
+     "ج",
+     "▁",
+     "▯",
+     "▯",
+     "ت",
+     "ھ",
+     "▁ن",
+     "▯",
+     "▯",
+     "▁آسی",
+     "▯",
+     "▯"
+    ]
+   },
+   {
+    "n": 53,
+    "tokens": [
+     "اسی",
+     "▁طر",
+     "ا",
+     "▯",
+     "▯",
+     "▁خی",
+     "بر",
+     "▁پ",
+     "خت",
+     "ون",
+     "خ",
+     "وا",
+     "▁ا",
+     "چ",
+     "▁ب",
+     "ولی",
+     "▁جان",
+     "▯",
+     "▯",
+     "ی",
+     "▯",
+     "▯",
+     "▁ولی",
+     "▁دیگر",
+     "▁زبان",
+     "ا",
+     "▯",
+     "▯",
+     "▁د",
+     "▯",
+     "▯",
+     "▁اد",
+     "ب",
+     "▁و",
+     "▁ث",
+     "ق",
+     "افت",
+     "▁دی",
+     "▁بی",
+     "▁نم",
+     "ائ",
+     "ندگی",
+     "▁ک",
+     "یتی",
+     "▁گ",
+     "ئ",
+     "ی",
+     "▁ا",
+     "َ",
+     "ئ",
+     "ی",
+     "▯",
+     "▯"
+    ]
+   }
+  ],
+  "llama-4": [
+   {
+    "n": 35,
+    "tokens": [
+     "ای",
+     "ہ",
+     "ہ",
+     "▁بہت",
+     "▁کہ",
+     "ٹ",
+     "▁لو",
+     "ک",
+     "▁جان",
+     "ڑ",
+     "د",
+     "ین",
+     "▁کہ",
+     "▁ڈ",
+     "ر",
+     "ام",
+     "ے",
+     "▁دی",
+     "▁پید",
+     "ائ",
+     "ش",
+     "▁ک",
+     "سر",
+     "اں",
+     "▁ت",
+     "ے",
+     "▁ک",
+     "ت",
+     "ھ",
+     "ے",
+     "▁ہو",
+     "ئی",
+     "▁آ",
+     "ئی",
+     "۔"
+    ]
+   },
+   {
+    "n": 42,
+    "tokens": [
+     "اج",
+     "▁اس",
+     "اں",
+     "▁اگر",
+     "▁آپ",
+     "ڑ",
+     "یں",
+     "▁بزرگ",
+     "اں",
+     "▁دی",
+     "▁انہ",
+     "اں",
+     "▁گ",
+     "لا",
+     "ں",
+     "▁ت",
+     "ے",
+     "▁غ",
+     "ور",
+     "▁نہ",
+     "▁ک",
+     "یت",
+     "ا",
+     "▁ت",
+     "ے",
+     "▁پ",
+     "چھ",
+     "ت",
+     "او",
+     "ے",
+     "▁د",
+     "ے",
+     "▁س",
+     "وا",
+     "▁ک",
+     "ج",
+     "▁ہ",
+     "ت",
+     "ھ",
+     "▁نہ",
+     "▁آسی",
+     "۔"
+    ]
+   },
+   {
+    "n": 42,
+    "tokens": [
+     "اسی",
+     "▁ط",
+     "را",
+     "ں",
+     "▁خی",
+     "بر",
+     "▁پ",
+     "خت",
+     "ون",
+     "خ",
+     "وا",
+     "▁ا",
+     "چ",
+     "▁ب",
+     "ولی",
+     "▁جان",
+     "ڑ",
+     "یں",
+     "▁ولی",
+     "▁دیگر",
+     "▁زبان",
+     "اں",
+     "▁د",
+     "ے",
+     "▁اد",
+     "ب",
+     "▁و",
+     "▁ث",
+     "ق",
+     "افت",
+     "▁دی",
+     "▁بی",
+     "▁نم",
+     "ائ",
+     "ندگی",
+     "▁کی",
+     "تی",
+     "▁گئی",
+     "▁ا▯",
+     "▯",
+     "ئی",
+     "۔"
+    ]
+   }
+  ],
+  "qwen-3.5": [
+   {
+    "n": 36,
+    "tokens": [
+     "ای",
+     "ہ",
+     "ہ",
+     "▁بہ",
+     "ت",
+     "▁کہ",
+     "ٹ",
+     "▁لو",
+     "ک",
+     "▁جان",
+     "ڑ",
+     "د",
+     "ین",
+     "▁کہ",
+     "▁ڈ",
+     "رام",
+     "ے",
+     "▁دی",
+     "▁پ",
+     "ید",
+     "ائ",
+     "ش",
+     "▁ک",
+     "س",
+     "را",
+     "ں",
+     "▁ت",
+     "ے",
+     "▁ک",
+     "ت",
+     "ھے",
+     "▁ہو",
+     "ئی",
+     "▁آ",
+     "ئی",
+     "۔"
+    ]
+   },
+   {
+    "n": 47,
+    "tokens": [
+     "اج",
+     "▁اسا",
+     "ں",
+     "▁اگر",
+     "▁آپ",
+     "ڑ",
+     "یں",
+     "▁ب",
+     "زر",
+     "گا",
+     "ں",
+     "▁دی",
+     "▁ان",
+     "ہ",
+     "ا",
+     "ں",
+     "▁گ",
+     "لا",
+     "ں",
+     "▁ت",
+     "ے",
+     "▁غ",
+     "ور",
+     "▁نہ",
+     "▁ک",
+     "یت",
+     "ا",
+     "▁ت",
+     "ے",
+     "▁پ",
+     "چ",
+     "ھ",
+     "تا",
+     "و",
+     "ے",
+     "▁د",
+     "ے",
+     "▁س",
+     "وا",
+     "▁ک",
+     "ج",
+     "▁ہ",
+     "ت",
+     "ھ",
+     "▁نہ",
+     "▁آسی",
+     "۔"
+    ]
+   },
+   {
+    "n": 45,
+    "tokens": [
+     "اس",
+     "ی",
+     "▁ط",
+     "را",
+     "ں",
+     "▁خی",
+     "بر",
+     "▁پ",
+     "خت",
+     "ون",
+     "خوا",
+     "▁ا",
+     "چ",
+     "▁ب",
+     "ولی",
+     "▁جان",
+     "ڑ",
+     "یں",
+     "▁ولی",
+     "▁دیگر",
+     "▁ز",
+     "ب",
+     "انا",
+     "ں",
+     "▁د",
+     "ے",
+     "▁اد",
+     "ب",
+     "▁و",
+     "▁ثق",
+     "افت",
+     "▁دی",
+     "▁بی",
+     "▁نم",
+     "ائ",
+     "ند",
+     "گی",
+     "▁ک",
+     "یتی",
+     "▁گ",
+     "ئی",
+     "▁ا",
+     "َ",
+     "ئی",
+     "۔"
+    ]
+   }
+  ],
+  "deepseek-v3": [
+   {
+    "n": 41,
+    "tokens": [
+     "ای",
+     "ہ",
+     "ہ",
+     "▁ب",
+     "ہ",
+     "ت",
+     "▁ک",
+     "ہ",
+     "ٹ",
+     "▁لو",
+     "ک",
+     "▁جان",
+     "ڑ",
+     "د",
+     "ین",
+     "▁ک",
+     "ہ",
+     "▁",
+     "ڈ",
+     "ر",
+     "ام",
+     "ے",
+     "▁دی",
+     "▁پید",
+     "ائ",
+     "ش",
+     "▁ک",
+     "سر",
+     "اں",
+     "▁ت",
+     "ے",
+     "▁ک",
+     "ت",
+     "ھ",
+     "ے",
+     "▁ہ",
+     "وئ",
+     "ی",
+     "▁آ",
+     "ئی",
+     "۔"
+    ]
+   },
+   {
+    "n": 47,
+    "tokens": [
+     "اج",
+     "▁اس",
+     "اں",
+     "▁اگر",
+     "▁آ",
+     "پ",
+     "ڑ",
+     "ی",
+     "ں",
+     "▁بزرگ",
+     "اں",
+     "▁دی",
+     "▁ان",
+     "ہ",
+     "اں",
+     "▁گ",
+     "لا",
+     "ں",
+     "▁ت",
+     "ے",
+     "▁غ",
+     "ور",
+     "▁ن",
+     "ہ",
+     "▁ک",
+     "یت",
+     "ا",
+     "▁ت",
+     "ے",
+     "▁پ",
+     "چ",
+     "ھ",
+     "ت",
+     "او",
+     "ے",
+     "▁دے",
+     "▁س",
+     "وا",
+     "▁ک",
+     "ج",
+     "▁ہ",
+     "ت",
+     "ھ",
+     "▁ن",
+     "ہ",
+     "▁آسی",
+     "۔"
+    ]
+   },
+   {
+    "n": 43,
+    "tokens": [
+     "اسی",
+     "▁طر",
+     "اں",
+     "▁خ",
+     "ی",
+     "بر",
+     "▁پ",
+     "خت",
+     "ون",
+     "خ",
+     "وا",
+     "▁ا",
+     "چ",
+     "▁ب",
+     "ولی",
+     "▁جان",
+     "ڑ",
+     "ی",
+     "ں",
+     "▁ولی",
+     "▁دیگر",
+     "▁زبان",
+     "اں",
+     "▁دے",
+     "▁اد",
+     "ب",
+     "▁و",
+     "▁ث",
+     "ق",
+     "افت",
+     "▁دی",
+     "▁بی",
+     "▁نم",
+     "ائ",
+     "ندگی",
+     "▁ک",
+     "یتی",
+     "▁گ",
+     "ئی",
+     "▁ا",
+     "َ",
+     "ئی",
+     "۔"
+    ]
+   }
+  ],
+  "bloom": [
+   {
+    "n": 25,
+    "tokens": [
+     "ا",
+     "یہ",
+     "ہ",
+     "▁بہت",
+     "▁کہ",
+     "ٹ",
+     "▁لوک",
+     "▁جان",
+     "ڑ",
+     "د",
+     "ین",
+     "▁کہ",
+     "▁ڈرام",
+     "ے",
+     "▁دی",
+     "▁پیدائش",
+     "▁ک",
+     "سر",
+     "اں",
+     "▁تے",
+     "▁کت",
+     "ھے",
+     "▁ہوئی",
+     "▁آئی",
+     "۔"
+    ]
+   },
+   {
+    "n": 34,
+    "tokens": [
+     "اج",
+     "▁اس",
+     "اں",
+     "▁اگر",
+     "▁آپ",
+     "ڑ",
+     "یں",
+     "▁بزرگ",
+     "اں",
+     "▁دی",
+     "▁انہ",
+     "اں",
+     "▁گ",
+     "لا",
+     "ں",
+     "▁تے",
+     "▁غور",
+     "▁نہ",
+     "▁کی",
+     "تا",
+     "▁تے",
+     "▁پچھ",
+     "تا",
+     "وے",
+     "▁دے",
+     "▁سوا",
+     "▁ک",
+     "ج",
+     "▁ہ",
+     "تھ",
+     "▁نہ",
+     "▁آ",
+     "سی",
+     "۔"
+    ]
+   },
+   {
+    "n": 27,
+    "tokens": [
+     "اسی",
+     "▁طر",
+     "اں",
+     "▁خیبر",
+     "▁پختونخوا",
+     "▁اچ",
+     "▁بولی",
+     "▁جان",
+     "ڑ",
+     "یں",
+     "▁ولی",
+     "▁دیگر",
+     "▁زبان",
+     "اں",
+     "▁دے",
+     "▁ادب",
+     "▁و",
+     "▁ثقافت",
+     "▁دی",
+     "▁بی",
+     "▁نمائندگی",
+     "▁کی",
+     "تی",
+     "▁گئی",
+     "▁اَ",
+     "ئی",
+     "۔"
+    ]
+   }
+  ],
+  "roberta-urdu": [
+   {
+    "n": 21,
+    "tokens": [
+     "ایہ",
+     "ہ",
+     "▁بہت",
+     "▁کہ",
+     "ٹ",
+     "▁لوک",
+     "▁جان",
+     "ڑ",
+     "دین",
+     "▁کہ",
+     "▁ڈرامے",
+     "▁دی",
+     "▁پیدائش",
+     "▁کسر",
+     "اں",
+     "▁تے",
+     "▁کت",
+     "ھے",
+     "▁ہوئی",
+     "▁آئی",
+     "۔"
+    ]
+   },
+   {
+    "n": 26,
+    "tokens": [
+     "اج",
+     "▁اساں",
+     "▁اگر",
+     "▁آپ",
+     "ڑیں",
+     "▁بزرگ",
+     "اں",
+     "▁دی",
+     "▁انہ",
+     "اں",
+     "▁گلا",
+     "ں",
+     "▁تے",
+     "▁غور",
+     "▁نہ",
+     "▁کی",
+     "تا",
+     "▁تے",
+     "▁پچھتاوے",
+     "▁دے",
+     "▁سوا",
+     "▁کج",
+     "▁ہتھ",
+     "▁نہ",
+     "▁آسی",
+     "۔"
+    ]
+   },
+   {
+    "n": 28,
+    "tokens": [
+     "اسی",
+     "▁طر",
+     "اں",
+     "▁خیبر",
+     "▁پختونخوا",
+     "▁اچ",
+     "▁بولی",
+     "▁جان",
+     "ڑیں",
+     "▁ولی",
+     "▁دیگر",
+     "▁زبان",
+     "اں",
+     "▁دے",
+     "▁ادب",
+     "▁و",
+     "▁ثقافت",
+     "▁دی",
+     "▁بی",
+     "▁نمائندگی",
+     "▁کی",
+     "تی",
+     "▁گئی",
+     "▁ا",
+     "▯",
+     "▯",
+     "ئی",
+     "۔"
+    ]
+   }
+  ],
+  "mt5": [
+   {
+    "n": 31,
+    "tokens": [
+     "▁ای",
+     "ہ",
+     "ہ",
+     "▁ب",
+     "ہت",
+     "▁کہ",
+     "ٹ",
+     "▁",
+     "لوک",
+     "▁جان",
+     "ڑ",
+     "دین",
+     "▁کہ",
+     "▁ڈ",
+     "رام",
+     "ے",
+     "▁دی",
+     "▁پی",
+     "دائ",
+     "ش",
+     "▁کس",
+     "ر",
+     "اں",
+     "▁",
+     "تے",
+     "▁کت",
+     "ھے",
+     "▁ہوئ",
+     "ی",
+     "▁آئی",
+     "۔"
+    ]
+   },
+   {
+    "n": 40,
+    "tokens": [
+     "▁اج",
+     "▁اس",
+     "اں",
+     "▁",
+     "اگر",
+     "▁آپ",
+     "ڑ",
+     "یں",
+     "▁بزرگ",
+     "اں",
+     "▁دی",
+     "▁ان",
+     "ہاں",
+     "▁گل",
+     "اں",
+     "▁",
+     "تے",
+     "▁",
+     "غور",
+     "▁نہ",
+     "▁کی",
+     "تا",
+     "▁",
+     "تے",
+     "▁پ",
+     "چھ",
+     "تا",
+     "و",
+     "ے",
+     "▁",
+     "دے",
+     "▁سوا",
+     "▁ک",
+     "ج",
+     "▁",
+     "ہ",
+     "تھ",
+     "▁نہ",
+     "▁آس",
+     "ی۔"
+    ]
+   },
+   {
+    "n": 42,
+    "tokens": [
+     "▁",
+     "اسی",
+     "▁طر",
+     "اں",
+     "▁خی",
+     "بر",
+     "▁پ",
+     "خت",
+     "و",
+     "نخوا",
+     "▁",
+     "اچ",
+     "▁ب",
+     "ولی",
+     "▁جان",
+     "ڑ",
+     "یں",
+     "▁",
+     "ولی",
+     "▁دیگر",
+     "▁",
+     "زبان",
+     "اں",
+     "▁",
+     "دے",
+     "▁",
+     "ادب",
+     "▁و",
+     "▁",
+     "ثقاف",
+     "ت",
+     "▁دی",
+     "▁بی",
+     "▁نما",
+     "ئ",
+     "ندگی",
+     "▁کی",
+     "تی",
+     "▁گئی",
+     "▁ا",
+     "َ",
+     "ئی۔"
+    ]
+   }
+  ]
+ }
+}

eval/health_gates_and_support.csv

@@ -0,0 +1,5 @@
+id,role,G1_dev_strict,G2_pass,G2_tested,G2_failures,G3_pass,G3_corpus_marker_occurrences,G4_pass,G5_pass,G5_sub_character_tokens,learned_tokens,train_D1_freq_eq0,train_D1_freq_lt20,train_D1_freq_lt20_pct,train_D1_freq_lt100,train_D1_freq_lt100_pct,train_D1_median_freq,train_D1_tokens,partial_utf8_tokens,dev_bytes_per_token,dev_fertility,dev_strr,tokenizer_sha256
+R2-A4-SPnat-D2-32k,released default,836/836,True,32441,0,True,0,True,True,0,32441,2,13182,40.633766,25413,78.336056,27.0,7305029,0,7.076413,1.140879,0.886016,1496e9a7471b5548730fbff8787d9de9a6c0bc06e0ecc0c1f8d2c676ea512d8f
+R2-A10-MinGram-P1r3-D2-48k,pre-registered pick,836/836,True,48831,0,True,0,True,True,0,48831,1,27246,55.796523,41802,85.605456,16.0,7158278,0,7.223664,1.122182,0.897536,4dd12e627f770ce734c635f657ff12e15816fdc3ca817b67ba0d3cd4bd2fd3cc
+R2-A10-MinGram-P1r3-D2-32k,alternative,836/836,True,32447,0,True,0,True,True,0,32447,0,11329,34.9154,25148,77.504854,32.0,7316797,0,7.125844,1.138092,0.887061,12fcd260b8eaee8c514e16ad6ae9b3c912ce1d2fce1447963a56b56cdf409119
+A1-P1r3-D2-16k,baseline (standard recipe),836/836,True,16055,0,True,0,True,,,16064,33,577,3.591882,8491,52.857321,91.0,7762075,9,6.788263,1.196596,0.848037,a725ff94390bc1a552f269677b6b51b37541d3a8013d2cc86ef8511673d6b081

eval/lm_key_contrasts.json

@@ -0,0 +1,358 @@
+{
+ "what": "Key LM bits-per-byte contrasts cited in README.md (hierarchical cluster bootstrap, 10,000 replicates). Full tables: dev_lm_confirm_ranking.csv, test_lm_confirm.csv, dev_lm_large.csv, test_lm_large.csv.",
+ "dev_large_top_set_pairs": {
+  "R2-A4-SPnat-D2-32k - R2-A10-MinGram-P1r3-D2-48k": {
+   "delta": -0.010331,
+   "delta_pct": -0.903037,
+   "ci95": [
+    -0.015725,
+    -0.006244
+   ],
+   "ci95_pct": [
+    -1.347158,
+    -0.557606
+   ],
+   "ci90": [
+    -0.014546,
+    -0.006893
+   ],
+   "p": 0.0,
+   "p_plus1": 0.0002,
+   "se": 0.002375,
+   "se_pct": 0.200841,
+   "tost_equivalent": false,
+   "ci95_excludes_0": true,
+   "mde80_pct": 0.562673
+  },
+  "R2-A4-SPnat-D2-32k - R2-A10-MinGram-P1r3-D2-32k": {
+   "delta": -0.00713,
+   "delta_pct": -0.625,
+   "ci95": [
+    -0.012026,
+    -0.003405
+   ],
+   "ci95_pct": [
+    -1.043154,
+    -0.305626
+   ],
+   "ci90": [
+    -0.01119,
+    -0.003914
+   ],
+   "p": 0.0004,
+   "p_plus1": 0.0006,
+   "se": 0.00221,
+   "se_pct": 0.18929,
+   "tost_equivalent": false,
+   "ci95_excludes_0": true,
+   "mde80_pct": 0.530313
+  },
+  "R2-A10-MinGram-P1r3-D2-32k - R2-A10-MinGram-P1r3-D2-48k": {
+   "delta": -0.003201,
+   "delta_pct": -0.279786,
+   "ci95": [
+    -0.005269,
+    -0.000834
+   ],
+   "ci95_pct": [
+    -0.46409,
+    -0.073777
+   ],
+   "ci90": [
+    -0.004942,
+    -0.001272
+   ],
+   "p": 0.011,
+   "p_plus1": 0.011199,
+   "se": 0.001122,
+   "se_pct": 0.098256,
+   "tost_equivalent": false,
+   "ci95_excludes_0": true,
+   "mde80_pct": 0.275272
+  }
+ },
+ "test_large_boot_2seeds_3cands": {
+  "R2-A4-SPnat-D2-32k - A1-P1r3-D2-16k": {
+   "delta": -0.008684786018473334,
+   "delta_pct": -0.7701554030632566,
+   "ci95_pct": [
+    -1.1521793550485442,
+    -0.3981661969961223
+   ],
+   "ci90": [
+    -0.012104198271208922,
+    -0.005240198401928611
+   ],
+   "p": 0.0,
+   "p_plus1": 0.00019998000199980003,
+   "se_pct": 0.19094983287369818,
+   "ci95_excludes_0": true,
+   "tost_equivalent": false,
+   "mde80_pct": 0.5349622291800946
+  },
+  "R2-A10-MinGram-P1r3-D2-48k - A1-P1r3-D2-16k": {
+   "delta": -0.0028933137990660995,
+   "delta_pct": -0.25657526280652554,
+   "ci95_pct": [
+    -0.4354746500167368,
+    -0.08159379637904507
+   ],
+   "ci90": [
+    -0.004526817078030976,
+    -0.0012422862460522944
+   ],
+   "p": 0.004,
+   "p_plus1": 0.004199580041995801,
+   "se_pct": 0.09050337810749172,
+   "ci95_excludes_0": true,
+   "tost_equivalent": false,
+   "mde80_pct": 0.2535529262952376
+  },
+  "R2-A4-SPnat-D2-32k - R2-A10-MinGram-P1r3-D2-48k": {
+   "delta": -0.005791472219407234,
+   "delta_pct": -0.5149012494908062,
+   "ci95_pct": [
+    -0.8634677179864138,
+    -0.21702516982561376
+   ],
+   "ci90": [
+    -0.008881838988751922,
+    -0.0029836125164287245
+   ],
+   "p": 0.0014,
+   "p_plus1": 0.0015998400159984002,
+   "se_pct": 0.16320821808817637,
+   "ci95_excludes_0": true,
+   "tost_equivalent": false,
+   "mde80_pct": 0.4572417312703925
+  }
+ },
+ "test_large_boot_seed1_4cands": {
+  "R2-A4-SPnat-D2-32k - A1-P1r3-D2-16k": {
+   "delta": -0.009749268125608612,
+   "delta_pct": -0.8640569198004335,
+   "ci95_pct": [
+    -1.2490589169222628,
+    -0.5221700105007353
+   ],
+   "ci90": [
+    -0.013105525700701049,
+    -0.006637920011845844
+   ],
+   "p": 0.0,
+   "p_plus1": 0.00019998000199980003,
+   "se_pct": 0.18159743082505547,
+   "ci95_excludes_0": true,
+   "tost_equivalent": false,
+   "mde80_pct": 0.5087606778467678
+  },
+  "R2-A10-MinGram-P1r3-D2-32k - A1-P1r3-D2-16k": {
+   "delta": -0.005916025958605076,
+   "delta_pct": -0.5243248109901171,
+   "ci95_pct": [
+    -0.7333558799186555,
+    -0.2505256598055685
+   ],
+   "ci90": [
+    -0.0078102004497957785,
+    -0.003451502718269095
+   ],
+   "p": 0.0002,
+   "p_plus1": 0.00039996000399960006,
+   "se_pct": 0.12239615123434773,
+   "ci95_excludes_0": true,
+   "tost_equivalent": false,
+   "mde80_pct": 0.34290324805206795
+  },
+  "R2-A10-MinGram-P1r3-D2-48k - A1-P1r3-D2-16k": {
+   "delta": -0.0035672107921167484,
+   "delta_pct": -0.31615431328829585,
+   "ci95_pct": [
+    -0.4909429266021485,
+    -0.16054069054834635
+   ],
+   "ci90": [
+    -0.005149563958236636,
+    -0.0020913877814648712
+   ],
+   "p": 0.0,
+   "p_plus1": 0.00019998000199980003,
+   "se_pct": 0.08528340828868017,
+   "ci95_excludes_0": true,
+   "tost_equivalent": false,
+   "mde80_pct": 0.23892873601185938
+  },
+  "R2-A4-SPnat-D2-32k - R2-A10-MinGram-P1r3-D2-32k": {
+   "delta": -0.0038332421670035366,
+   "delta_pct": -0.34152279757318016,
+   "ci95_pct": [
+    -0.7391926839990545,
+    -0.10171298736361388
+   ],
+   "ci90": [
+    -0.00736603239301361,
+    -0.0014457211118628212
+   ],
+   "p": 0.0,
+   "p_plus1": 0.00019998000199980003,
+   "se_pct": 0.162594645953095,
+   "ci95_excludes_0": true,
+   "tost_equivalent": false,
+   "mde80_pct": 0.45552275664650255
+  },
+  "R2-A4-SPnat-D2-32k - R2-A10-MinGram-P1r3-D2-48k": {
+   "delta": -0.006182057333491864,
+   "delta_pct": -0.549640318085336,
+   "ci95_pct": [
+    -0.8940447142057342,
+    -0.2488457905349242
+   ],
+   "ci90": [
+    -0.00924963772336348,
+    -0.0033845498604445205
+   ],
+   "p": 0.0006,
+   "p_plus1": 0.0007999200079992001,
+   "se_pct": 0.16206454279153712,
+   "ci95_excludes_0": true,
+   "tost_equivalent": false,
+   "mde80_pct": 0.4540376274650071
+  },
+  "R2-A10-MinGram-P1r3-D2-48k - R2-A10-MinGram-P1r3-D2-32k": {
+   "delta": 0.0023488151664883272,
+   "delta_pct": 0.20926774038606374,
+   "ci95_pct": [
+    -0.06444365120164039,
+    0.44287168269519417
+   ],
+   "ci90": [
+    -0.0002884015121531357,
+    0.004546200404592969
+   ],
+   "p": 0.1366,
+   "p_plus1": 0.1367863213678632,
+   "se_pct": 0.12949901051134746,
+   "ci95_excludes_0": false,
+   "tost_equivalent": false,
+   "mde80_pct": 0.36280251360884697
+  }
+ },
+ "test_large_parametric_all_seeds": {
+  "pairs": {
+   "R2-A4-SPnat-D2-32k - A1-P1r3-D2-16k": {
+    "delta": -0.008684786018473334,
+    "delta_pct": -0.7701554030632566,
+    "ci95_pct": [
+     -1.2683506028519904,
+     -0.28392151290853485
+    ],
+    "ci90": [
+     -0.013318493428309297,
+     -0.004133055845341472
+    ],
+    "p": 0.0018,
+    "p_plus1": 0.001999800019998,
+    "se_pct": 0.2516827853983878,
+    "ci95_excludes_0": true,
+    "tost_equivalent": false,
+    "mde80_pct": 0.7051107712256217
+   },
+   "R2-A10-MinGram-P1r3-D2-32k - A1-P1r3-D2-16k": {
+    "delta": -0.005269543998672743,
+    "delta_pct": -0.4672962320113417,
+    "ci95_pct": [
+     -0.9419097590776901,
+     0.042593733706161235
+    ],
+    "ci90": [
+     -0.009689398904420075,
+     -0.0005186912539030928
+    ],
+    "p": 0.0708,
+    "p_plus1": 0.070992900709929,
+    "se_pct": 0.2507693914187916,
+    "ci95_excludes_0": false,
+    "tost_equivalent": false,
+    "mde80_pct": 0.7025518201540716
+   },
+   "R2-A10-MinGram-P1r3-D2-48k - A1-P1r3-D2-16k": {
+    "delta": -0.0028933137990660995,
+    "delta_pct": -0.25657526280652554,
+    "ci95_pct": [
+     -0.5497971900054238,
+     0.0343370747906838
+    ],
+    "ci90": [
+     -0.005623487622485324,
+     -0.00013111641818942617
+    ],
+    "p": 0.0854,
+    "p_plus1": 0.08559144085591441,
+    "se_pct": 0.14822563760616336,
+    "ci95_excludes_0": false,
+    "tost_equivalent": false,
+    "mde80_pct": 0.415266755262797
+   },
+   "R2-A4-SPnat-D2-32k - R2-A10-MinGram-P1r3-D2-32k": {
+    "delta": -0.0034152420198005906,
+    "delta_pct": -0.3042810650033998,
+    "ci95_pct": [
+     -0.9452945588944334,
+     0.2739421620049433
+    ],
+    "ci90": [
+     -0.00936036830246204,
+     0.001908147621246279
+    ],
+    "p": 0.2996,
+    "p_plus1": 0.2997700229977002,
+    "se_pct": 0.307931856086911,
+    "ci95_excludes_0": false,
+    "tost_equivalent": false,
+    "mde80_pct": 0.8626973361991798
+   },
+   "R2-A4-SPnat-D2-32k - R2-A10-MinGram-P1r3-D2-48k": {
+    "delta": -0.005791472219407234,
+    "delta_pct": -0.5149012494908062,
+    "ci95_pct": [
+     -1.019560360215281,
+     -0.03635460358197493
+    ],
+    "ci90": [
+     -0.010475266194312948,
+     -0.001285244703859535
+    ],
+    "p": 0.0356,
+    "p_plus1": 0.0357964203579642,
+    "se_pct": 0.24936744311133924,
+    "ci95_excludes_0": true,
+    "tost_equivalent": false,
+    "mde80_pct": 0.6986241424993546
+   },
+   "R2-A10-MinGram-P1r3-D2-48k - R2-A10-MinGram-P1r3-D2-32k": {
+    "delta": 0.0023762301996066437,
+    "delta_pct": 0.21171028338184003,
+    "ci95_pct": [
+     -0.30725835331838786,
+     0.7166814161587275
+    ],
+    "ci90": [
+     -0.0025137876669922396,
+     0.007022484614695488
+    ],
+    "p": 0.4218,
+    "p_plus1": 0.42195780421957807,
+    "se_pct": 0.2593685536234114,
+    "ci95_excludes_0": false,
+    "tost_equivalent": false,
+    "mde80_pct": 0.72664310587469
+   }
+  },
+  "p_first_within_top_set": {
+   "R2-A4-SPnat-D2-32k": 0.8448,
+   "R2-A10-MinGram-P1r3-D2-48k": 0.0087,
+   "R2-A10-MinGram-P1r3-D2-32k": 0.1465
+  },
+  "what": "clusters resampled (the test cluster draws), each candidate's available seeds averaged, plus relative seed noise N(0, sigma_c^2/n_c); sigma_c = max(dev large 2-seed s.d., dev confirm 5-seed s.d.) from amendment1_stats.json; normal draws from default_rng(12346)"
+ }
+}

eval/test_lm_confirm.csv

@@ -0,0 +1,7 @@
+test_rank_of6,id,role,bundle,n_vocab,params_total,params_non_embedding,test_bytes_per_token,n_seeds,test_mean_bpb,test_seed_bpb,test_seed_sd_rel_pct,test_delta_vs_baseline_pct,test_delta_vs_baseline_ci95_pct,test_p_vs_baseline,dev_mean_bpb,dev_rank_of17,dev_delta_vs_baseline_pct,dev_delta_vs_baseline_ci95_pct,change_dev_to_test_pp,change_dev_to_test_ci95_pp,test_delta_vs_best_of6_pct,test_delta_vs_best_of6_ci95_pct,test_holm_p_vs_best_of6
+1,R2-A10-MinGram-P1r3-D2-48k,pre-registered pick (DECISION.md),f54c929ba1ab,49152,11256960,1772928,7.373362,5,1.205428270630087,"[1.2047896494901702, 1.2059034101351314, 1.20633928949385, 1.205412639294452, 1.2046963647368312]",0.05865259252616563,-1.3160397372781005,"[-1.467033141440053, -1.1643594058582694]",0.0,1.2263581740516902,2,-1.5817597957708232,"[-1.8675208666941552, -1.355869455836522]",0.2657200584927226,"[0.018640026346452922, 0.5532296179772263]",,,
+2,R2-A10-MinGram-P1r3-D2-32k,top-set member (Amendment 1 fallback),f54c929ba1ab,32768,8112384,1772928,7.274954,5,1.2061960938127751,"[1.2058776331394079, 1.2063087984686824, 1.2093894631968845, 1.2069417847891937, 1.2024627894697058]",0.20645467091462297,-1.253180889269175,"[-1.4797207661189518, -1.03708384604859]",0.0,1.2284434797265598,3,-1.4144089196964234,"[-1.662170488805242, -1.191419602468561]",0.16122803042724843,"[-0.04231045435541847, 0.3860564894084195]",0.06369712751857491,"[-0.15251248513420793, 0.2953023739022093]",0.5822
+3,R2-A4-SPnat-D2-32k,released default (AMENDMENT_1.md),f54c929ba1ab,32768,8112192,1772928,7.277326,5,1.2091072993866336,"[1.2129939093356459, 1.2090638021044307, 1.2080341771212149, 1.2062515856466753, 1.2091930227252015]",0.20435856219186863,-1.0148512414859043,"[-1.3337703266599765, -0.719411106580577]",0.0,1.2258361359998111,1,-1.6236546250896033,"[-2.1070539470058685, -1.2360590866135281]",0.608803383603699,"[0.172267052994226, 1.1171782340898435]",0.30520511640428255,"[-0.019494874894211908, 0.6262727704237294]",0.1296
+4,R2-A1-P1r3-D2-48k,equal-vocabulary BPE control for the 48k pick (report-only),94175d26497f,49152,11257152,1772928,7.352626,5,1.211112897205615,"[1.2092966317143177, 1.2119648668982443, 1.2129522019294052, 1.2110089578596441, 1.210341827626464]",0.11686507492736961,-0.850660355728833,"[-1.0468293493009588, -0.6530838548134663]",0.0,1.2337413764600247,5,-0.9892397608619384,"[-1.2117041492054477, -0.7787421586001484]",0.1385794051331054,"[-0.10002961497460199, 0.38457145452512503]",0.4715856359131806,"[0.31370175058109206, 0.6434340570746931]",0.0
+5,A1-P1r3-D2-32k,round-1 confirmatory winner (report-only),94175d26497f,32768,8112576,1772928,7.246036,5,1.2145897732763742,"[1.2157356635342742, 1.2152889816577066, 1.2150796722014732, 1.213604199219369, 1.2132403497690487]",0.09051181811235785,-0.5660213536703774,"[-0.7539943802074569, -0.38067108247804304]",0.0,1.2359686388208577,6,-0.8104965138527125,"[-1.0007432256783066, -0.6574930429130209]",0.24447516018233506,"[0.025092332962363524, 0.48592810710440854]",0.760020556137981,"[0.5867749515939401, 0.9398737865247105]",0.0
+6,A1-P1r3-D2-16k,baseline (standard recipe),f54c929ba1ab,16384,4969536,1772928,6.948493,5,1.2215037453107163,"[1.2209580951297607, 1.2201410927166643, 1.2220834417794444, 1.2208247446814235, 1.2235113522462882]",0.10816063260614522,,,,1.2460679763292413,9,,,,,1.3335903157660747,"[1.1780764498105485, 1.4888754375437618]",0.0

eval/test_lm_large.csv

@@ -0,0 +1,5 @@
+id,n_seeds,seeds,test_seed_bpb,test_mean_bpb,test_delta_vs_baseline_pct_means,dev_large_seed_bpb,dev_large_mean_bpb,params_total,note
+R2-A4-SPnat-D2-32k,2,"[1, 2]","[1.11856391381851, 1.1193999141129158]",1.118981913965713,-0.7701554030632566,"[1.1331864564693357, 1.134282581917815]",1.1337345191935753,23305344,
+R2-A10-MinGram-P1r3-D2-32k,1,[1],[1.1223971559855135],1.1223971559855135,-0.4672962320113417,"[1.1401575170755545, 1.1415723291801942]",1.1408649231278742,23305728,seed 2 missing: Colab disconnected at step 6588/7325 and the free GPU quota was then exhausted
+R2-A10-MinGram-P1r3-D2-48k,2,"[1, 2]","[1.1247459711520018, 1.1248008012182387]",1.1247733861851201,-0.25657526280652554,"[1.1433569638393768, 1.1447747551522283]",1.1440658594958024,29594880,
+A1-P1r3-D2-16k,2,"[1, 2]","[1.1283131819441186, 1.127020218024254]",1.1276666999841862,,"[1.148993474869845, 1.1491682556793037]",1.1490808652745743,17020032,

eval/vocab_audit_summary.json

@@ -0,0 +1,28 @@
+{
+ "what": "Headline counts of the vocabulary audit (F:/Hindko/_tokenizer/sota/vocab_audit/VOCAB_AUDIT.md, 2026-09-27). The piece lists themselves are kept private (they are for native-speaker review).",
+ "verdict": "no fix to the tokenizer needed before publishing",
+ "learned_pieces": 32441,
+ "arabic_script_learned_pieces_pct": 90.9,
+ "junk_artifact_pieces": 478,
+ "junk_artifact_pieces_pct_of_learned": 1.47,
+ "junk_test_token_share_pct": 0.29,
+ "longest_digit_run_in_any_piece": 4,
+ "ascii_digit_pieces": "single digits 0-9 only",
+ "urdu_digit_run_pieces_2_to_4": 345,
+ "pii_placeholder_dedicated_pieces": 0,
+ "pieces_identifying_private_individuals_found": 0,
+ "offensive_sense_attested": 13,
+ "sensitive_group_labels": 17,
+ "ordinary_words_usable_as_insults": 24,
+ "false_positive_candidates": 45,
+ "pieces_removed": 0,
+ "special_ids_emitted_on_11786145_real_tokens": 0,
+ "split_special_tokens_true_prevents_literal_special_match": false,
+ "sections": {
+  "composition": "2",
+  "junk": "3",
+  "privacy": "4",
+  "offensive": "5",
+  "specials": "7"
+ }
+}