Hindko Embeddings v1.0.0: multilingual-e5-large fine-tuned for Hindko (model, card, benchmarks)

.gitattributes

@@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+benchmarks/00_scorecard.png filter=lfs diff=lfs merge=lfs -text
+benchmarks/01_test_overall_all_models.png filter=lfs diff=lfs merge=lfs -text
+benchmarks/02_test_per_task.png filter=lfs diff=lfs merge=lfs -text
+benchmarks/03_dev_candidates.png filter=lfs diff=lfs merge=lfs -text
+benchmarks/04_round1_vs_round2.png filter=lfs diff=lfs merge=lfs -text
+benchmarks/05_size_vs_score.png filter=lfs diff=lfs merge=lfs -text
+benchmarks/06_booktitle_detail.png filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

1_Pooling/config.json

@@ -0,0 +1,5 @@
+{
+    "embedding_dimension": 1024,
+    "pooling_mode": "mean",
+    "include_prompt": true
+}

2_Normalize/config.json

@@ -0,0 +1,4 @@
+{
+    "module_input_name": "sentence_embedding",
+    "module_output_name": "sentence_embedding"
+}

LICENSE

@@ -0,0 +1,446 @@
+Hindko Embeddings 1.0.0 (junaid008/hindko-embeddings)
+Copyright (c) 2026 junaid008
+
+The files in this repository (model weights, configuration, model card,
+evaluation outputs and benchmark images) are licensed under the Creative
+Commons Attribution 4.0 International License (CC BY 4.0), reproduced below
+and at https://creativecommons.org/licenses/by/4.0/legalcode
+
+Attribution: "Hindko Embeddings" by junaid008, https://huggingface.co/junaid008/hindko-embeddings
+
+The model is a fine-tuned version of intfloat/multilingual-e5-large, which is
+released under the MIT License. Its copyright and permission notice is
+reproduced in the Third-party notice at the end of this file and applies to
+the portions of the base model contained in the weights, tokenizer and
+configuration files.
+
+The licence covers the files in this repository only. The texts the model
+was trained and evaluated on are not included and no rights in them are
+granted.
+
+-----------------------------------------------------------------------
+
+Attribution 4.0 International
+
+=======================================================================
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright
+and certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+     Considerations for licensors: Our public licenses are
+     intended for use by those authorized to give the public
+     permission to use material in ways otherwise restricted by
+     copyright and certain other rights. Our licenses are
+     irrevocable. Licensors should read and understand the terms
+     and conditions of the license they choose before applying it.
+     Licensors should also secure all rights necessary before
+     applying our licenses so that the public can reuse the
+     material as expected. Licensors should clearly mark any
+     material not subject to the license. This includes other CC-
+     licensed material, or material used under an exception or
+     limitation to copyright. More considerations for licensors:
+    wiki.creativecommons.org/Considerations_for_licensors
+
+     Considerations for the public: By using one of our public
+     licenses, a licensor grants the public permission to use the
+     licensed material under specified terms and conditions. If
+     the licensor's permission is not necessary for any reason--for
+     example, because of any applicable exception or limitation to
+     copyright--then that use is not regulated by the license. Our
+     licenses grant only permissions under copyright and certain
+     other rights that a licensor has authority to grant. Use of
+     the licensed material may still be restricted for other
+     reasons, including because others have copyright or other
+     rights in the material. A licensor may make special requests,
+     such as asking that all changes be marked or described.
+     Although not required by our licenses, you are encouraged to
+     respect those requests where reasonable. More considerations
+     for the public:
+    wiki.creativecommons.org/Considerations_for_licensees
+
+=======================================================================
+
+Creative Commons Attribution 4.0 International Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution 4.0 International Public License ("Public License"). To the
+extent this Public License may be interpreted as a contract, You are
+granted the Licensed Rights in consideration of Your acceptance of
+these terms and conditions, and the Licensor grants You such rights in
+consideration of benefits the Licensor receives from making the
+Licensed Material available under these terms and conditions.
+
+
+Section 1 -- Definitions.
+
+  a. Adapted Material means material subject to Copyright and Similar
+     Rights that is derived from or based upon the Licensed Material
+     and in which the Licensed Material is translated, altered,
+     arranged, transformed, or otherwise modified in a manner requiring
+     permission under the Copyright and Similar Rights held by the
+     Licensor. For purposes of this Public License, where the Licensed
+     Material is a musical work, performance, or sound recording,
+     Adapted Material is always produced where the Licensed Material is
+     synched in timed relation with a moving image.
+
+  b. Adapter's License means the license You apply to Your Copyright
+     and Similar Rights in Your contributions to Adapted Material in
+     accordance with the terms and conditions of this Public License.
+
+  c. Copyright and Similar Rights means copyright and/or similar rights
+     closely related to copyright including, without limitation,
+     performance, broadcast, sound recording, and Sui Generis Database
+     Rights, without regard to how the rights are labeled or
+     categorized. For purposes of this Public License, the rights
+     specified in Section 2(b)(1)-(2) are not Copyright and Similar
+     Rights.
+
+  d. Effective Technological Measures means those measures that, in the
+     absence of proper authority, may not be circumvented under laws
+     fulfilling obligations under Article 11 of the WIPO Copyright
+     Treaty adopted on December 20, 1996, and/or similar international
+     agreements.
+
+  e. Exceptions and Limitations means fair use, fair dealing, and/or
+     any other exception or limitation to Copyright and Similar Rights
+     that applies to Your use of the Licensed Material.
+
+  f. Licensed Material means the artistic or literary work, database,
+     or other material to which the Licensor applied this Public
+     License.
+
+  g. Licensed Rights means the rights granted to You subject to the
+     terms and conditions of this Public License, which are limited to
+     all Copyright and Similar Rights that apply to Your use of the
+     Licensed Material and that the Licensor has authority to license.
+
+  h. Licensor means the individual(s) or entity(ies) granting rights
+     under this Public License.
+
+  i. Share means to provide material to the public by any means or
+     process that requires permission under the Licensed Rights, such
+     as reproduction, public display, public performance, distribution,
+     dissemination, communication, or importation, and to make material
+     available to the public including in ways that members of the
+     public may access the material from a place and at a time
+     individually chosen by them.
+
+  j. Sui Generis Database Rights means rights other than copyright
+     resulting from Directive 96/9/EC of the European Parliament and of
+     the Council of 11 March 1996 on the legal protection of databases,
+     as amended and/or succeeded, as well as other essentially
+     equivalent rights anywhere in the world.
+
+  k. You means the individual or entity exercising the Licensed Rights
+     under this Public License. Your has a corresponding meaning.
+
+
+Section 2 -- Scope.
+
+  a. License grant.
+
+       1. Subject to the terms and conditions of this Public License,
+          the Licensor hereby grants You a worldwide, royalty-free,
+          non-sublicensable, non-exclusive, irrevocable license to
+          exercise the Licensed Rights in the Licensed Material to:
+
+            a. reproduce and Share the Licensed Material, in whole or
+               in part; and
+
+            b. produce, reproduce, and Share Adapted Material.
+
+       2. Exceptions and Limitations. For the avoidance of doubt, where
+          Exceptions and Limitations apply to Your use, this Public
+          License does not apply, and You do not need to comply with
+          its terms and conditions.
+
+       3. Term. The term of this Public License is specified in Section
+          6(a).
+
+       4. Media and formats; technical modifications allowed. The
+          Licensor authorizes You to exercise the Licensed Rights in
+          all media and formats whether now known or hereafter created,
+          and to make technical modifications necessary to do so. The
+          Licensor waives and/or agrees not to assert any right or
+          authority to forbid You from making technical modifications
+          necessary to exercise the Licensed Rights, including
+          technical modifications necessary to circumvent Effective
+          Technological Measures. For purposes of this Public License,
+          simply making modifications authorized by this Section 2(a)
+          (4) never produces Adapted Material.
+
+       5. Downstream recipients.
+
+            a. Offer from the Licensor -- Licensed Material. Every
+               recipient of the Licensed Material automatically
+               receives an offer from the Licensor to exercise the
+               Licensed Rights under the terms and conditions of this
+               Public License.
+
+            b. No downstream restrictions. You may not offer or impose
+               any additional or different terms or conditions on, or
+               apply any Effective Technological Measures to, the
+               Licensed Material if doing so restricts exercise of the
+               Licensed Rights by any recipient of the Licensed
+               Material.
+
+       6. No endorsement. Nothing in this Public License constitutes or
+          may be construed as permission to assert or imply that You
+          are, or that Your use of the Licensed Material is, connected
+          with, or sponsored, endorsed, or granted official status by,
+          the Licensor or others designated to receive attribution as
+          provided in Section 3(a)(1)(A)(i).
+
+  b. Other rights.
+
+       1. Moral rights, such as the right of integrity, are not
+          licensed under this Public License, nor are publicity,
+          privacy, and/or other similar personality rights; however, to
+          the extent possible, the Licensor waives and/or agrees not to
+          assert any such rights held by the Licensor to the limited
+          extent necessary to allow You to exercise the Licensed
+          Rights, but not otherwise.
+
+       2. Patent and trademark rights are not licensed under this
+          Public License.
+
+       3. To the extent possible, the Licensor waives any right to
+          collect royalties from You for the exercise of the Licensed
+          Rights, whether directly or through a collecting society
+          under any voluntary or waivable statutory or compulsory
+          licensing scheme. In all other cases the Licensor expressly
+          reserves any right to collect such royalties.
+
+
+Section 3 -- License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+  a. Attribution.
+
+       1. If You Share the Licensed Material (including in modified
+          form), You must:
+
+            a. retain the following if it is supplied by the Licensor
+               with the Licensed Material:
+
+                 i. identification of the creator(s) of the Licensed
+                    Material and any others designated to receive
+                    attribution, in any reasonable manner requested by
+                    the Licensor (including by pseudonym if
+                    designated);
+
+                ii. a copyright notice;
+
+               iii. a notice that refers to this Public License;
+
+                iv. a notice that refers to the disclaimer of
+                    warranties;
+
+                 v. a URI or hyperlink to the Licensed Material to the
+                    extent reasonably practicable;
+
+            b. indicate if You modified the Licensed Material and
+               retain an indication of any previous modifications; and
+
+            c. indicate the Licensed Material is licensed under this
+               Public License, and include the text of, or the URI or
+               hyperlink to, this Public License.
+
+       2. You may satisfy the conditions in Section 3(a)(1) in any
+          reasonable manner based on the medium, means, and context in
+          which You Share the Licensed Material. For example, it may be
+          reasonable to satisfy the conditions by providing a URI or
+          hyperlink to a resource that includes the required
+          information.
+
+       3. If requested by the Licensor, You must remove any of the
+          information required by Section 3(a)(1)(A) to the extent
+          reasonably practicable.
+
+       4. If You Share Adapted Material You produce, the Adapter's
+          License You apply must not prevent recipients of the Adapted
+          Material from complying with this Public License.
+
+
+Section 4 -- Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that
+apply to Your use of the Licensed Material:
+
+  a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+     to extract, reuse, reproduce, and Share all or a substantial
+     portion of the contents of the database;
+
+  b. if You include all or a substantial portion of the database
+     contents in a database in which You have Sui Generis Database
+     Rights, then the database in which You have Sui Generis Database
+     Rights (but not its individual contents) is Adapted Material; and
+
+  c. You must comply with the conditions in Section 3(a) if You Share
+     all or a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not
+replace Your obligations under this Public License where the Licensed
+Rights include other Copyright and Similar Rights.
+
+
+Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+
+  a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+     EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+     AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+     ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+     IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+     WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+     PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+     ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+     KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+     ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+
+  b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+     TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+     NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+     INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+     COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+     USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+     ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+     DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+     IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+
+  c. The disclaimer of warranties and limitation of liability provided
+     above shall be interpreted in a manner that, to the extent
+     possible, most closely approximates an absolute disclaimer and
+     waiver of all liability.
+
+
+Section 6 -- Term and Termination.
+
+  a. This Public License applies for the term of the Copyright and
+     Similar Rights licensed here. However, if You fail to comply with
+     this Public License, then Your rights under this Public License
+     terminate automatically.
+
+  b. Where Your right to use the Licensed Material has terminated under
+     Section 6(a), it reinstates:
+
+       1. automatically as of the date the violation is cured, provided
+          it is cured within 30 days of Your discovery of the
+          violation; or
+
+       2. upon express reinstatement by the Licensor.
+
+     For the avoidance of doubt, this Section 6(b) does not affect any
+     right the Licensor may have to seek remedies for Your violations
+     of this Public License.
+
+  c. For the avoidance of doubt, the Licensor may also offer the
+     Licensed Material under separate terms or conditions or stop
+     distributing the Licensed Material at any time; however, doing so
+     will not terminate this Public License.
+
+  d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+     License.
+
+
+Section 7 -- Other Terms and Conditions.
+
+  a. The Licensor shall not be bound by any additional or different
+     terms or conditions communicated by You unless expressly agreed.
+
+  b. Any arrangements, understandings, or agreements regarding the
+     Licensed Material not stated herein are separate from and
+     independent of the terms and conditions of this Public License.
+
+
+Section 8 -- Interpretation.
+
+  a. For the avoidance of doubt, this Public License does not, and
+     shall not be interpreted to, reduce, limit, restrict, or impose
+     conditions on any use of the Licensed Material that could lawfully
+     be made without permission under this Public License.
+
+  b. To the extent possible, if any provision of this Public License is
+     deemed unenforceable, it shall be automatically reformed to the
+     minimum extent necessary to make it enforceable. If the provision
+     cannot be reformed, it shall be severed from this Public License
+     without affecting the enforceability of the remaining terms and
+     conditions.
+
+  c. No term or condition of this Public License will be waived and no
+     failure to comply consented to unless expressly agreed to by the
+     Licensor.
+
+  d. Nothing in this Public License constitutes or may be interpreted
+     as a limitation upon, or waiver of, any privileges and immunities
+     that apply to the Licensor or You, including from the legal
+     processes of any jurisdiction or authority.
+
+
+=======================================================================
+
+Creative Commons is not a party to its public
+licenses. Notwithstanding, Creative Commons may elect to apply one of
+its public licenses to material it publishes and in those instances
+will be considered the “Licensor.” The text of the Creative Commons
+public licenses is dedicated to the public domain under the CC0 Public
+Domain Dedication. Except for the limited purpose of indicating that
+material is shared under a Creative Commons public license or as
+otherwise permitted by the Creative Commons policies published at
+creativecommons.org/policies, Creative Commons does not authorize the
+use of the trademark "Creative Commons" or any other trademark or logo
+of Creative Commons without its prior written consent including,
+without limitation, in connection with any unauthorized modifications
+to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For
+the avoidance of doubt, this paragraph does not form part of the
+public licenses.
+
+Creative Commons may be contacted at creativecommons.org.
+
+-----------------------------------------------------------------------
+
+Third-party notice: intfloat/multilingual-e5-large
+
+Base model: https://huggingface.co/intfloat/multilingual-e5-large
+License: MIT
+
+MIT License
+
+Copyright (c) Microsoft Corporation.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,425 @@
+---
+language:
+- hno
+- hnd
+license: cc-by-4.0
+library_name: sentence-transformers
+pipeline_tag: sentence-similarity
+base_model: intfloat/multilingual-e5-large
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+- hindko
+- embeddings
+- low-resource
+- urdu-script
+- perso-arabic
+model-index:
+- name: hindko-embeddings
+  results:
+  - task:
+      type: Retrieval
+    dataset:
+      name: HindkoEmbedBench v1.0 HeadlineRetrieval
+      type: HindkoEmbedBench-HeadlineRetrieval
+      split: test
+    metrics:
+    - type: ndcg_at_10
+      name: nDCG@10
+      value: 0.6372
+    - type: recall_at_1
+      name: Recall@1
+      value: 0.5625
+    - type: recall_at_10
+      name: Recall@10
+      value: 0.7125
+  - task:
+      type: Retrieval
+    dataset:
+      name: HindkoEmbedBench v1.0 BookTitleRetrieval
+      type: HindkoEmbedBench-BookTitleRetrieval
+      split: test
+    metrics:
+    - type: ndcg_at_10
+      name: nDCG@10
+      value: 0.3460
+    - type: recall_at_1
+      name: Recall@1
+      value: 0.1548
+    - type: recall_at_10
+      name: Recall@10
+      value: 0.4494
+    - type: mrr_at_10
+      name: MRR@10
+      value: 0.3608
+  - task:
+      type: Retrieval
+    dataset:
+      name: HindkoEmbedBench v1.0 PassageContinuation
+      type: HindkoEmbedBench-PassageContinuation
+      split: test
+    metrics:
+    - type: ndcg_at_10
+      name: nDCG@10
+      value: 0.8756
+  - task:
+      type: Classification
+    dataset:
+      name: HindkoEmbedBench v1.0 VerseProseClassification
+      type: HindkoEmbedBench-VerseProseClassification
+      split: test
+    metrics:
+    - type: f1
+      name: macro-F1
+      value: 0.9801
+  - task:
+      type: Clustering
+    dataset:
+      name: HindkoEmbedBench v1.0 BookClustering
+      type: HindkoEmbedBench-BookClustering
+      split: test
+    metrics:
+    - type: v_measure
+      name: V-measure
+      value: 0.8566
+---
+
+# Hindko Embeddings
+
+> **License: [CC BY 4.0](LICENSE)** for this model's weights and files. The base model, intfloat/multilingual-e5-large, is MIT-licensed; its notice is kept in [`LICENSE`](LICENSE). See [License](#license).
+
+A sentence-embedding model for **Hindko** (Northern `hno` and Southern `hnd`), the Indo-Aryan language of Peshawar, Hazara and Kohat in Khyber Pakhtunkhwa, Pakistan, written in Urdu (Perso-Arabic) script. It turns a Hindko headline, sentence, verse or passage into a vector, so that texts about the same thing end up close together: for search, retrieval for RAG, similarity, clustering and classification. It is [intfloat/multilingual-e5-large](https://huggingface.co/intfloat/multilingual-e5-large) fine-tuned on 41,533 training pairs from a Hindko corpus (39,517 of them from Hindko-variety records), and to our knowledge it is the first embedding model trained for Hindko.
+
+On the held-out test split of **HindkoEmbedBench v1.0**, the five-task benchmark built for this project (first used in round 1, for our earlier model), it scores **0.7391** overall. A character n-gram TF-IDF baseline scores 0.6864, and the best public embedding model tested, bge-m3, scores 0.6332 at the same 512-token length. The model is ahead of TF-IDF on four of the five tasks and **behind it on one**: finding a book passage from its table-of-contents title (see [Limitations](#limitations-and-bias)).
+
+## At a glance
+
+| | |
+|---|---|
+| **Type** | Sentence-transformers bi-encoder: transformer encoder, mean pooling, L2-normalized output |
+| **Base model** | [intfloat/multilingual-e5-large](https://huggingface.co/intfloat/multilingual-e5-large) (MIT), 560M parameters; its XLM-RoBERTa tokenizer is unchanged |
+| **Output** | 1,024-dimensional vectors; compare with cosine similarity (dot product of normalized vectors) |
+| **Input length** | `max_seq_length` 512 tokens; longer input is truncated |
+| **Prefixes** | `query: ` for queries and for similarity, clustering and classification; `passage: ` for documents to be retrieved ([details](#prefixes)) |
+| **Library** | sentence-transformers 6.0 or later with transformers 5.x (saved with sentence-transformers 6.1.0 and transformers 5.3.0); older versions may not load the saved configuration |
+| **Language** | Hindko in Urdu script (`hno`, `hnd`). Not evaluated on Romanized Hindko or other languages |
+| **Training data** | 41,533 pairs (87,176 distinct texts) from the train split of a Hindko corpus of newspapers, books and web pages, with 23,061 mined hard negatives |
+| **Test score** | **0.7391** overall; TF-IDF 0.6864, bge-m3 0.6332, our round-1 model 0.6923 |
+| **Known weak spot** | Book-title retrieval: 0.3460 nDCG@10 on test, below TF-IDF (0.3756) and bge-m3 (0.3778) |
+| **Weights** | float16 safetensors, about 1,068 MB: the same weights that were benchmarked |
+| **Version** | 1.0.0, 2026-10-01 |
+| **Sibling project** | [Hindko Tokenizer](https://huggingface.co/junaid008/hindko-tokenizer) ([GitHub](https://github.com/junaidaslam2006/Hindko-Tokenizer)). This model does **not** use it; it keeps its base model's tokenizer |
+| **Code** | The benchmark and the training code will be released |
+
+## Highlights
+
+- **Highest overall test score of all models tested**: 0.7391, against 0.6864 for TF-IDF (+0.0527), 0.6332 for bge-m3 (+0.1059) and 0.5977 for its own base model without fine-tuning (+0.1414).
+- **Large gains on passage tasks**: continuation retrieval 0.8756 nDCG@10 (TF-IDF 0.7899, bge-m3 0.7174) and book clustering 0.8566 V-measure (TF-IDF 0.6895, bge-m3 0.5168).
+- **Better than our round-1 model** (0.6923 on test) on four of five tasks; slightly lower on verse/prose classification (0.9801 vs 0.9827).
+- **Chosen on dev only**, by a rule fixed in advance; this model was scored on test once, after the choice.
+- **Weak spot**: book-title retrieval, where TF-IDF and bge-m3 score higher on test (28 queries) and TF-IDF scores higher on dev (262 queries); on dev the model is also narrowly behind TF-IDF on verse/prose (0.9406 vs 0.9463).
+
+![Scorecard: Hindko Embeddings on the HindkoEmbedBench test split](benchmarks/00_scorecard.png)
+
+## Usage
+
+```bash
+pip install -U "sentence-transformers>=6.0"
+```
+
+### Search (retrieval)
+
+```python
+from sentence_transformers import SentenceTransformer
+
+model = SentenceTransformer("junaid008/hindko-embeddings")
+if model.device.type == "cpu":
+    model = model.float()          # weights are stored in float16; use float32 on a CPU
+
+query = "query: ڈرامے دی پیدائش"
+passages = [
+    "passage: ایہہ بہت کہٹ لوک جانڑدین کہ ڈرامے دی پیدائش کسراں تے کتھے ہوئی آئی۔",
+    "passage: اج اساں اگر آپڑیں بزرگاں دی انہاں گلاں تے غور نہ کیتا تے پچھتاوے دے سوا کج ہتھ نہ آسی۔",
+    "passage: اسی طراں خیبر پختونخوا اچ بولی جانڑیں ولی دیگر زباناں دے ادب و ثقافت دی بی نمائندگی کیتی گئی اَئی۔",
+]
+
+q = model.encode([query], normalize_embeddings=True)
+p = model.encode(passages, normalize_embeddings=True)
+scores = model.similarity(q, p)    # cosine similarity, shape (1, 3); higher = more relevant
+ranking = scores[0].argsort(descending=True)
+```
+
+### Similarity, clustering and classification
+
+Use `query: ` on every text when the texts play the same role:
+
+```python
+sentences = [
+    "query: ایہہ بہت کہٹ لوک جانڑدین کہ ڈرامے دی پیدائش کسراں تے کتھے ہوئی آئی۔",
+    "query: اج اساں اگر آپڑیں بزرگاں دی انہاں گلاں تے غور نہ کیتا تے پچھتاوے دے سوا کج ہتھ نہ آسی۔",
+    "query: اسی طراں خیبر پختونخوا اچ بولی جانڑیں ولی دیگر زباناں دے ادب و ثقافت دی بی نمائندگی کیتی گئی اَئی۔",
+]
+emb = model.encode(sentences, normalize_embeddings=True)
+sim = model.similarity(emb, emb)   # 3 x 3 cosine-similarity matrix
+```
+
+The embeddings can be fed to any clustering or classification method. The benchmark uses k-means and logistic regression from scikit-learn on these normalized vectors. Use the scores to rank or group texts; they are not calibrated probabilities, so set any threshold on your own data.
+
+The example sentences are from the held-out test split of the Hindko Tokenizer, and they also occur in HindkoEmbedBench test documents; they were already published in the tokenizer card. Approximate meanings: (1) "Very few people know how and where drama was born." (2) "If we do not reflect on these words of our elders today, we will be left with nothing but regret." (3) "In the same way, the literature and culture of the other languages spoken in Khyber Pakhtunkhwa were also represented." The query means "the birth of drama".
+
+### Prefixes
+
+The model was trained and evaluated with the e5 prefixes below; results without them were not measured, so keep them.
+
+| Text | Prefix |
+|---|---|
+| Search query, headline, title, first line of a poem: anything used to look something up | `query: ` |
+| Document, article, passage or poem to be found | `passage: ` |
+| Both sides of a similarity comparison; texts to cluster or classify | `query: ` |
+
+In training, title, verse, prose-sentence, headline and continuation pairs used `query: ` for the anchor and `passage: ` for the positive and the negatives; pairs of neighbouring chunks used `query: ` on both sides. The benchmark uses the same scheme.
+
+Write the prefix into the text and call `model.encode`, as in the examples. `encode_query()` and `encode_document()` do **not** add these prefixes for this model.
+
+### Normalize input first
+
+Training and benchmark text was put in one canonical form: Unicode NFC, invisible and control characters removed, spaces collapsed, Arabic presentation forms mapped to letters, and Arabic-keyboard YEH/KAF folded where the word is provably Urdu or Hindko spelling. The Hindko Tokenizer repository ships a self-contained `normalize()` that produces this form:
+
+```python
+import os, sys
+from huggingface_hub import hf_hub_download
+
+qs = hf_hub_download("junaid008/hindko-tokenizer", "examples/quickstart.py")
+sys.path.insert(0, os.path.dirname(qs))
+from quickstart import normalize
+
+raw_text = "ایہہ  بہت کہٹ لوک جانڑدین"   # any Hindko input
+text = normalize(raw_text)            # here: the double space is collapsed
+```
+
+### Practical notes
+
+- **Length.** Input is cut at 512 tokens. For long documents, split them into passages (for example at sentence boundaries) and embed each one.
+- **Precision.** The weights are float16, and all benchmark numbers were measured with them on a GPU in float16. On a CPU, convert to float32 as shown above.
+- **Normalization of vectors.** Pass `normalize_embeddings=True`, as the benchmark did; cosine similarity and dot product then agree.
+
+## HindkoEmbedBench v1.0
+
+To our knowledge there was no benchmark for Hindko text embeddings, so one was built for this project (first used in round 1). It has five tasks in the style of MTEB, each with a **dev** version (used for model selection) and a **test** version (scored once per model, after selection). It was built **automatically from corpus metadata**: headlines and their articles, table-of-contents titles and their passages, the verse/prose label of book passages, and the book each passage comes from. No human relevance judgments were collected.
+
+| Task | Type | Main metric | Dev | Test |
+|---|---|---|---|---|
+| HeadlineRetrieval | retrieval: newspaper headline or web page title finds its article | nDCG@10 | 85 queries / 784 docs | 80 queries / 517 docs |
+| BookTitleRetrieval | retrieval: table-of-contents title (often a poem's first line) finds its passage(s) | nDCG@10 | 262 queries / 784 docs | 28 queries (57 relevant) / 517 docs |
+| PassageContinuation | retrieval: first half of a passage finds its second half | nDCG@10 | 416 queries / 416 docs | 360 queries / 360 docs |
+| VerseProseClassification | logistic regression on frozen embeddings | macro-F1 | 1,600 train / 297 eval | 1,600 train / 157 eval |
+| BookClustering | k-means, k = number of books | V-measure | 355 passages, 8 books | 176 passages, 5 books |
+
+**Overall score** = unweighted mean of the five main metrics.
+
+**How it was built.**
+- **Group-disjoint, leak-checked splits.** Whole newspaper editions, books and web sites sit on one side of the split. The benchmark uses the same record-level splits as the Hindko Tokenizer, published as [`split_manifest.jsonl`](https://github.com/junaidaslam2006/Hindko-Tokenizer/blob/main/tokenizer/splits/split_manifest.jsonl) (sha256 `76582d3a1e0afefe…`). Only Hindko-variety records are used.
+- **Shortcut filters.** A record's own title is stripped from its text; generic or repeated titles, titles that appear verbatim in the target, and near-duplicate documents are removed. Documents are cut at a sentence boundary at or below 3,000 characters. One known artifact remains: book-title relevant passages rarely start with a short genre label line (a line reading only "ghazal" or "naat", for example), while about 31 % of test book documents (5 % on dev) do, so a model could gain by ranking such documents low. Our training data strips these lines from both positives and mined negatives, so training does not reward this cue.
+- **Classification and clustering** follow the MTEB protocol: 10 logistic-regression runs on 256 examples per label drawn from a train-split pool of 1,600 passages; 10 k-means runs. The train pool and the evaluation books are disjoint.
+- **Frozen.** Every task file is hashed, and the harness refuses to run if a file changes.
+
+**Release status.** The benchmark data contains newspaper and book text and is **not released yet**. This card describes it; the benchmark code will be released with the training code.
+
+## Results
+
+All models were run with the same harness: sentence-transformers, `normalize_embeddings=True`, cosine scoring, each model with its own recommended prefixes. Neural models were run at `max_seq_length` 512 in float16, except the two round-1 models (256 tokens, float32). Full numbers: [`eval/results.csv`](eval/results.csv).
+
+### Test split (scored once per model, after selection)
+
+| Model | Tokens | Overall | Headline nDCG@10 | BookTitle nDCG@10 | Continuation nDCG@10 | Verse/prose macro-F1 | Clustering V-measure |
+|---|---:|---:|---:|---:|---:|---:|---:|
+| **Hindko Embeddings (this model)** | 512 | **0.7391** | **0.6372** | 0.3460 | **0.8756** | 0.9801 | **0.8566** |
+| Round-1 e5-small fine-tune (ours) | 256 | 0.7197 | 0.6091 | 0.3664 | 0.8314 | 0.9736 | 0.8177 |
+| Round-1 model, e5-base fine-tune (ours) | 256 | 0.6923 | 0.6097 | 0.2740 | 0.8340 | **0.9827** | 0.7612 |
+| Round-2 e5-base fine-tune (ours, runner-up on dev) | 512 | 0.6919 | 0.6027 | 0.2422 | 0.8350 | 0.9818 | 0.7979 |
+| TF-IDF char 2-4-gram (lexical reference) | n/a | 0.6864 | 0.6173 | 0.3756 | 0.7899 | 0.9597 | 0.6895 |
+| BAAI/bge-m3 | 512 | 0.6332 | 0.5960 | **0.3778** | 0.7174 | 0.9578 | 0.5168 |
+| intfloat/multilingual-e5-large (base model) | 512 | 0.5977 | 0.5886 | 0.3122 | 0.6706 | 0.9413 | 0.4758 |
+| intfloat/multilingual-e5-small | 512 | 0.5619 | 0.5381 | 0.2934 | 0.5664 | 0.9197 | 0.4918 |
+| Qwen/Qwen3-Embedding-0.6B | 512 | 0.5576 | 0.5634 | 0.2784 | 0.6186 | 0.9051 | 0.4227 |
+| intfloat/multilingual-e5-base | 512 | 0.5532 | 0.5626 | 0.2776 | 0.6139 | 0.8540 | 0.4580 |
+| sentence-transformers/LaBSE | 512 | 0.5314 | 0.5064 | 0.1608 | 0.5383 | 0.9616 | 0.4899 |
+| sentence-transformers/paraphrase-multilingual-mpnet-base-v2 | 512 | 0.4428 | 0.3399 | 0.0815 | 0.5030 | 0.9018 | 0.3879 |
+
+Best value per column in bold. The public models are untuned. They were first run at 256 tokens (float32) in round 1 and re-run at 512 tokens (float16) for this comparison, so length and precision changed together. On test this raised six of the seven (bge-m3: 0.6185 to 0.6332; multilingual-e5-large: 0.5605 to 0.5977) and left paraphrase-multilingual-mpnet unchanged (0.4429 to 0.4428), so the 512-token numbers are the stronger comparison overall. On dev, LaBSE (0.5200 to 0.5139) and multilingual-e5-small (0.5371 to 0.5353) fell slightly.
+
+Of our own models, only this model and the round-2 runner-up were scored on test in round 2; the two round-1 models were scored on test in round 1. The round-1 e5-small model scores second on test (0.7197) but was not chosen in round 1, because its dev score was lower.
+
+### Dev split (used for model selection)
+
+| Model | Tokens | Overall | Headline | BookTitle | Continuation | Verse/prose | Clustering |
+|---|---:|---:|---:|---:|---:|---:|---:|
+| **Hindko Embeddings (this model, `r2-e5l`)** | 512 | **0.7117** | 0.6357 | 0.4257 | **0.8308** | 0.9406 | 0.7258 |
+| Round-2 e5-base fine-tune (`r2-e5b`) | 512 | 0.7079 | 0.6006 | 0.4464 | 0.8176 | 0.9411 | **0.7339** |
+| Round-2 e5-base, no mined negatives or book blocks (`r2-e5b-plain`) | 512 | 0.7066 | 0.5941 | 0.4408 | 0.8295 | 0.9453 | 0.7234 |
+| Round-2 e5-small fine-tune (`r2-e5s`) | 512 | 0.7039 | 0.6191 | 0.4358 | 0.8094 | **0.9480** | 0.7070 |
+| Round-2 bge-m3 fine-tune (`r2-bgem3`) | 512 | 0.7008 | **0.6459** | 0.4435 | 0.8246 | 0.9429 | 0.6470 |
+| Round-1 model, e5-base fine-tune | 256 | 0.6851 | 0.6299 | 0.3288 | 0.8081 | 0.9432 | 0.7157 |
+| Round-1 e5-small fine-tune | 256 | 0.6700 | 0.6193 | 0.2778 | 0.7943 | 0.9382 | 0.7204 |
+| TF-IDF char 2-4-gram | n/a | 0.6667 | 0.6049 | **0.4594** | 0.7655 | 0.9463 | 0.5573 |
+| BAAI/bge-m3 | 512 | 0.5923 | 0.6183 | 0.2443 | 0.6617 | 0.9363 | 0.5011 |
+| intfloat/multilingual-e5-large | 512 | 0.5872 | 0.6028 | 0.2004 | 0.6524 | 0.9425 | 0.5379 |
+| intfloat/multilingual-e5-base | 512 | 0.5737 | 0.5728 | 0.1827 | 0.6293 | 0.9381 | 0.5457 |
+| Qwen/Qwen3-Embedding-0.6B | 512 | 0.5449 | 0.5327 | 0.1735 | 0.6297 | 0.8985 | 0.4902 |
+| intfloat/multilingual-e5-small | 512 | 0.5353 | 0.5467 | 0.1186 | 0.5698 | 0.9311 | 0.5101 |
+| sentence-transformers/LaBSE | 512 | 0.5139 | 0.4165 | 0.2089 | 0.5036 | 0.9455 | 0.4948 |
+| sentence-transformers/paraphrase-multilingual-mpnet-base-v2 | 512 | 0.4280 | 0.2822 | 0.1138 | 0.4360 | 0.8869 | 0.4210 |
+
+Same metrics as the test table. On dev this model is ahead of TF-IDF on three of five tasks (headline, continuation, clustering) and behind it on book titles (0.4257 vs 0.4594) and, narrowly, on verse/prose (0.9406 vs 0.9463).
+
+### Retrieval details (test)
+
+| Model | Headline R@1 | Headline R@10 | Headline nDCG@10, newspaper (72 q) | Headline nDCG@10, web (8 q) | BookTitle R@1 | BookTitle R@10 | BookTitle MRR@10 |
+|---|---:|---:|---:|---:|---:|---:|---:|
+| **This model** | **0.5625** | **0.7125** | **0.6072** | 0.9077 | 0.1548 | 0.4494 | 0.3608 |
+| TF-IDF char 2-4-gram | 0.5500 | 0.6875 | 0.5879 | 0.8827 | 0.1875 | **0.5030** | 0.3579 |
+| BAAI/bge-m3 (512) | 0.5500 | 0.6500 | 0.5581 | 0.9375 | **0.2262** | 0.4286 | **0.4113** |
+| Round-1 model (256) | 0.5375 | 0.6750 | 0.5715 | **0.9539** | 0.1012 | 0.4315 | 0.2731 |
+
+R@k is recall at k; a book-title query can have several relevant passages (57 for 28 queries).
+
+### Charts
+
+The images are in [`benchmarks/`](benchmarks) (3200 x 1800 PNG, free to reuse under the licence) and are drawn from [`eval/results.csv`](eval/results.csv) and [`eval/selection.json`](eval/selection.json) by [`make_charts.py`](make_charts.py).
+
+![Overall test score of every model](benchmarks/01_test_overall_all_models.png)
+
+![Per-task test scores: this model, TF-IDF and bge-m3](benchmarks/02_test_per_task.png)
+
+![Dev selection pool and the tie band](benchmarks/03_dev_candidates.png)
+
+![Round 2 vs round 1 on test](benchmarks/04_round1_vs_round2.png)
+
+![Dev score against model size](benchmarks/05_size_vs_score.png)
+
+![Book-title retrieval in detail, dev and test](benchmarks/06_booktitle_detail.png)
+
+## Training
+
+### Data
+
+Pairs were built only from the **train split** of a Hindko corpus: issues of the *Weekly Hindkowan* (Peshawar), the *Gandhara Hindko Academy* book series (much of it poetry) and Hindko web pages. Each pair is an anchor and a positive; some rows also carry mined hard negatives.
+
+| Dataset | Pair (anchor → positive) | Rows | Mined negatives per row |
+|---|---|---:|---:|
+| title | table-of-contents title → its passage (1,886); first line of a poem → the rest of the poem (2,505) | 4,391 | 3 (2 from the same book, 1 from another book) |
+| verse_ict | one line of a poem → the poem with that line removed (the line is kept in about 10 % of rows) | 2,949 | 1 (same book) |
+| prose_q | headline or web page title → article body (1,585: 1,447 news, 138 web); one sentence → the surrounding text (up to 900 characters) with that sentence removed, though it is kept in about 10 % of rows (5,354) | 6,939 | 1 |
+| continuation | first half of a passage of up to 1,200 characters → second half | 12,904 | 0 |
+| neighbour | a chunk of about 500 characters → the next chunk of the same text | 14,350 | 0 |
+| **Total** | 87,176 distinct texts | **41,533** | **23,061** |
+
+- **Sources:** 24,745 rows from books, 14,073 from newspapers and 2,715 from web pages.
+- **Variety:** 39,517 rows come from Hindko-variety records. The title, verse and prose-sentence datasets also use 1,268 Urdu-variety and 748 mixed-variety rows from the same corpus.
+- **Hard negatives** were mined with a character 2-4-gram TF-IDF index fit only on training documents (13,519 documents). A candidate was rejected if it was the same record, in the same near-duplicate cluster or the same chapter, contained the anchor (also in variant spellings), or re-used a large part of the positive.
+
+**Leak prevention.**
+- Only train-split records were used, and none from a split group that has dev or test records (230 such train records were excluded).
+- One train book (204 records) was excluded because it is a re-edition of a dev book.
+- Any text with word 8-gram containment of 0.3 or more in any dev or test text was dropped, under two text normalizations. Anchors equal to or close to a dev or test query (character 3-gram Jaccard 0.5 or more), and anchors of 12 or more characters contained in any dev or test text, were dropped as well.
+- A separately implemented audit script (it shares no code with the pair builder) re-checked every row against the split manifest and the frozen benchmark files: **0 violations** in all six checks. The highest remaining 8-gram containment is 0.2973.
+- The tokenizer is the base model's, unchanged; no vocabulary was added.
+
+### Recipe
+
+- **Loss:** `CachedMultipleNegativesRankingLoss` with scale 20, i.e. temperature 0.05. Every other positive and negative in the batch also serves as a negative.
+- **Batches:** 256 rows, each batch drawn from one dataset. Title and verse rows are drawn in **same-book blocks of 32**, so many in-batch negatives are other poems and chapters of the same book. A conflict rule keeps duplicate texts, two rows from one record or duplicate cluster, two passages of one chapter, and a row whose mined negative is another row's positive out of the same batch.
+- **Hyperparameters:** learning rate 2e-5, 2 epochs, 322 optimizer steps (161 per epoch), linear warmup over 10 % of the steps and linear decay, weight decay 0.01, AdamW, bf16 mixed precision, `max_seq_length` 512, gradient-cache mini-batch 32, seed 20261001.
+- **Hardware:** one NVIDIA A100-SXM4-80GB GPU; 1,620 s of training (27 minutes), 25.4 GB peak memory.
+- **Output:** saved in float16. The dev and test scores of this model were measured from the float16 weights that are published here.
+
+**What changed from round 1.** The round-1 model (multilingual-e5-base) was trained on 30,431 pairs (headline, book title, continuation and neighbour pairs) at 256 tokens, with in-batch negatives only. Round 2 added the first-line, verse-line and prose-sentence pairs (4,391 title rows against 1,390 book-title pairs in round 1), mined hard negatives, same-book blocks, 512 tokens and the larger base model.
+
+**Ablation.** The same e5-base recipe without mined negatives and same-book blocks (`r2-e5b-plain`) scored 0.7066 on dev against 0.7079 with them: a small difference overall (+0.0013), with gains on headlines (+0.0065), book titles (+0.0056) and clustering (+0.0105), and losses on continuation (-0.0119) and verse/prose (-0.0042). It is one seed on dev, so these differences are within noise.
+
+## Model selection protocol
+
+1. **The benchmark was frozen first.** Task files are hashed, and the harness refuses to run on changed files.
+2. **Recipe gate.** After the first round-2 run (`r2-e5b`), a gate fixed in advance decided the recipe for the remaining runs: book-title nDCG@10 at least 0.36, overall at least 0.685 and clustering at least 0.68 on dev. It passed (0.4464, 0.7079, 0.7339), so the remaining runs (e5-large, e5-small and bge-m3) were trained with mined negatives and same-book blocks.
+3. **Candidates.** Five round-2 models (e5-small, e5-base, e5-large, bge-m3, and e5-base without mined negatives) were trained on the same pairs. Each was evaluated on dev from its saved float16 weights at 512 tokens. The pool also held the two round-1 models, with their round-1 dev scores (0.6851 and 0.6700).
+4. **Rule, written before any round-2 result:** take the highest dev overall score; among models within 0.003 of it, take the smallest. `r2-e5l` scored 0.7117. The next best, `r2-e5b`, scored 0.7079 (0.0038 lower) and was outside the band, so `r2-e5l` was selected and `r2-e5b` became the runner-up. The selection file was written before any round-2 test result existed.
+5. **Test, once.** The selected model and the runner-up were scored on the test split once. The untuned public models were re-run at 512 tokens after the selection and were never in the pool.
+
+The round-2 choice used dev only, and test was scored once per model, after it. Test is not untouched across rounds, though: the round-1 test scores of the public models and of our round-1 models were known when round 2 was planned, and the plan cited them, plus a test-split document-length statistic, when it chose the candidate base models and adopted 512 tokens. No checkpoint, hyperparameter or threshold was chosen on test. The dev scores of the round-2 candidates (0.7008 to 0.7117) differ by about the dev standard error estimated in the training plan (about 0.01), so the choice of e5-large over e5-base is not statistically established on dev. On test the two models moved in different directions: 0.7391 for e5-large and 0.6919 for e5-base.
+
+## Intended uses
+
+- Semantic search and retrieval over Hindko text in Urdu script, including retrieval for RAG systems.
+- Similarity, clustering and classification features for Hindko documents, verses and sentences.
+- A baseline for research on embeddings for low-resource, Perso-Arabic-script languages.
+
+**Out of scope.** Romanized Hindko, speech transcripts, cross-lingual search (for example an English query over Hindko documents) and other languages were not evaluated; the base model's multilingual ability may or may not survive fine-tuning. Do not use similarity scores alone for decisions about people.
+
+## Limitations and bias
+
+- **Book-title retrieval is a weak spot.** On test, this model scores 0.3460 nDCG@10 against 0.3756 for TF-IDF and 0.3778 for bge-m3; our round-1 e5-small model also scored higher (0.3664). On dev it is also behind TF-IDF (0.4257 vs 0.4594). It also has the lowest dev book-title score of the five round-2 candidates (0.4257, against 0.4358 to 0.4464); it was selected on the overall score. For title or first-line lookup, combine it with a lexical search (hybrid retrieval).
+- **Residual overlap.** The leak filters drop training texts with word 8-gram containment of 0.3 or more in dev or test text. A separate line-level fuzzy check still finds one test book-title query whose relevant passage has at least half of its lines closely matched in train-split newspaper reprints, and 1 to 3 dev book-title queries per book with at least 30 %. These may slightly favour models trained on this corpus on book-title retrieval; this model still scores below TF-IDF there.
+- **Small test sets.** The test split has 80 headline queries, only 28 book-title queries (from 5 books), 157 verse/prose items (the verse class is essentially one book) and 176 clustering passages from 5 books. Differences of a few points on these tasks are within noise. Dev and test are both reported for this reason.
+- **Automatic benchmark.** Relevance comes from metadata (headline to article, contents title to passage, book identity), not from human judgments. A few headlines are attached to the wrong article, and book identity is only a proxy for topic and style.
+- **Domain.** Training and evaluation text is newspaper prose (mostly from one Peshawar weekly), literary books (much of it poetry) and a small amount of web text. Expect weaker results on conversational, social-media, technical or legal text.
+- **Dialect and spelling.** Peshawari and literary spellings dominate; Hazara and other varieties are less represented. Hindko spelling varies (retroflex nasal, harakat, Arabic-keyboard letters). The model was trained on normalized text, so [normalize input](#normalize-input-first) the same way.
+- **Script.** Only Urdu (Perso-Arabic) script was evaluated; Romanized Hindko was not.
+- **Truncation.** Retrieval mostly sees the start of each document: benchmark documents are cut at 3,000 characters and the model reads at most 512 tokens.
+- **Selection noise.** See [Model selection protocol](#model-selection-protocol): the dev margin over the runner-up is smaller than the dev standard error.
+- **Bias.** The model reflects the viewpoints and vocabulary of the published newspapers and books it was trained on, and any biases of its multilingual base model. No bias evaluation was done.
+
+## Ethical considerations and privacy
+
+- **No training text is included.** The repository holds model weights, configuration and evaluation outputs only.
+- **Provenance.** The training and benchmark texts belong to their publishers and authors (*Weekly Hindkowan*, the *Gandhara Hindko Academy* and web sources). They are not distributed here, and the licence below does not cover them.
+- **Use with care.** Embeddings of personal text can reveal what the text is about. Treat stored embeddings with the same care as the text they come from.
+
+## License
+
+**[Creative Commons Attribution 4.0 International (CC BY 4.0)](https://creativecommons.org/licenses/by/4.0/)** for this model; full text in [`LICENSE`](LICENSE).
+
+- You may use, share and adapt the model, including commercially.
+- Give credit: "Hindko Embeddings" by junaid008, with a link to this repository (the citation below is enough).
+- The base model, [intfloat/multilingual-e5-large](https://huggingface.co/intfloat/multilingual-e5-large), is released under the MIT License (Copyright (c) Microsoft Corporation). Its notice is reproduced in [`LICENSE`](LICENSE) and must be kept with copies of the weights.
+- The licence covers the files in this repository. It grants no rights in the texts the model was trained or evaluated on, which are not included.
+
+## Citation
+
+```bibtex
+@misc{hindko_embeddings_2026,
+  title        = {Hindko Embeddings: a sentence-embedding model for Hindko},
+  author       = {junaid008},
+  year         = {2026},
+  version      = {1.0.0},
+  howpublished = {\url{https://huggingface.co/junaid008/hindko-embeddings}},
+  note         = {CC BY 4.0. Fine-tuned from intfloat/multilingual-e5-large; selected on the dev split of HindkoEmbedBench v1.0 by a pre-registered rule; test overall 0.7391}
+}
+```
+
+Please also cite the base model:
+
+```bibtex
+@article{wang2024multilingual,
+  title   = {Multilingual E5 Text Embeddings: A Technical Report},
+  author  = {Wang, Liang and Yang, Nan and Huang, Xiaolong and Yang, Linjun and Majumder, Rangan and Wei, Furu},
+  journal = {arXiv preprint arXiv:2402.05672},
+  year    = {2024}
+}
+```
+
+## Acknowledgements
+
+Heartfelt thanks to the **Chief Minister's Policy Office**, the **Weekly Hindkowan** and the **Gandhara Hindko Board** for supporting this work and sharing the texts that made it possible.
+
+This project was built together with **Muhammad Ozair**.
+
+## Contact
+
+**junaid008** on Hugging Face. Please open a discussion on this repository for questions, errors or clearance matters.
+
+## Changelog
+
+| Version | Date | Changes |
+|---|---|---|
+| 1.0.0 | 2026-10-01 | First release: multilingual-e5-large fine-tuned on 41,533 pairs from a Hindko corpus (round 2, `r2-e5l`); HindkoEmbedBench v1.0 results, selection record and benchmark images. |

config.json

@@ -0,0 +1,30 @@
+{
+  "add_cross_attention": false,
+  "architectures": [
+    "XLMRobertaModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "dtype": "float16",
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "is_decoder": false,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "xlm-roberta",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "tie_word_embeddings": true,
+  "transformers_version": "5.3.0",
+  "type_vocab_size": 1,
+  "use_cache": false,
+  "vocab_size": 250002
+}

config_sentence_transformers.json

@@ -0,0 +1,14 @@
+{
+  "__version__": {
+    "pytorch": "2.11.0+cu128",
+    "sentence_transformers": "6.1.0",
+    "transformers": "5.3.0"
+  },
+  "default_prompt_name": null,
+  "model_type": "SentenceTransformer",
+  "prompts": {
+    "document": "",
+    "query": ""
+  },
+  "similarity_fn_name": "cosine"
+}

eval/results.csv

@@ -0,0 +1,28 @@
+model,model_id,base_model,kind,role,profile,split,max_seq_length,dtype,overall,HeadlineRetrieval,BookTitleRetrieval,PassageContinuation,VerseProseClassification,BookClustering,HL_R1,HL_R10,HL_news,HL_web,HL_nq,BT_R1,BT_R10,BT_MRR
+LaBSE,sentence-transformers/LaBSE,sentence-transformers/LaBSE,public,baseline,labse,dev,512,float16,0.5139,0.4165,0.2089,0.5036,0.9455,0.4948,0.3176,0.5294,0.4239,0.372,85,0.1412,0.3015,0.181
+LaBSE,sentence-transformers/LaBSE,sentence-transformers/LaBSE,public,baseline,labse,test,512,float16,0.5314,0.5064,0.1608,0.5383,0.9616,0.4899,0.4,0.6375,0.4896,0.658,80,0.0714,0.2232,0.153
+Qwen3-Embedding-0.6B,Qwen/Qwen3-Embedding-0.6B,Qwen/Qwen3-Embedding-0.6B,public,baseline,qwen3,dev,512,float16,0.5449,0.5327,0.1735,0.6297,0.8985,0.4902,0.4353,0.6353,0.4797,0.8552,85,0.1107,0.2519,0.1495
+Qwen3-Embedding-0.6B,Qwen/Qwen3-Embedding-0.6B,Qwen/Qwen3-Embedding-0.6B,public,baseline,qwen3,test,512,float16,0.5576,0.5634,0.2784,0.6186,0.9051,0.4227,0.4875,0.6375,0.5297,0.8663,80,0.1458,0.3393,0.3135
+bge-m3,BAAI/bge-m3,BAAI/bge-m3,public,baseline,bge-m3,dev,512,float16,0.5923,0.6183,0.2443,0.6617,0.9363,0.5011,0.5412,0.6824,0.5743,0.8859,85,0.1641,0.3435,0.2135
+bge-m3,BAAI/bge-m3,BAAI/bge-m3,public,baseline,bge-m3,test,512,float16,0.6332,0.596,0.3778,0.7174,0.9578,0.5168,0.55,0.65,0.5581,0.9375,80,0.2262,0.4286,0.4113
+multilingual-e5-base,intfloat/multilingual-e5-base,intfloat/multilingual-e5-base,public,baseline,e5,dev,512,float16,0.5737,0.5728,0.1827,0.6293,0.9381,0.5457,0.4588,0.6706,0.5452,0.7411,85,0.1031,0.2901,0.1504
+multilingual-e5-base,intfloat/multilingual-e5-base,intfloat/multilingual-e5-base,public,baseline,e5,test,512,float16,0.5532,0.5626,0.2776,0.6139,0.854,0.458,0.4875,0.6375,0.5243,0.9077,80,0.1488,0.3512,0.3161
+multilingual-e5-large,intfloat/multilingual-e5-large,intfloat/multilingual-e5-large,public,baseline,e5,dev,512,float16,0.5872,0.6028,0.2004,0.6524,0.9425,0.5379,0.4941,0.7294,0.5646,0.8348,85,0.126,0.2977,0.1709
+multilingual-e5-large,intfloat/multilingual-e5-large,intfloat/multilingual-e5-large,public,baseline,e5,test,512,float16,0.5977,0.5886,0.3122,0.6706,0.9413,0.4758,0.525,0.6625,0.5565,0.8772,80,0.1756,0.378,0.3971
+multilingual-e5-small,intfloat/multilingual-e5-small,intfloat/multilingual-e5-small,public,baseline,e5,dev,512,float16,0.5353,0.5467,0.1186,0.5698,0.9311,0.5101,0.4824,0.6235,0.5115,0.7609,85,0.084,0.1641,0.1047
+multilingual-e5-small,intfloat/multilingual-e5-small,intfloat/multilingual-e5-small,public,baseline,e5,test,512,float16,0.5619,0.5381,0.2934,0.5664,0.9197,0.4918,0.4875,0.5875,0.4988,0.8914,80,0.1726,0.3661,0.2995
+paraphrase-multilingual-mpnet-base-v2,sentence-transformers/paraphrase-multilingual-mpnet-base-v2,sentence-transformers/paraphrase-multilingual-mpnet-base-v2,public,baseline,mpnet,dev,512,float16,0.428,0.2822,0.1138,0.436,0.8869,0.421,0.1882,0.4,0.2702,0.3551,85,0.0573,0.187,0.0914
+paraphrase-multilingual-mpnet-base-v2,sentence-transformers/paraphrase-multilingual-mpnet-base-v2,sentence-transformers/paraphrase-multilingual-mpnet-base-v2,public,baseline,mpnet,test,512,float16,0.4428,0.3399,0.0815,0.503,0.9018,0.3879,0.2375,0.4625,0.3353,0.3814,80,0,0.1637,0.0686
+Round-1 model (e5-base fine-tune),r1-ft-e5-base,intfloat/multilingual-e5-base,round-1,incumbent; selected in round 1,e5,dev,256,float32,0.6851,0.6299,0.3288,0.8081,0.9432,0.7157,0.4941,0.7765,0.6024,0.7968,85,0.229,0.458,0.2894
+Round-1 model (e5-base fine-tune),r1-ft-e5-base,intfloat/multilingual-e5-base,round-1,incumbent; selected in round 1,e5,test,256,float32,0.6923,0.6097,0.274,0.834,0.9827,0.7612,0.5375,0.675,0.5715,0.9539,80,0.1012,0.4315,0.2731
+Round-1 e5-small fine-tune,r1-ft-e5-small,intfloat/multilingual-e5-small,round-1,incumbent,e5,dev,256,float32,0.67,0.6193,0.2778,0.7943,0.9382,0.7204,0.5294,0.7176,0.5912,0.7906,85,0.1679,0.416,0.2353
+Round-1 e5-small fine-tune,r1-ft-e5-small,intfloat/multilingual-e5-small,round-1,incumbent,e5,test,256,float32,0.7197,0.6091,0.3664,0.8314,0.9736,0.8177,0.55,0.675,0.5708,0.9539,80,0.247,0.4196,0.4438
+TF-IDF char 2-4-gram,tfidf-char2-4,,lexical,reference,,dev,,,0.6667,0.6049,0.4594,0.7655,0.9463,0.5573,0.5294,0.6824,0.5656,0.8442,85,0.3282,0.6069,0.4132
+TF-IDF char 2-4-gram,tfidf-char2-4,,lexical,reference,,test,,,0.6864,0.6173,0.3756,0.7899,0.9597,0.6895,0.55,0.6875,0.5879,0.8827,80,0.1875,0.503,0.3579
+Round-2 bge-m3 fine-tune,r2-bgem3,BAAI/bge-m3,round-2,candidate,bge-m3,dev,512,float16,0.7008,0.6459,0.4435,0.8246,0.9429,0.647,0.5529,0.7294,0.6064,0.8859,85,0.3397,0.5573,0.4077
+"Round-2 e5-base fine-tune, no mined negatives",r2-e5b-plain,intfloat/multilingual-e5-base,round-2,candidate (ablation),e5,dev,512,float16,0.7066,0.5941,0.4408,0.8295,0.9453,0.7234,0.4824,0.7059,0.5722,0.7272,85,0.3359,0.5534,0.4053
+Round-2 e5-base fine-tune,r2-e5b,intfloat/multilingual-e5-base,round-2,runner-up,e5,dev,512,float16,0.7079,0.6006,0.4464,0.8176,0.9411,0.7339,0.4941,0.7059,0.5744,0.7598,85,0.3473,0.5496,0.4137
+Round-2 e5-base fine-tune,r2-e5b,intfloat/multilingual-e5-base,round-2,runner-up,e5,test,512,float16,0.6919,0.6027,0.2422,0.835,0.9818,0.7979,0.5375,0.65,0.5706,0.8914,80,0.1071,0.3542,0.251
+Hindko Embeddings (this model),r2-e5l,intfloat/multilingual-e5-large,round-2,selected (released),e5,dev,512,float16,0.7117,0.6357,0.4257,0.8308,0.9406,0.7258,0.5294,0.7412,0.5963,0.875,85,0.2977,0.563,0.3826
+Hindko Embeddings (this model),r2-e5l,intfloat/multilingual-e5-large,round-2,selected (released),e5,test,512,float16,0.7391,0.6372,0.346,0.8756,0.9801,0.8566,0.5625,0.7125,0.6072,0.9077,80,0.1548,0.4494,0.3608
+Round-2 e5-small fine-tune,r2-e5s,intfloat/multilingual-e5-small,round-2,candidate,e5,dev,512,float16,0.7039,0.6191,0.4358,0.8094,0.948,0.707,0.5294,0.7176,0.5899,0.7968,85,0.3397,0.5458,0.4015

eval/selection.json

@@ -0,0 +1,40 @@
+{
+ "rule": "pool = round-2 dev results + incumbents ft-e5-base (size 2) and ft-e5-small (size 1) with their round-1 dev scores; best = max overall; near = overall >= best - 0.003; selected = smallest size rank in near (ties by higher score); runner-up = best-scoring other model",
+ "dev_scores": {
+  "ft-e5-base": 0.6851297501603847,
+  "ft-e5-small": 0.6700276166417629,
+  "r2-bgem3": 0.7007568407810918,
+  "r2-e5b": 0.7079351835925609,
+  "r2-e5b-plain": 0.7066095544829916,
+  "r2-e5l": 0.7117165057564516,
+  "r2-e5s": 0.7038754488016046
+ },
+ "near": [
+  "r2-e5l"
+ ],
+ "selected": "r2-e5l",
+ "runner_up": "r2-e5b",
+ "tie": 0.003,
+ "written_utc": "2026-10-01T17:48:17Z",
+ "gate": {
+  "pass": true,
+  "decided_utc": "2026-10-01T15:38:50Z",
+  "BookTitleRetrieval": 0.446437329167755,
+  "overall": 0.7079351835925609,
+  "BookClustering": 0.7339101300956627
+ },
+ "pairs_sha256": {
+  "rows.jsonl": "9ddde8f6ef2750166e4d895b8aba55ba394ba954961169ab0aec0f5d4a541faa",
+  "texts.jsonl": "f0c31d84f79e7c3cee0b45e708ec1bb1929f9e2d99d7fbdec66ecb47a1024a28"
+ },
+ "size_rank": {
+  "r2-e5s": 1,
+  "ft-e5-small": 1,
+  "r2-e5b": 2,
+  "r2-e5b-plain": 2,
+  "ft-e5-base": 2,
+  "r2-e5l": 3,
+  "r2-bgem3": 4
+ },
+ "notes": "dev_scores of ft-e5-base and ft-e5-small are their round-1 dev results (max_seq_length 256, float32); all other pool members were evaluated at max_seq_length 512 in float16. Test was run once, after this selection, on the selected model and the runner-up. gate = the pre-registered recipe gate evaluated on r2-e5b dev (BookTitleRetrieval >= 0.36, overall >= 0.685, BookClustering >= 0.68); pass -> the large runs used mined hard negatives and same-book blocks."
+}

eval/training.json

@@ -0,0 +1,106 @@
+{
+ "runs": {
+  "r2-bgem3": {
+   "base": "BAAI/bge-m3",
+   "lr": 1.5e-05,
+   "epochs": 2,
+   "batch": 256,
+   "mini": 32,
+   "max_seq_length": 512,
+   "negatives": "on",
+   "block": 32,
+   "steps": 322,
+   "train_seconds": 1608.7,
+   "peak_gb": 25.564,
+   "gpu": "NVIDIA A100-SXM4-80GB"
+  },
+  "r2-e5b-plain": {
+   "base": "intfloat/multilingual-e5-base",
+   "lr": 3e-05,
+   "epochs": 2,
+   "batch": 256,
+   "mini": 64,
+   "max_seq_length": 512,
+   "negatives": "off",
+   "block": 0,
+   "steps": 322,
+   "train_seconds": 369.8,
+   "peak_gb": 17.084,
+   "gpu": "NVIDIA A100-SXM4-80GB"
+  },
+  "r2-e5b": {
+   "base": "intfloat/multilingual-e5-base",
+   "lr": 3e-05,
+   "epochs": 2,
+   "batch": 256,
+   "mini": 64,
+   "max_seq_length": 512,
+   "negatives": "on",
+   "block": 32,
+   "steps": 322,
+   "train_seconds": 548.6,
+   "peak_gb": 17.091,
+   "gpu": "NVIDIA A100-SXM4-80GB"
+  },
+  "r2-e5l": {
+   "base": "intfloat/multilingual-e5-large",
+   "lr": 2e-05,
+   "epochs": 2,
+   "batch": 256,
+   "mini": 32,
+   "max_seq_length": 512,
+   "negatives": "on",
+   "block": 32,
+   "steps": 322,
+   "train_seconds": 1620.1,
+   "peak_gb": 25.447,
+   "gpu": "NVIDIA A100-SXM4-80GB"
+  },
+  "r2-e5s": {
+   "base": "intfloat/multilingual-e5-small",
+   "lr": 5e-05,
+   "epochs": 2,
+   "batch": 256,
+   "mini": 128,
+   "max_seq_length": 512,
+   "negatives": "on",
+   "block": 32,
+   "steps": 322,
+   "train_seconds": 309.6,
+   "peak_gb": 14.999,
+   "gpu": "NVIDIA A100-SXM4-80GB"
+  }
+ },
+ "train_rows": {
+  "title": 4391,
+  "verse_ict": 2949,
+  "prose_q": 6939,
+  "continuation": 12904,
+  "neighbour": 14350,
+  "total": 41533
+ },
+ "common": {
+  "epochs": 2,
+  "batch_size": 256,
+  "warmup": "10% of steps, linear decay",
+  "weight_decay": 0.01,
+  "loss": "CachedMultipleNegativesRankingLoss(scale=20.0, mini_batch_size=mini)",
+  "seed": 20261001,
+  "prefixes_e5": {
+   "title, verse_ict, prose_q, continuation": [
+    "query: ",
+    "passage: "
+   ],
+   "neighbour": [
+    "query: ",
+    "query: "
+   ]
+  },
+  "mixed_precision": "bf16 on GPUs with compute capability >= 8 (A100)",
+  "saved_dtype": "float16"
+ },
+ "released": {
+  "name": "r2-e5l",
+  "weights": "1068 MB fp16"
+ }
+}

make_charts.py

@@ -0,0 +1,402 @@
+"""Benchmark images for Hindko Embeddings (model card), in the style of the Hindko Tokenizer charts.
+
+Every score is read from eval/results.csv and eval/selection.json in this folder; nothing is typed in by hand
+except the task sizes (HindkoEmbedBench v1.0 description) and the parameter counts of the public models (PARAMS).
+Writes benchmarks/<nn>_<name>.png (3200 x 1800): HTML pages rendered by headless Chrome at 2x, with the Google
+Fonts embedded so they always load.
+
+    python make_charts.py            # all charts
+    python make_charts.py 00,03      # only these
+    python make_charts.py --keep-html
+"""
+import base64
+import csv
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import urllib.request
+
+sys.dont_write_bytecode = True
+HERE = os.path.dirname(os.path.abspath(__file__))
+EVAL = os.path.join(HERE, 'eval')
+OUT = os.path.join(HERE, 'benchmarks')
+WORK = os.path.join(HERE, '.render')          # HTML sources and the Chrome profile; removed afterwards
+CHROME = (os.environ.get('CHROME') or shutil.which('chrome') or shutil.which('google-chrome') or shutil.which('chromium')
+          or os.path.join(os.environ.get('PROGRAMFILES', ''), 'Google', 'Chrome', 'Application', 'chrome.exe'))
+UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0 Safari/537.36'}
+W, H = 1600, 900
+
+# ------------------------------------------------------------------------------------------------ data
+R = {}
+for r in csv.DictReader(open(os.path.join(EVAL, 'results.csv'), encoding='utf-8')):
+    R[(r['model_id'], r['split'])] = r
+SEL = json.load(open(os.path.join(EVAL, 'selection.json'), encoding='utf-8'))
+
+OURS = SEL['selected']                        # r2-e5l
+RUNNER = SEL['runner_up']                     # r2-e5b
+TFIDF = 'tfidf-char2-4'
+R1 = 'r1-ft-e5-base'                          # round-1 model (selected in round 1)
+R1S = 'r1-ft-e5-small'
+BGE = 'BAAI/bge-m3'
+E5L = 'intfloat/multilingual-e5-large'
+POOL_ID = {'ft-e5-base': R1, 'ft-e5-small': R1S}   # selection.json names -> results.csv model_id
+TASKS = ['HeadlineRetrieval', 'BookTitleRetrieval', 'PassageContinuation', 'VerseProseClassification', 'BookClustering']
+TASK_LABEL = {'overall': ('Overall', 'mean of 5'), 'HeadlineRetrieval': ('Headline', 'nDCG@10'),
+              'BookTitleRetrieval': ('Book title', 'nDCG@10'), 'PassageContinuation': ('Continuation', 'nDCG@10'),
+              'VerseProseClassification': ('Verse / prose', 'macro-F1'), 'BookClustering': ('Book clustering', 'V-measure')}
+# HindkoEmbedBench v1.0 task sizes (benchmark description)
+SIZES = {'dev': '85 headline, 262 book-title and 416 continuation queries; 297 verse/prose and 355 clustering items',
+         'test': '80 headline, 28 book-title and 360 continuation queries; 157 verse/prose and 176 clustering items'}
+# parameters in millions: e5-small/-base and mpnet counted from their safetensors; e5-large, bge-m3, LaBSE and
+# Qwen3-Embedding-0.6B as listed on their Hugging Face model pages. Fine-tuning does not change the count.
+PARAMS = {'intfloat/multilingual-e5-small': 118, 'intfloat/multilingual-e5-base': 278, 'intfloat/multilingual-e5-large': 560,
+          'BAAI/bge-m3': 568, 'sentence-transformers/LaBSE': 471, 'Qwen/Qwen3-Embedding-0.6B': 596,
+          'sentence-transformers/paraphrase-multilingual-mpnet-base-v2': 278}
+
+PUBLIC = [(m, s) for (m, s), r in R.items() if r['kind'] == 'public' and s == 'test']
+
+
+def v(mid, split, col='overall'):
+    return float(R[(mid, split)][col])
+
+
+def f3(x):
+    return '%.3f' % x
+
+
+def f4(x):
+    return '%.4f' % x
+
+
+def short(mid):
+    return mid.split('/')[-1]
+
+
+# ------------------------------------------------------------------------------------------------ page
+CSS = """
+:root{--ink:#0F172A;--muted:#5B6576;--grid:#E4E8EE;--accent:#2F6FDB;--accent-ink:#1D4FA8;--bar:#C9D0DA;--soft:#EAF1FC}
+*{box-sizing:border-box}
+html,body{margin:0;width:%(W)dpx;height:%(H)dpx;overflow:hidden;background:#FFFFFF;color:var(--ink);font-family:'IBM Plex Sans',sans-serif}
+.page{position:absolute;inset:0;padding:56px 80px 0}
+.eyebrow{font-size:17px;font-weight:600;letter-spacing:.16em;text-transform:uppercase;color:var(--accent)}
+h1{font-family:'Source Serif 4',serif;font-weight:600;font-size:54px;line-height:1.08;letter-spacing:-.015em;margin:12px 0 10px}
+.sub{font-size:23px;line-height:1.4;color:var(--muted);max-width:1400px;margin:0}
+.foot{position:absolute;left:80px;right:80px;bottom:34px;display:flex;justify-content:space-between;gap:40px;font-size:16px;color:var(--muted);border-top:1px solid var(--grid);padding-top:14px}
+.foot b{color:var(--ink);font-weight:600}
+.bars{margin-top:30px}
+.row{display:grid;grid-template-columns:380px 1fr 120px;align-items:center;column-gap:22px;height:%(rowh)dpx}
+.lab{font-size:21px;line-height:1.15;text-align:right}
+.lab small{display:block;font-size:15px;color:var(--muted)}
+.trk{position:relative;height:%(barh)dpx}
+.fill{position:absolute;left:0;top:0;bottom:0;background:var(--bar);border-radius:0 4px 4px 0}
+.val{font-size:21px;font-variant-numeric:tabular-nums;color:var(--ink)}
+.row.me .lab{font-weight:700}.row.me .fill{background:var(--accent)}.row.me .val{font-weight:700;color:var(--accent-ink)}
+.row.ref .fill{background:#8E9AAB}
+.gl{position:absolute;top:-8px;bottom:-8px;border-left:1px dashed var(--grid)}
+table{border-collapse:collapse;width:100%%;margin-top:30px;font-variant-numeric:tabular-nums}
+th{font-size:15px;font-weight:600;letter-spacing:.06em;text-transform:uppercase;color:var(--muted);text-align:right;padding:0 18px 12px;border-bottom:2px solid var(--ink);vertical-align:bottom}
+th small{display:block;font-size:13px;letter-spacing:.02em;text-transform:none;font-weight:400}
+th:first-child,td:first-child{text-align:left;padding-left:0}
+td{font-size:22px;text-align:right;padding:13px 18px;border-bottom:1px solid var(--grid)}
+td small{font-size:15px;color:var(--muted);margin-left:8px;font-weight:400}
+td .best{border-bottom:3px solid currentColor;padding-bottom:1px}
+tr.me td{font-weight:700;color:var(--accent-ink);background:var(--soft)}
+tr.me td:first-child{padding-left:14px;border-radius:6px 0 0 6px}
+.legend{display:flex;gap:34px;font-size:19px;margin-top:26px}
+.legend span{display:inline-flex;align-items:center;gap:10px}
+.legend i{display:inline-block;width:22px;height:16px;border-radius:3px}
+.note{font-size:19px;line-height:1.45;color:var(--muted);max-width:1440px;margin-top:18px}
+"""
+
+HEAD = ('<!doctype html><html><head><meta charset="utf-8">'
+        '<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Sans:wght@400;600;700'
+        '&family=Source+Serif+4:opsz,wght@8..60,600&display=block">'
+        '<style>%s</style></head><body><div class="page">')
+
+
+def page(title, sub, body, foot_left, rowh=52, barh=30):
+    return (HEAD % (CSS % dict(W=W, H=H, rowh=rowh, barh=barh)) +
+            '<div class="eyebrow">Hindko Embeddings &middot; Benchmarks</div><h1>%s</h1><p class="sub">%s</p>%s'
+            '<div class="foot"><span>%s</span><span style="white-space:nowrap">huggingface.co/junaid008/hindko-embeddings</span></div></div></body></html>'
+            % (title, sub, body, foot_left))
+
+
+def foot(split, extra=''):
+    name = 'Test split, scored once after selection on dev' if split == 'test' else 'Dev (validation) split'
+    return '<b>HindkoEmbedBench v1.0 &middot; %s</b>%s' % (name, extra)
+
+
+def bars(items, vmax, grid=()):
+    """items: (label, sublabel, value, value_html, cls) with cls '', 'me' or 'ref'."""
+    out = ['<div class="bars">']
+    for lab, sublab, val, vh, cls in items:
+        gl = ''.join('<span class="gl" style="left:%.3f%%"></span>' % (100 * g / vmax) for g in grid)
+        out.append('<div class="row %s"><div class="lab">%s%s</div><div class="trk">%s<div class="fill" style="width:%.3f%%"></div></div>'
+                   '<div class="val">%s</div></div>' % (cls, lab, '<small>%s</small>' % sublab if sublab else '', gl, 100 * val / vmax, vh))
+    out.append('</div>')
+    return ''.join(out)
+
+
+def legend(series):
+    return '<div class="legend">%s</div>' % ''.join('<span><i style="background:%s"></i>%s</span>' % (c, n) for n, c in series)
+
+
+def grouped(groups, series, vmax=1.0, width=1440, height=470, ticks=(0.2, 0.4, 0.6, 0.8, 1.0), x0=60, label_fmt=f3,
+            bar_w=None, gap=10, lab_size=16, below=None):
+    """Vertical grouped columns. groups: [(title, subtitle)]; series: [(name, color, [values], bold)]."""
+    y0, y1 = 22, height - 70
+    plot_w = width - x0 - 10
+    gw = plot_w / len(groups)
+    n = len(series)
+    bw = bar_w or min(64, (gw - 50 - gap * (n - 1)) / n)
+    svg = ['<svg viewBox="0 0 %d %d" width="%d" height="%d" style="display:block;margin-top:14px;overflow:visible">' % (width, height, width, height)]
+    Y = lambda val: y1 - (y1 - y0) * val / vmax
+    for t in ticks:
+        svg.append('<line x1="%d" x2="%d" y1="%.1f" y2="%.1f" stroke="#E4E8EE"/><text x="%d" y="%.1f" font-size="15" fill="#5B6576" text-anchor="end">%.1f</text>'
+                   % (x0, width - 10, Y(t), Y(t), x0 - 10, Y(t) + 5, t))
+    svg.append('<line x1="%d" x2="%d" y1="%d" y2="%d" stroke="#0F172A" stroke-width="1.5"/>' % (x0, width - 10, y1, y1))
+    for gi, (gt, gs) in enumerate(groups):
+        gx = x0 + gi * gw
+        total = n * bw + (n - 1) * gap
+        bx0 = gx + (gw - total) / 2
+        for si, (name, col, vals, bold) in enumerate(series):
+            val = vals[gi]
+            if val is None:
+                continue
+            x = bx0 + si * (bw + gap)
+            svg.append('<rect x="%.1f" y="%.1f" width="%.1f" height="%.1f" fill="%s" rx="3"/>' % (x, Y(val), bw, y1 - Y(val), col))
+            svg.append('<text x="%.1f" y="%.1f" font-size="%d" font-weight="%d" fill="%s" text-anchor="middle">%s</text>'
+                       % (x + bw / 2, Y(val) - 8, lab_size, 700 if bold else 400, '#1D4FA8' if bold else '#0F172A', label_fmt(val)))
+        svg.append('<text x="%.1f" y="%d" font-size="19" font-weight="600" fill="#0F172A" text-anchor="middle">%s</text>' % (gx + gw / 2, y1 + 28, gt))
+        svg.append('<text x="%.1f" y="%d" font-size="15" fill="#5B6576" text-anchor="middle">%s</text>' % (gx + gw / 2, y1 + 50, gs))
+        if below:
+            svg.append(below(gi, gx, gw, y1))
+    svg.append('</svg>')
+    return ''.join(svg)
+
+
+# ------------------------------------------------------------------------------------------------ 00 scorecard
+def c00():
+    rows = [(OURS, 'Hindko Embeddings', 'this model &middot; 512 tokens'),
+            (TFIDF, 'TF-IDF char 2-4-gram', 'lexical reference'),
+            (RUNNER, 'Round-2 e5-base fine-tune', 'runner-up on dev &middot; 512 tokens'),
+            (R1, 'Round-1 model', 'our e5-base fine-tune &middot; 256 tokens'),
+            (R1S, 'Round-1 e5-small fine-tune', '256 tokens'),
+            (BGE, 'bge-m3', 'best public model &middot; 512 tokens'),
+            (E5L, 'multilingual-e5-large', 'base model, untuned &middot; 512 tokens')]
+    rows.sort(key=lambda r: -v(r[0], 'test'))     # ranked by overall test score, like the README table
+    cols = ['overall'] + TASKS
+    best = {c: max(v(m, 'test', c) for m, _, _ in rows) for c in cols}
+    head = ''.join('<th>%s<small>%s</small></th>' % TASK_LABEL[c] for c in cols)
+    trs = []
+    for m, name, sub in rows:
+        tds = ''.join('<td>%s</td>' % (('<span class="best">%s</span>' if v(m, 'test', c) == best[c] else '%s') % f3(v(m, 'test', c))) for c in cols)
+        trs.append('<tr%s><td>%s<small>%s</small></td>%s</tr>' % (' class="me"' if m == OURS else '', name, sub, tds))
+    body = '<table><tr><th>Model</th>%s</tr>%s</table>' % (head, ''.join(trs))
+    d_tf = v(OURS, 'test') - v(TFIDF, 'test')
+    d_bge = v(OURS, 'test') - v(BGE, 'test')
+    body += ('<p class="note">Overall: <b style="color:#0F172A">+%.3f</b> over TF-IDF and <b style="color:#0F172A">+%.3f</b> over bge-m3. '
+             'Ahead of TF-IDF on four of the five tasks; behind it, and behind bge-m3, on book-title retrieval (28 test queries).</p>' % (d_tf, d_bge))
+    return page('Hindko Embeddings on the held-out test set',
+                'Five automatic Hindko tasks; higher is better. The best value in each column is underlined.', body,
+                foot('test', ' &middot; neural models at max_seq_length 512 unless noted'))
+
+
+# ------------------------------------------------------------------------------------------------ 01 overall, all models
+def c01():
+    items = [('Hindko Embeddings', 'this model &middot; e5-large fine-tune', v(OURS, 'test'), f4(v(OURS, 'test')), 'me'),
+             ('Round-1 e5-small fine-tune', 'ours, round 1 &middot; 256 tokens', v(R1S, 'test'), f4(v(R1S, 'test')), ''),
+             ('Round-1 model', 'ours, round 1 &middot; e5-base &middot; 256 tokens', v(R1, 'test'), f4(v(R1, 'test')), ''),
+             ('Round-2 e5-base fine-tune', 'ours, runner-up on dev', v(RUNNER, 'test'), f4(v(RUNNER, 'test')), ''),
+             ('TF-IDF char 2-4-gram', 'lexical reference', v(TFIDF, 'test'), f4(v(TFIDF, 'test')), 'ref')]
+    for m, s in PUBLIC:
+        items.append((short(m).replace('paraphrase-', ''), 'public, untuned', v(m, 'test'), f4(v(m, 'test')), ''))
+    items.sort(key=lambda x: -x[2])
+    return page('Overall score on the test set',
+                'Mean of the five task scores for every model tested. Public models run untuned at 512 tokens; higher is better.',
+                bars(items, 0.8, grid=(0.2, 0.4, 0.6)) +
+                '<div style="display:grid;grid-template-columns:380px 1fr 120px;column-gap:22px;margin-top:2px"><span></span>'
+                '<div style="position:relative;height:20px;font-size:15px;color:#5B6576">%s</div><span></span></div>'
+                % ''.join('<span style="position:absolute;left:%.3f%%;transform:translateX(-50%%)">%.1f</span>' % (100 * t / 0.8, t) for t in (0.0, 0.2, 0.4, 0.6, 0.8)),
+                foot('test', ' &middot; max_seq_length 512, except the round-1 models (256) and TF-IDF'), rowh=47, barh=26)
+
+
+# ------------------------------------------------------------------------------------------------ 02 per task, test
+def c02():
+    cols = ['overall'] + TASKS
+    series = [('Hindko Embeddings (this model)', '#2F6FDB', [v(OURS, 'test', c) for c in cols], True),
+              ('TF-IDF char 2-4-gram', '#8E9AAB', [v(TFIDF, 'test', c) for c in cols], False),
+              ('bge-m3, best public model', '#C9D0DA', [v(BGE, 'test', c) for c in cols], False)]
+    body = legend([(n, c) for n, c, _, _ in series]) + grouped([TASK_LABEL[c] for c in cols], series, height=500)
+    return page('Per-task scores on the test set',
+                'Ahead of TF-IDF on four of five tasks. On book-title retrieval (only 28 test queries) TF-IDF and bge-m3 score higher.',
+                body, foot('test', ' &middot; neural models at max_seq_length 512'))
+
+
+# ------------------------------------------------------------------------------------------------ 03 dev candidates
+def c03():
+    pool = sorted(SEL['dev_scores'].items(), key=lambda kv: -kv[1])
+    best = pool[0][1]
+    lo, hi = 0.660, 0.720
+    X = lambda val: 100 * (val - lo) / (hi - lo)
+    sub = {'r2-e5l': 'e5-large &middot; 560M &middot; 512 tokens', 'r2-e5b': 'e5-base &middot; 278M &middot; 512 tokens',
+           'r2-e5b-plain': 'e5-base, no mined negatives or book blocks', 'r2-e5s': 'e5-small &middot; 118M &middot; 512 tokens',
+           'r2-bgem3': 'bge-m3 &middot; 568M &middot; 512 tokens', 'ft-e5-base': 'round-1 model &middot; e5-base &middot; 256 tokens',
+           'ft-e5-small': 'round-1 &middot; e5-small &middot; 256 tokens'}
+    tag = {SEL['selected']: 'selected', SEL['runner_up']: 'runner-up', 'ft-e5-base': 'incumbent', 'ft-e5-small': 'incumbent'}
+    rows = []
+    for name, val in pool:
+        me = name == SEL['selected']
+        dot = ('<div style="position:absolute;top:50%%;left:0;width:%.3f%%;height:2px;margin-top:-1px;background:%s"></div>'
+               '<div style="position:absolute;top:50%%;left:%.3f%%;width:18px;height:18px;margin:-9px 0 0 -9px;border-radius:50%%;background:%s"></div>'
+               % (X(val), '#9DB9EA' if me else '#D3D9E1', X(val), '#2F6FDB' if me else '#8E9AAB'))
+        rows.append('<div class="row%s" style="grid-template-columns:330px 1fr 230px;height:64px"><div class="lab">%s<small>%s</small></div>'
+                    '<div class="trk" style="height:30px">%s</div><div class="val">%s<small style="font-size:15px;color:#5B6576;margin-left:10px;font-weight:400">%s</small></div></div>'
+                    % (' me' if me else '', name, sub[name], dot, f4(val), tag.get(name, '')))
+    left = 'calc(330px + 22px + (100%% - 330px - 230px - 44px)*%.5f)'
+    band = ('<div style="position:absolute;top:0;bottom:0;left:%s;width:calc((100%% - 330px - 230px - 44px)*%.5f);background:#EAF1FC;border-left:1px solid #9DB9EA;border-right:1px solid #9DB9EA"></div>'
+            % (left % (X(best - SEL['tie']) / 100), (X(best) - X(best - SEL['tie'])) / 100))
+    tf = v(TFIDF, 'dev')
+    ref = ('<div style="position:absolute;top:0;bottom:0;left:%s;border-left:2px dashed #8E9AAB"></div>' % (left % (X(tf) / 100)))
+    ticks = ''.join('<span style="position:absolute;left:%s;transform:translateX(-50%%);font-size:15px;color:#5B6576">%.2f</span>' % (left % (X(t) / 100), t)
+                    for t in (0.66, 0.67, 0.68, 0.69, 0.70, 0.71, 0.72))
+    body = ('<div style="position:relative;margin-top:26px">%s%s%s</div><div style="position:relative;height:22px;margin-top:4px">%s</div>'
+            '<p class="note" style="margin-top:14px">Shaded: the pre-registered tie band (within %.3f of the best). Only the best model falls in it, so it was selected; '
+            'the runner-up is %.4f below. Dashed line: TF-IDF on dev, %s. Axis starts at 0.66.</p>'
+            % (band, ref, ''.join(rows), ticks, SEL['tie'], best - SEL['dev_scores'][SEL['runner_up']], f4(tf)))
+    return page('Selected on dev by a fixed rule',
+                'Overall dev score of every model in the selection pool. Rule: highest dev score; within %.3f of it, the smaller model wins.' % SEL['tie'],
+                body, foot('dev', ' &middot; round-2 models at 512 tokens in float16; incumbents at their round-1 setting'))
+
+
+# ------------------------------------------------------------------------------------------------ 04 round 1 vs round 2
+def c04():
+    cols = ['overall'] + TASKS
+    a = [v(R1, 'test', c) for c in cols]
+    b = [v(OURS, 'test', c) for c in cols]
+    series = [('Round-1 model (e5-base fine-tune, 256 tokens)', '#9DB9EA', a, False),
+              ('Hindko Embeddings, round 2 (e5-large fine-tune, 512 tokens)', '#2F6FDB', b, True)]
+
+    def below(gi, gx, gw, y1):
+        d = float(f3(b[gi])) - float(f3(a[gi]))     # from the rounded values printed on the bars
+        return ('<text x="%.1f" y="%d" font-size="17" font-weight="700" fill="%s" text-anchor="middle">%s</text>'
+                % (gx + gw / 2, y1 + 76, '#1D4FA8' if d >= 0 else '#B4442C', ('%+.3f' % d).replace('-', '&minus;')))
+    body = legend([(n, c) for n, c, _, _ in series]) + grouped([TASK_LABEL[c] for c in cols], series, height=500, below=below)
+    return page('Round 2 vs round 1',
+                'Test scores, with the change below each group. Round 2 changed the base (e5-base to e5-large), the data (new short-query pairs, mined hard negatives, same-book batches) and the length (256 to 512 tokens).',
+                body, foot('test'))
+
+
+# ------------------------------------------------------------------------------------------------ 05 size vs score
+def c05():
+    width, height = 1440, 520
+    x0, x1, y0, y1 = 80, 1280, 20, height - 60
+    xmax, ylo, yhi = 650, 0.40, 0.75
+    X = lambda p: x0 + (x1 - x0) * p / xmax
+    Y = lambda s: y1 - (y1 - y0) * (s - ylo) / (yhi - ylo)
+    svg = ['<svg viewBox="0 0 %d %d" width="%d" height="%d" style="display:block;margin-top:26px;overflow:visible">' % (width, height, width, height)]
+    for t in (0.40, 0.45, 0.50, 0.55, 0.60, 0.65, 0.70, 0.75):
+        svg.append('<line x1="%d" x2="%d" y1="%.1f" y2="%.1f" stroke="#E4E8EE"/><text x="%d" y="%.1f" font-size="15" fill="#5B6576" text-anchor="end">%.2f</text>' % (x0, x1, Y(t), Y(t), x0 - 10, Y(t) + 5, t))
+    for p in (0, 100, 200, 300, 400, 500, 600):
+        svg.append('<text x="%.1f" y="%d" font-size="15" fill="#5B6576" text-anchor="middle">%d</text>' % (X(p), y1 + 24, p))
+    svg.append('<text x="%.1f" y="%d" font-size="16" fill="#5B6576" text-anchor="middle">parameters (millions)</text>' % ((x0 + x1) / 2, y1 + 50))
+    svg.append('<text x="18" y="%d" font-size="16" fill="#5B6576" transform="rotate(-90 18 %d)" text-anchor="middle">dev overall score</text>' % ((y0 + y1) // 2, (y0 + y1) // 2))
+    tf = v(TFIDF, 'dev')
+    svg.append('<line x1="%d" x2="%d" y1="%.1f" y2="%.1f" stroke="#8E9AAB" stroke-width="2" stroke-dasharray="7 6"/>' % (x0, x1, Y(tf), Y(tf)))
+    svg.append('<text x="%d" y="%.1f" font-size="16" fill="#5B6576" text-anchor="start">TF-IDF %s</text>' % (x0 + 8, Y(tf) - 8, f4(tf)))
+    e5 = ['intfloat/multilingual-e5-small', 'intfloat/multilingual-e5-base', 'intfloat/multilingual-e5-large']
+    ft = {'intfloat/multilingual-e5-small': 'r2-e5s', 'intfloat/multilingual-e5-base': 'r2-e5b', 'intfloat/multilingual-e5-large': OURS, BGE: 'r2-bgem3'}
+    for ids, col, w in (([ft[m] for m in e5], '#9DB9EA', 3), (e5, '#C9D0DA', 3)):
+        pts = ' '.join('%.1f,%.1f' % (X(PARAMS[m if m in PARAMS else [k for k, x in ft.items() if x == m][0]]), Y(v(m, 'dev'))) for m in ids)
+        svg.append('<polyline points="%s" fill="none" stroke="%s" stroke-width="%d"/>' % (pts, col, w))
+    for base, run in ft.items():          # fine-tuning gain per base: dotted vertical
+        svg.append('<line x1="%.1f" x2="%.1f" y1="%.1f" y2="%.1f" stroke="#C9D0DA" stroke-width="1.5" stroke-dasharray="3 4"/>'
+                   % (X(PARAMS[base]), X(PARAMS[base]), Y(v(base, 'dev')), Y(v(run, 'dev'))))
+    # (id, label, dx, dy, anchor, kind)
+    pts = [(OURS, 'e5-large fine-tune (this model)', -14, -14, 'end', 'me'), ('r2-bgem3', 'bge-m3 fine-tune', 14, 26, 'start', 'ft'),
+           ('r2-e5b', 'e5-base fine-tune', 0, -16, 'middle', 'ft'), ('r2-e5s', 'e5-small fine-tune', 0, -16, 'middle', 'ft'),
+           (E5L, 'e5-large', -14, -12, 'end', 'pub'), (BGE, 'bge-m3', 14, -10, 'start', 'pub'),
+           ('intfloat/multilingual-e5-base', 'e5-base', 14, -10, 'start', 'pub'), ('intfloat/multilingual-e5-small', 'e5-small', 0, 28, 'middle', 'pub'),
+           ('sentence-transformers/LaBSE', 'LaBSE', -14, 6, 'end', 'pub'), ('Qwen/Qwen3-Embedding-0.6B', 'Qwen3-Embedding-0.6B', 14, 6, 'start', 'pub'),
+           ('sentence-transformers/paraphrase-multilingual-mpnet-base-v2', 'mpnet-base-v2', 14, 6, 'start', 'pub')]
+    inv = {r: b for b, r in ft.items()}
+    for mid, lab, dx, dy, anc, kind in pts:
+        p = PARAMS[mid] if mid in PARAMS else PARAMS[inv[mid]]
+        s = v(mid, 'dev')
+        col = {'me': '#2F6FDB', 'ft': '#6F95DE', 'pub': '#8E9AAB'}[kind]
+        r = 11 if kind == 'me' else 8
+        svg.append('<circle cx="%.1f" cy="%.1f" r="%d" fill="%s" stroke="#FFFFFF" stroke-width="2"/>' % (X(p), Y(s), r, col))
+        svg.append('<text x="%.1f" y="%.1f" font-size="%d" font-weight="%d" fill="%s" text-anchor="%s" stroke="#FFFFFF" stroke-width="5" paint-order="stroke">%s '
+                   '<tspan font-weight="400" fill="#5B6576">%s</tspan></text>'
+                   % (X(p) + dx, Y(s) + dy, 18 if kind == 'me' else 16, 700 if kind == 'me' else 600, '#1D4FA8' if kind == 'me' else '#0F172A', anc, lab, f4(s)))
+    svg.append('</svg>')
+    body = (legend([('fine-tuned on Hindko (round 2)', '#6F95DE'), ('public model, untuned', '#8E9AAB'), ('TF-IDF reference', '#8E9AAB')]).replace(
+        '<i style="background:#8E9AAB"></i>TF-IDF', '<i style="background:none;border-top:2px dashed #8E9AAB;height:0;width:28px"></i>TF-IDF') + ''.join(svg))
+    return page('Fine-tuning matters more than size',
+                'Dev score against model size. Every fine-tuned model scores above every untuned one; among the fine-tuned e5 models the larger ones score slightly higher.',
+                body, foot('dev', ' &middot; neural models at max_seq_length 512 in float16; TF-IDF is lexical'))
+
+
+# ------------------------------------------------------------------------------------------------ 06 book-title detail
+def c06():
+    mets = [('BT_R1', 'Recall@1'), ('BT_R10', 'Recall@10'), ('BT_MRR', 'MRR@10'), ('BookTitleRetrieval', 'nDCG@10')]
+    panels = []
+    for split, title in (('dev', 'Dev: 262 queries'), ('test', 'Test: 28 queries from 5 books')):
+        series = [('this model', '#2F6FDB', [v(OURS, split, c) for c, _ in mets], True),
+                  ('TF-IDF', '#8E9AAB', [v(TFIDF, split, c) for c, _ in mets], False),
+                  ('bge-m3', '#C9D0DA', [v(BGE, split, c) for c, _ in mets], False)]
+        svg = grouped([(lab, '') for _, lab in mets], series, vmax=0.8, width=700, height=430, ticks=(0.2, 0.4, 0.6, 0.8), x0=50,
+                      bar_w=44, gap=6, lab_size=15)
+        panels.append('<div><div style="font-size:22px;font-weight:600;margin-top:6px">%s</div>%s</div>' % (title, svg))
+    body = (legend([('Hindko Embeddings (this model)', '#2F6FDB'), ('TF-IDF char 2-4-gram', '#8E9AAB'), ('bge-m3 (untuned, 512 tokens)', '#C9D0DA')]) +
+            '<div style="display:grid;grid-template-columns:1fr 1fr;gap:40px">%s</div>' % ''.join(panels) +
+            '<p class="note" style="margin-top:0">The test set has only 28 queries (57 relevant passages), so its numbers are noisy; the dev gap to TF-IDF points the same way.</p>')
+    return page('Weak spot: finding a passage from its book title',
+                'BookTitleRetrieval: a table-of-contents title (often the first line of a poem) must retrieve its passage. TF-IDF beats this model on nDCG@10 on both dev and test.',
+                body, foot('test', '; dev panel on the dev split &middot; neural models at max_seq_length 512'))
+
+
+CHARTS = [('00_scorecard', c00), ('01_test_overall_all_models', c01), ('02_test_per_task', c02), ('03_dev_candidates', c03),
+          ('04_round1_vs_round2', c04), ('05_size_vs_score', c05), ('06_booktitle_detail', c06)]
+
+
+def embed_fonts(html):
+    for m in re.finditer(r'<link[^>]+href="(https://fonts.googleapis.com/css2?[^"]+)"[^>]*>', html):
+        url = m.group(1).replace('&amp;', '&')
+        css = urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=30).read().decode()
+        for u in sorted(set(re.findall(r'url\((https://fonts.gstatic.com/[^)]+)\)', css))):
+            data = urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=60).read()
+            css = css.replace(u, 'data:font/woff2;base64,' + base64.b64encode(data).decode())
+        html = html.replace(m.group(0), '<style>' + css + '</style>')
+    return html
+
+
+def main():
+    args = [a for a in sys.argv[1:] if not a.startswith('--')]
+    keep = '--keep-html' in sys.argv
+    only = args[0].split(',') if args else None
+    os.makedirs(OUT, exist_ok=True)
+    os.makedirs(WORK, exist_ok=True)
+    env = dict(os.environ, TEMP=WORK, TMP=WORK)
+    try:
+        for name, fn in CHARTS:
+            if only and name[:2] not in only:
+                continue
+            html = os.path.join(WORK, name + '.html')
+            open(html, 'w', encoding='utf-8', newline='\n').write(embed_fonts(fn()))
+            png = os.path.join(OUT, name + '.png')
+            subprocess.run([CHROME, '--headless=new', '--disable-gpu', '--hide-scrollbars', '--force-device-scale-factor=2',
+                            '--user-data-dir=' + os.path.join(WORK, 'chrome'), '--no-first-run', '--disable-crash-reporter',
+                            '--window-size=%d,%d' % (W, H), '--virtual-time-budget=20000', '--screenshot=' + png,
+                            'file:///' + html.replace('\\', '/')], check=True, capture_output=True, timeout=300, env=env)
+            print(name)
+    finally:
+        if not keep:
+            shutil.rmtree(WORK, ignore_errors=True)
+
+
+if __name__ == '__main__':
+    main()

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f14d62eff2fd75d55322ecce26b5b38818f9fd66a155370551fdaad93d86278b
+size 1119825632

modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.base.modules.transformer.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.sentence_transformer.modules.pooling.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.base.modules.normalize.Normalize"
+  }
+]

sentence_bert_config.json

@@ -0,0 +1,10 @@
+{
+    "transformer_task": "feature-extraction",
+    "modality_config": {
+        "text": {
+            "method": "forward",
+            "method_output_name": "last_hidden_state"
+        }
+    },
+    "module_output_name": "token_embeddings"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:883b037111086fd4dfebbbc9b7cee11e1517b5e0c0514879478661440f137085
+size 17082987

tokenizer_config.json

@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": true,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "is_local": true,
+  "mask_token": "<mask>",
+  "max_length": 512,
+  "model_max_length": 512,
+  "pad_to_multiple_of": null,
+  "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "</s>",
+  "stride": 0,
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "<unk>"
+}