Private review: measured five-language release, Hindi sentence mode and custom GGUFs

.gitattributes

@@ -1,35 +1,6 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
+*.q4 filter=lfs diff=lfs merge=lfs -text
+*.q8 filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.gguf filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+assets/banner.png filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,158 @@
+Creative Commons Attribution-NonCommercial 4.0 International
+
+ Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses.
+
+Considerations for licensors: Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. More considerations for licensors.
+
+Considerations for the public: By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. More considerations for the public.
+
+Creative Commons Attribution-NonCommercial 4.0 International Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-NonCommercial 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions.
+
+Section 1 – Definitions.
+
+     a.	Adapted Material means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image.
+
+     b.	Adapter's License means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License.
+
+     c.	Copyright and Similar Rights means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights.
+
+     d.	Effective Technological Measures means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements.
+
+     e.	Exceptions and Limitations means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material.
+
+     f.	Licensed Material means the artistic or literary work, database, or other material to which the Licensor applied this Public License.
+
+     g.	Licensed Rights means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license.
+
+     h.	Licensor means the individual(s) or entity(ies) granting rights under this Public License.
+
+     i.	NonCommercial means not primarily intended for or directed towards commercial advantage or monetary compensation. For purposes of this Public License, the exchange of the Licensed Material for other material subject to Copyright and Similar Rights by digital file-sharing or similar means is NonCommercial provided there is no payment of monetary compensation in connection with the exchange.
+
+     j.	Share means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them.
+
+     k.	Sui Generis Database Rights means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world.
+
+     l.	You means the individual or entity exercising the Licensed Rights under this Public License. Your has a corresponding meaning.
+
+Section 2 – Scope.
+
+     a.	License grant.
+
+          1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to:
+
+               A. reproduce and Share the Licensed Material, in whole or in part, for NonCommercial purposes only; and
+
+               B. produce, reproduce, and Share Adapted Material for NonCommercial purposes only.
+
+          2. Exceptions and Limitations. For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions.
+
+          3. Term. The term of this Public License is specified in Section 6(a).
+
+          4. Media and formats; technical modifications allowed. The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material.
+
+          5. Downstream recipients.
+
+               A. Offer from the Licensor – Licensed Material. Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License.
+
+               B. No downstream restrictions. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material.
+
+          6. No endorsement. Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i).
+
+     b.	Other rights.
+
+          1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise.
+
+          2. Patent and trademark rights are not licensed under this Public License.
+
+          3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties, including when the Licensed Material is used other than for NonCommercial purposes.
+
+Section 3 – License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the following conditions.
+
+     a.	Attribution.
+
+          1. If You Share the Licensed Material (including in modified form), You must:
+
+               A. retain the following if it is supplied by the Licensor with the Licensed Material:
+
+                    i.	identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated);
+
+                    ii.	a copyright notice;
+
+                    iii. a notice that refers to this Public License;
+
+                    iv.	a notice that refers to the disclaimer of warranties;
+
+                    v.	a URI or hyperlink to the Licensed Material to the extent reasonably practicable;
+
+               B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and
+
+               C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License.
+
+          2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information.
+
+          3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable.
+
+          4. If You Share Adapted Material You produce, the Adapter's License You apply must not prevent recipients of the Adapted Material from complying with this Public License.
+
+Section 4 – Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material:
+
+     a.	for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database for NonCommercial purposes only;
+
+     b.	if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material; and
+
+     c.	You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database.
+For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights.
+
+Section 5 – Disclaimer of Warranties and Limitation of Liability.
+
+     a.	Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.
+
+     b.	To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.
+
+     c.	The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability.
+
+Section 6 – Term and Termination.
+
+     a.	This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically.
+
+     b.	Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates:
+
+          1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or
+
+          2. upon express reinstatement by the Licensor.
+
+     For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License.
+
+     c.	For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License.
+
+     d.	Sections 1, 5, 6, 7, and 8 survive termination of this Public License.
+
+Section 7 – Other Terms and Conditions.
+
+     a.	The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed.
+
+     b.	Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License.
+
+Section 8 – Interpretation.
+
+     a.	For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License.
+
+     b.	To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions.
+
+     c.	No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor.
+
+     d.	Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority.
+
+Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at creativecommons.org/policies, Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses.
+
+Creative Commons may be contacted at creativecommons.org.

LICENSE-APACHE-2.0

@@ -0,0 +1,209 @@
+AksharaLP by Sourojit Dhua — https://huggingface.co/sourojitdotcom/AksharaLP-1M-v2
+Copyright 2026 Sourojit Dhua
+
+Licensed under the Apache License, Version 2.0 (the full text follows). Commercial use is permitted.
+Redistributions and derivative works must include the attribution notices in the accompanying
+NOTICE file, as required by Section 4(d) of the License.
+
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,75 @@
+AksharaLP-MultiIndic-8M — Sourojit Dhua, 2026
+https://huggingface.co/sourojitdotcom/AksharaLP-MultiIndic-8M
+
+Combined model-card licence: CC-BY-NC-4.0. NON-COMMERCIAL RELEASE.
+The new Bengali/Telugu/Kannada/Tamil weights include Samanantar-derived Aksharantar rows. The upstream Samanantar card specifies CC-BY-NC-4.0; other publisher pages describe CC0 packaging. This release does not certify those weights as commercially clean. An Apache-2.0 clean-data retrain is planned. Preserve source attributions and applicable upstream terms. Selecting CC-BY-NC does not resolve all derivative-rights questions.
+
+Original Hindi word/sentence components and runtime code retain Apache-2.0 and their existing notices. They are independently available at:
+https://huggingface.co/sourojitdotcom/AksharaLP-1M-v2
+https://huggingface.co/sourojitdotcom/AksharaLP-Sentence-2.8M
+
+Attributions:
+Aksharantar — AI4Bharat, Madhani et al. (2022), CC0 mined/existing packaging and CC-BY manual data.
+Samanantar — AI4Bharat, Ramesh et al. (2022), upstream card CC-BY-NC-4.0.
+Dakshina — Google Research, Roark et al., Processing South Asian Languages Written in the Latin Script, LREC 2020, https://github.com/google-research-datasets/dakshina ; CC-BY-SA-4.0 https://creativecommons.org/licenses/by-sa/4.0/ . Source-attributed training pairs appear inside Aksharantar. Retain the upstream terms.
+IndicCorp v2 — AI4Bharat, Doddapaneni et al. (ACL 2023), CC0. Dictionaries and Hindi LM training.
+IndicXlit — AI4Bharat, MIT. Distillation teacher and comparison model, not redistributed.
+Inter and Noto Sans Indic fonts in the Space — SIL Open Font License 1.1 (included beside each font).
+
+For the unchanged Hindi components, retain the following original notice as well:
+
+AksharaLP-Sentence-2.8M by Sourojit Dhua
+https://huggingface.co/sourojitdotcom/AksharaLP-Sentence-2.8M
+Copyright 2026 Sourojit Dhua
+
+AksharaLP-Sentence-2.8M is licensed under the Apache License, Version 2.0 (see LICENSE).
+Commercial use is permitted.
+
+What is in here
+  * The v2 word model: a 989K-parameter character Transformer, its reranker, the dictionary pack
+    (lexicon + word-bigram language model) and the JavaScript/Python runtime, unchanged from
+    AksharaLP-1M-v2 (https://huggingface.co/sourojitdotcom/AksharaLP-1M-v2).
+  * The sentence add-on: nlm.q8 (a 2,770,752-parameter Hindi neural language model trained only on
+    IndicCorp v2 Hindi, CC0, with our own tokenizer, training code and runtime), plus nlm.mjs, v3.mjs
+    and sent.mjs. This add-on and its training data are commercially clean; the full system still
+    inherits the v2 word package's terms, which are Apache-2.0 as set out here.
+
+ATTRIBUTION
+Under Section 4(d) of the Apache License, any redistribution of this work or of a Derivative Work
+(for example: shipping these files in an app, a web page, a browser extension or an SDK, or
+releasing a fine-tuned, quantized or converted model based on it) must include a readable copy of
+this NOTICE file. Please use the following attribution text:
+
+    Transliteration by AksharaLP (Sourojit Dhua) — https://huggingface.co/sourojitdotcom/AksharaLP-Sentence-2.8M
+
+Recommended in-app credit (e.g. in an About, Settings or Licenses screen):
+
+    Hindi typing powered by AksharaLP by Sourojit Dhua
+
+The model files in this repository were built with the help of the following
+third-party resources. No third-party data is redistributed in this repository
+other than as learned model parameters and derived word lists.
+
+------------------------------------------------------------------------------
+IndicCorp v2 — AI4Bharat
+  https://github.com/AI4Bharat/IndicBERT/tree/main#indiccorp-v2
+  License: CC0
+  Used for: training the sentence language model (nlm.q8); the word-model lexicon and
+  word-bigram language model.
+
+Aksharantar — AI4Bharat
+  https://huggingface.co/datasets/ai4bharat/Aksharantar
+  License: CC0 (mined data and existing sources) / CC-BY (manually collected data)
+  Used for: romanized <-> Devanagari word pairs for training the word model.
+
+IndicXlit — AI4Bharat
+  https://github.com/AI4Bharat/IndicXlit
+  License: MIT
+  Used for: teacher model for word-model distillation.
+
+CMUdict — Carnegie Mellon University
+  https://github.com/cmusphinx/cmudict
+  License: BSD-2-Clause-style
+  Used for: English word cues in the word model.
+
+Fonts served by the demo Space (Inter, Noto Sans Devanagari) are under the SIL Open Font License 1.1.

README.md

@@ -0,0 +1,305 @@
+---
+license: cc-by-nc-4.0
+language:
+- hi
+- bn
+- te
+- kn
+- ta
+tags:
+- transliteration
+- indic
+- multilingual
+- tiny
+- browser
+- javascript
+- offline
+- input-method
+- gguf
+- custom-architecture
+datasets:
+- ai4bharat/Aksharantar
+- ai4bharat/IndicCorpV2
+metrics:
+- accuracy
+---
+
+<p align="center"><img src="assets/banner.svg" alt="AksharaLP MultiIndic — five scripts, one small footprint" width="100%"/></p>
+<p align="center"><a href="https://huggingface.co/spaces/sourojitdotcom/AksharaLP-MultiIndic-demo">Try the private browser demo ↗</a> · <a href="#benchmarks">Measured benchmarks</a> · <a href="#gguf-containers">13 GGUF files</a> · <a href="#licence--data">Licence & data</a></p>
+
+# AksharaLP-MultiIndic-8M
+
+### Five word transliterators. One optional Hindi sentence model. Your device.
+
+**Hindi · Bengali · Telugu · Kannada · Tamil** — familiar Roman letters in, native script out. Five independent ~989K-parameter character Transformers, a lazily loaded dictionary per language, and the already-shipped **2,770,752-parameter Hindi sentence LM**. No inference server and no GPU required for inference. This is **transliteration, not translation or a general-purpose language model**.
+
+> **Private review · non-commercial release.** This bundle's model-card licence is **CC-BY-NC-4.0**, not Apache-2.0. The new Bengali/Telugu/Kannada/Tamil weights include Samanantar-derived Aksharantar rows with non-commercial upstream terms. A commercially clean Apache-2.0 retrain is planned. The Hindi word and sentence components remain independently available under Apache-2.0 in their original repositories. See [Licence & data](#licence--data).
+
+<table><tr><td align="center"><h2>4.94M</h2>five word models combined</td><td align="center"><h2>7.72M</h2>with the optional Hindi LM</td><td align="center"><h2>20.55 MB</h2>all word payloads, gzip-9</td><td align="center"><h2>23.48 MB</h2>including Hindi sentence mode</td></tr></table>
+
+The **8M name rounds 7,715,520 neural parameters**, not a billion-parameter model. Dictionaries and small candidate-reranker coefficients are separate from that count. Loading one language does not load the other four. Browser inference uses float32 arithmetic after loading low-bit weights: small files do **not** mean equally small RAM use.
+
+## What this release does — and does not prove
+
+- Every word Transformer is **988,928 parameters**, except Bengali (**989,056**, with one additional output character). The five sum to **4,944,768**; optional Hindi LM adds **2,770,752**.
+- New word weights are about **0.54 MB gzip-9 per language**; model-plus-dictionary payloads are about **3.05–3.23 MB** each. Hindi keeps its larger, established 3M-word dictionary pack.
+- IndicXlit has **11,487,744 neural parameters** in the comparison port. Its per-language original dictionaries are much larger; the exact byte comparison below uses their released JSON formats, not an optimized repacking.
+- **We trail IndicXlit's full package in every new language on top-1, top-3 and top-8.** This release is a storage/accuracy trade-off, not a state-of-the-art or “best in class” claim.
+- Sentence-level neural rescoring is **Hindi-only**. Bengali, Telugu, Kannada and Tamil convert word by word. The separate Bengali real-chat research front-end is **not** part of this bundle.
+
+## Benchmarks
+
+<img src="assets/accuracy.png" alt="Per-language top-1 test comparison, including confidence intervals" width="100%"/>
+
+**Aksharantar test, exact match, any accepted reference.** Top-1 %, 95% bootstrap intervals in brackets; paired differences are percentage points, ours minus IndicXlit.
+
+| Language | Roman words | Ours + dictionary | IndicXlit full package | Paired gap [95% CI] |
+|---|---:|---:|---:|---:|
+| Hindi | 9,991 | 70.04 [69.16, 70.90] | 69.33 [68.44, 70.23] | +0.71 [-0.04, +1.40] |
+| Bengali | 13,367 | 64.64 [63.83, 65.44] | 70.52 [69.76, 71.34] | -5.88 [-6.52, -5.22] |
+| Telugu | 9,936 | 78.74 [77.93, 79.53] | 82.42 [81.65, 83.14] | -3.67 [-4.27, -3.07] |
+| Kannada | 11,165 | 77.37 [76.61, 78.13] | 82.61 [81.92, 83.29] | -5.24 [-5.83, -4.67] |
+| Tamil | 11,405 | 71.78 [71.01, 72.61] | 75.95 [75.20, 76.73] | -4.16 [-4.75, -3.59] |
+
+| Language | Ours top-1 / top-3 / top-8 % | IndicXlit full top-1 / top-3 / top-8 % |
+|---|---:|---:|
+| Hindi | 70.04 / 88.31 / 93.63 | 69.33 / 85.40 / 88.29 |
+| Bengali | 64.64 / 75.66 / 77.97 | 70.52 / 82.25 / 85.44 |
+| Telugu | 78.74 / 86.87 / 88.63 | 82.42 / 90.63 / 92.75 |
+| Kannada | 77.37 / 85.80 / 87.68 | 82.61 / 90.68 / 92.41 |
+| Tamil | 71.78 / 80.05 / 82.61 | 75.95 / 84.21 / 87.26 |
+
+**Protocol and caveats**
+
+1. For Bengali/Telugu/Kannada/Tamil, the shipped q4 file, top-500k IndicCorp frequency list and eight-feature linear reranker were frozen before test. The reranker was fitted on **all VALID**, not on test, using the same fixed feature design and L2 regularization selected during validation work. Quantized VALID candidates were regenerated before that fit. There was **one inference pass on the full test per frozen artifact**, with cached predictions scored once; no post-test tuning.
+2. NFC exact match, ZWJ/ZWNJ removed, any reference accepted. Top-k means at least one reference among the first k candidates, not “k times more accurate.” Intervals use **2,000 bootstrap resamples, seed 0**; paired intervals resample the same words together. Full top-k intervals, hashes and frozen configurations are in [evaluation/](evaluation/).
+3. IndicXlit scores reuse existing **beam-8 PyTorch-port predictions with the full official frequency dictionary formula**, rescored on exactly the same rows. This is not a fresh run of the official fairseq package. The port masks output to each language's script block. Bengali has one empty Roman input skipped by that runner; its missing prediction is counted incorrect and the row is not dropped.
+4. Hindi numbers are copied from the measured [AksharaLP-1M-v2 release](https://huggingface.co/sourojitdotcom/AksharaLP-1M-v2), **not re-tested**. Its candidate generation/reranker and 3M-word dictionary differ from the four new models. The old Hindi full-package comparison interval includes zero; it does not establish a significant top-1 win.
+5. Training reports record **zero valid/test native-word and Roman-form overlap** for all four new training files. The general IndicCorp frequency dictionaries are not test-filtered and can naturally contain test reference words. Distillation teachers may have their own historical overlap. This is not an unseen-vocabulary-only benchmark, and Aksharantar words are not a WhatsApp sentence benchmark.
+
+## Hindi sentence mode — the shipped add-on, unchanged
+
+The [AksharaLP-Sentence-2.8M](https://huggingface.co/sourojitdotcom/AksharaLP-Sentence-2.8M) LM chooses jointly among candidates already proposed for each typed Hindi word. It cannot translate, invent a missing candidate, reorder words, or answer questions. The runtime exposes the existing candidate-membership guard.
+
+| Dakshina Hindi sentences, 4,976 test items | Existing Hindi v2 word package + bigram context | With the neural sentence LM |
+|---|---:|---:|
+| Word accuracy % | 89.02 | 89.44 |
+| Entire sentences exactly right % | 25.70 | 26.73 |
+| Character error rate % | 4.07 | 3.93 |
+| Word error rate % | 11.20 | 10.78 |
+
+These are the **existing one-time published results, not new evaluations**. The 89.02/25.70 baseline already uses the older Hindi bigram context decoder; it is not the context-off demo toggle. Paired reported gain: word accuracy **+0.42 [0.33, 0.51]** points, sentence exact match **+1.02 [0.40, 1.65]** points (the difference of rounded sentence percentages is 1.03). This release retains the default **q8** LM and original settings. No sentence claims are made for the other four languages.
+
+## Footprint & latency
+
+<img src="assets/sizes.png" alt="Canonical uncompressed per-language payload comparison on a logarithmic scale" width="100%"/>
+
+| Language payload | Canonical raw bytes | Sum gzip-9 bytes | JS CPU p50 / p95 ms per word |
+|---|---:|---:|---:|
+| Hindi | 16,221,771 | 7,905,041 (7.91 MB) | 207.02 / 319.99 |
+| Bengali | 14,658,574 | 3,049,313 (3.05 MB) | 51.61 / 74.00 |
+| Telugu | 16,579,274 | 3,222,365 (3.22 MB) | 51.77 / 87.87 |
+| Kannada | 17,376,245 | 3,224,222 (3.22 MB) | 55.35 / 100.77 |
+| Tamil | 18,695,389 | 3,130,045 (3.13 MB) | 60.98 / 103.39 |
+| Shared word JS runtime + root configuration | 68,781 | 23,186 | — |
+| **All five word packages + shared runtime** | **83,600,034** | **20,554,172 (20.55 MB)** | — |
+| Optional Hindi q8 sentence LM + JS add-on | 3,256,675 | 2,928,839 (2.93 MB) | Not re-timed |
+| **All languages + Hindi sentence add-on** | **86,856,709** | **23,483,011 (23.48 MB)** | — |
+
+MB = 1,000,000 bytes. Canonical raw dictionary size means decompressed frequency TSV for new languages, and native lexicon/bigram binaries for Hindi. Browser downloads use gzip companions where provided. gzip-9 values are sums of per-file compression, not actual RAM, not total repository storage, and not a promise about HTTP Content-Encoding. Charts/docs/fonts/Python/GGUF alternatives and duplicated compressed companions are excluded. All exact component sizes are in [sizes.json](evaluation/sizes.json).
+
+Latency: **100 evenly spaced VALID words per language**, five warmups, cleared candidate memo per word, single JavaScript thread in **Node v24.22.1 on Intel(R) Xeon(R) Processor, Linux**; initialization excluded. This is **Node CPU latency, not a browser or phone speed claim**. Loading the large Hindi dictionary is substantially more expensive. Browser demo inference happens in a Web Worker and reports live timings. The 15 initial example word candidate lists matched Python exactly; a broader independent GGUF validation sample is below.
+
+IndicXlit size comparison: **45,950,976 bytes of FP32 neural parameters**, plus original dictionary JSON raw bytes: Hindi 358,118,856; Bengali 403,487,845; Telugu 761,940,466; Kannada 1,045,504,407; Tamil 1,044,620,827. These compare different precisions, vocabularies and data encodings; IndicXlit was not requantized or repacked. They do not prove that ours is the smallest possible system.
+
+## Real outputs
+
+Generated by the bundled JavaScript runtime, **not hand-written target strings**. These are fixed illustrative examples, not an accuracy suite or proof of sentence understanding. Multiword inputs outside Hindi are independent word conversions. Some outputs are wrong: in particular, Tamil `naan` currently produces **ணன்**, not the intended நான். We show it rather than hiding the failure.
+
+### Hindi
+
+| Roman input | Actual browser-runtime output |
+|---|---|
+| `namaste` | नमस्ते |
+| `kaise` | कैसे |
+| `dhanyavaad` | धन्यवाद |
+| `mera naam rahul hai` | मेरा नाम राहुल है |
+
+### Bengali
+
+| Roman input | Actual browser-runtime output |
+|---|---|
+| `nomoskar` | নমস্কার |
+| `ami` | আমি |
+| `bhalo` | ভালো |
+| `tumi kemon acho` | তুমি কেমন আছো |
+
+### Telugu
+
+| Roman input | Actual browser-runtime output |
+|---|---|
+| `namaskaram` | నమస్కారం |
+| `nenu` | నేను |
+| `manchi` | మంచి |
+| `meeru ela unnaru` | మీరు ఎలా ఉన్నారు |
+
+### Kannada
+
+| Roman input | Actual browser-runtime output |
+|---|---|
+| `namaskara` | ನಮಸ್ಕಾರ |
+| `nanu` | ನನು |
+| `chennagide` | ಚೆನ್ನಾಗಿದೆ |
+| `neevu hegiddira` | ನೀವು ಹೇಗಿದ್ದೀರಾ |
+
+### Tamil
+
+| Roman input | Actual browser-runtime output |
+|---|---|
+| `vanakkam` | வணக்கம் |
+| `naan` | ணன் |
+| `nandri` | நன்றி |
+| `neenga eppadi irukeenga` | நீங்க எப்படி இருகீங்க |
+
+### Hindi sentence examples
+
+| Input | Actual output with the optional sentence LM |
+|---|---|
+| `main theek hoon` | मैं ठीक हूं |
+| `ghar main kaun hai` | घर में कौन है |
+
+## JavaScript — browser, worker, or Node
+
+Copy `multiindic.mjs`, `aksharalp.mjs`, the `models/` tree and (for Hindi sentence mode) `nlm.mjs`, `v3.mjs`, `sent.mjs`. Serve over HTTPS or localhost; ES modules do not reliably work from `file://`.
+
+```js
+import { MultiIndic } from './multiindic.mjs';
+
+// Loads Bengali's weights + dictionary, not every language.
+const engine = await MultiIndic.load('./', { language: 'bn' });
+console.log(engine.word('nomoskar', 8));       // ranked {text, score} objects
+console.log(engine.transliterate('tumi kemon acho').output);
+
+await engine.setLanguage('hi');
+await engine.setSentence(true);               // lazily loads the q8 Hindi LM
+console.log(engine.transliterate('main theek hoon').output);
+await engine.setSentence(false);              // word-by-word, context off
+```
+
+```js
+// Node >= 18: no dependencies, uses local files.
+import {loadFromDir} from './multiindic.mjs';
+const engine = await loadFromDir('./', {language: 'ta'});
+console.log(engine.word('nandri', 3));
+```
+
+Browser gzip decoding requires `DecompressionStream` support; Node uses its built-in zlib. Switching languages retains already loaded models in memory for reuse. Recreate the engine to release that cache. The demo preserves URLs, email addresses, @mentions, #tags, digits and already-native text. Hindi retains the plain reference convention of changing a final full stop to danda. Roman English words are otherwise treated as transliteration input; there is no multilingual code-switch detector.
+
+## Python reference
+
+Requires `numpy` and `torch`. For a private review repository, authenticate with Hugging Face normally; never put a write token in browser code.
+
+```python
+from pathlib import Path
+import sys
+from huggingface_hub import snapshot_download
+
+root = Path(snapshot_download('sourojitdotcom/AksharaLP-MultiIndic-8M', token=True))
+sys.path.insert(0, str(root / 'python'))
+from multiindic import MultiIndic
+
+engine = MultiIndic(root, language='te', threads=1)
+print(engine.word('namaskaram', k=8))
+engine.set_language('hi').set_sentence(True)
+print(engine.transliterate('main theek hoon'))
+```
+
+The Python reference returns candidate strings from `word()`; JavaScript additionally exposes uncalibrated ranking scores. Those scores are **not probabilities**. Hindi uses the plain reference token conversion in both runtimes, not the older demo's extra brand/acronym heuristics. The model/decoder parity samples are measured separately.
+
+## GGUF containers
+
+**These are custom-architecture GGUF v3 tensor containers. They do NOT run in llama.cpp, Ollama, or LM Studio.** Use the bundled [gguf_read.py](gguf_read.py) plus our decoder. A GGUF is not a standalone dictionary or language-model runtime.
+
+The writer/reader follows the [existing v2 export](https://huggingface.co/sourojitdotcom/AksharaLP-1M-v2). Word **q4 GGUF carries the byte-exact shipped mixed q4/q8/f16 blob in an I8 tensor**; it is not standard GGML Q4_0. F16 materializes those already-quantized weights as half floats; it does not restore an original full-precision checkpoint. Exact reranker configuration is retained as JSON metadata, and the external dictionary/runtime remain required.
+
+For Hindi sentence mode, **q8 GGUF preserves the test-measured default byte-exactly**. An existing **q4 LM** and F16 materialization are included as alternatives with validation parity only. **Do not attach the default q8 test accuracies to q4 or F16 alternatives.** The q4 LM is not the demo default.
+
+| GGUF file | Raw bytes | gzip-9 bytes |
+|---|---:|---:|
+| [aksharalp-hi-word-q4.gguf](gguf/aksharalp-hi-word-q4.gguf) | 632,416 | 562,579 |
+| [aksharalp-hi-word-f16.gguf](gguf/aksharalp-hi-word-f16.gguf) | 2,026,048 | 1,348,288 |
+| [aksharalp-bn-word-q4.gguf](gguf/aksharalp-bn-word-q4.gguf) | 591,136 | 541,220 |
+| [aksharalp-bn-word-f16.gguf](gguf/aksharalp-bn-word-f16.gguf) | 1,984,864 | 1,321,177 |
+| [aksharalp-te-word-q4.gguf](gguf/aksharalp-te-word-q4.gguf) | 591,008 | 542,744 |
+| [aksharalp-te-word-f16.gguf](gguf/aksharalp-te-word-f16.gguf) | 1,984,608 | 1,329,379 |
+| [aksharalp-kn-word-q4.gguf](gguf/aksharalp-kn-word-q4.gguf) | 591,008 | 540,642 |
+| [aksharalp-kn-word-f16.gguf](gguf/aksharalp-kn-word-f16.gguf) | 1,984,608 | 1,319,622 |
+| [aksharalp-ta-word-q4.gguf](gguf/aksharalp-ta-word-q4.gguf) | 591,008 | 542,094 |
+| [aksharalp-ta-word-f16.gguf](gguf/aksharalp-ta-word-f16.gguf) | 1,984,608 | 1,328,348 |
+| [aksharalp-hi-sentence-q8.gguf](gguf/aksharalp-hi-sentence-q8.gguf) | 3,488,192 | 2,981,426 |
+| [aksharalp-hi-sentence-q4.gguf](gguf/aksharalp-hi-sentence-q4.gguf) | 2,110,912 | 1,596,808 |
+| [aksharalp-hi-sentence-f16.gguf](gguf/aksharalp-hi-sentence-f16.gguf) | 5,790,336 | 5,159,347 |
+
+### Validation-only parity — not a second test evaluation
+
+First **40 ASCII-alphabetic Roman words per language from full VALID** (200 total). Same dictionary and reranker; no fitting or tuning during parity. “Top-8 order” compares the complete ordered candidate list.
+
+| Language | q4 top-1 identical | q4 top-8 order identical | F16 top-1 identical | F16 top-8 order identical |
+|---|---:|---:|---:|---:|
+| Hindi | 40/40 | 40/40 | 40/40 | 40/40 |
+| Bengali | 40/40 | 40/40 | 40/40 | 40/40 |
+| Telugu | 40/40 | 40/40 | 40/40 | 40/40 |
+| Kannada | 40/40 | 40/40 | 40/40 | 40/40 |
+| Tamil | 40/40 | 40/40 | 40/40 | 40/40 |
+
+Hindi LM: first **12 Dakshina VALID sentences with at most 12 Roman words**, fixed word-model lattices. q8 reproduces **12/12** source decoder outputs; F16 **12/12**; q4 **11/12**. Maximum absolute sequence-score deltas vs q8 are 0.000000, 0.008702, and 6.343046, respectively. This is conversion agreement, **not accuracy**. Across the same validation sample, JavaScript and Python matched **200/200 top-1 and complete top-8 ordered lists**, and **12/12 Hindi sentence outputs**, after using the same plain-reference token policy. Both our structural reader and independent `gguf-py` read all 13 files; complete data is in [gguf_parity.json](evaluation/gguf_parity.json).
+
+```python
+# After the Python setup above:
+engine.set_language('bn')
+engine.load_word_gguf(root / 'gguf/aksharalp-bn-word-q4.gguf')
+print(engine.word('nomoskar'))
+engine.set_language('hi')
+engine.load_sentence_gguf(root / 'gguf/aksharalp-hi-sentence-q8.gguf')
+print(engine.transliterate('main theek hoon'))
+```
+
+## Licence & data
+
+**Model-card licence: CC-BY-NC-4.0. This release is non-commercial.** The Bengali, Telugu, Kannada and Tamil weights contain `aks-Samanantar` Aksharantar training rows. The [current upstream Samanantar card](https://huggingface.co/datasets/ai4bharat/samanantar) states **CC-BY-NC-4.0**. Aksharantar and the publisher's Samanantar packaging page describe CC0 packaging, so provenance terms conflict; this release does **not** certify those weights as commercially clean. A commercially clean **Apache-2.0 retrain is planned**, and would be separately evaluated before replacement.
+
+| Resource | Terms / role |
+|---|---|
+| [Aksharantar, AI4Bharat](https://huggingface.co/datasets/ai4bharat/Aksharantar) | CC0 mined/existing packaging; CC-BY manual data. New training includes Samanantar-derived rows: Bengali **114,558**, Telugu **42,413**, Kannada **36,482**, Tamil **34,447**; per-language training reports are bundled. |
+| [Samanantar, AI4Bharat](https://huggingface.co/datasets/ai4bharat/samanantar) | CC-BY-NC-4.0 upstream card; reason for this release's non-commercial restriction. |
+| [Dakshina, Google Research](https://github.com/google-research-datasets/dakshina) | CC-BY-SA-4.0 upstream. Source-attributed pairs appear within Aksharantar; original terms remain, and derivative-rights questions are not resolved by selecting a more restrictive model-card licence. Cite Roark et al., LREC 2020. |
+| [IndicCorp v2, AI4Bharat](https://huggingface.co/datasets/ai4bharat/IndicCorpV2) | CC0; frequency dictionaries and Hindi LM training. |
+| [IndicXlit, AI4Bharat](https://github.com/AI4Bharat/IndicXlit) | MIT teacher for distillation, and comparison baseline; original checkpoint not redistributed. |
+| Runtime code / original Hindi components | Apache-2.0, retained with their existing notices. Inter and Noto Indic demo fonts: SIL OFL 1.1. |
+
+The **Hindi components are also separately available under Apache-2.0**: [AksharaLP-1M-v2](https://huggingface.co/sourojitdotcom/AksharaLP-1M-v2) and [AksharaLP-Sentence-2.8M](https://huggingface.co/sourojitdotcom/AksharaLP-Sentence-2.8M). This combined release does not revoke or replace those existing grants. Code, dictionaries and individual components retain their own upstream terms; see [NOTICE](NOTICE), [LICENSE](LICENSE) and [LICENSE-APACHE-2.0](LICENSE-APACHE-2.0). This documentation is engineering provenance, not legal advice.
+
+## FAQ
+
+**Is this a multilingual LLM?** No. It cannot answer questions, reason generally, summarize, or translate between languages. It changes Romanized spellings into native script.
+
+**Why 8M when a word model has 989K parameters?** Five independent word models total 4,944,768; optional Hindi LM adds 2,770,752; total 7,715,520, rounded to 8M. That count excludes small reranker coefficients and dictionary data. Only selected components load.
+
+**Is it the best model for these languages?** Not established. It trails IndicXlit's full package on all four new test languages. We claim measured footprint and measured results, not smallest-ever or best-in-class status.
+
+**Does every language understand a sentence?** No. Only Hindi has neural sentence rescoring. Others concatenate independent word predictions, preserving text layout. Hindi rescoring cannot recover a spelling outside its candidate list.
+
+**Can I use it commercially?** Not this combined release. Its model-card licence is CC-BY-NC-4.0. Use independently Apache-licensed Hindi releases where appropriate, or wait for a cleared retrain. Retain all source attributions and review applicable component terms.
+
+**Does text leave the device?** The demo fetches local model assets from the Space when loading a language. Typed text is processed in the browser and is not sent to an inference API or telemetry service. Once the selected assets are loaded, typing needs no network. This is not a promise to avoid the initial asset download.
+
+**Can I run the GGUFs with llama.cpp?** No. GGUF is a container here, not a supported llama.cpp architecture. Use our reader, dictionaries and decoding code.
+
+## Limitations & responsible use
+
+Names, ambiguous short words, unseen vocabulary, English code-switching, dialects and chat abbreviations can fail. The output must be user-reviewable; do not silently replace important legal, medical or financial text. Candidate suggestions are uncalibrated. Input encoding truncates a word after 31 Roman characters. Type-as-you-go inference is debounced; older phones and long sentences may be slow. All measured compute here is CPU, not an on-device mobile benchmark. Tiny low-bit downloads still expand to float32 arrays and large dictionary structures in RAM.
+
+This private release records the current artifacts. No “best in class,” general-LLM capability, sub-5-MB complete bundle, commercial-clean, or multilingual-sentence claim is made.
+
+---
+**AksharaLP · Sourojit Dhua · 2026-11-10 (IST)** · Private review copy. Do not make public without the owner's approval.

SHA256SUMS

@@ -0,0 +1,101 @@
+d39ccdda1d2005a1b51113aec99f0c29a55d70dc0e35399656866ce0d9438e94  .gitattributes
+cc97a2c4ff7723a1f25179dc7c18f676fa84191bbb36ea7d0061acaf3c102559  LICENSE
+e61722325c93e58388069e84e208e43ca5b6294050676ee27194921cf453a071  LICENSE-APACHE-2.0
+8dd4007ec7ce19774061f88c9173ef176951e329083c079728c901ce15b15f13  NOTICE
+edc975f244807c134d306d1376fdfb8b29073d5db540e30db0bc90371b3c1e45  README.md
+f6484f5caaebaf2acb68540cc2cb29402030aee22a7e45b25d37424872f769a8  aksharalp.mjs
+e29d4e04be4a7235d4f2b94b6578cd580486960a7068b048e1bf6c1b1799679a  assets/accuracy.png
+c5378130af32052f6020b7cdc5a491aa5f1d11de763c37ee5491ba1eaef0bf49  assets/accuracy.svg
+27da3ac213f72636663b84026093cf2a1c80f825d8d67f110466ae9df0d3d6fc  assets/banner.png
+a9f8a61342401feb1e4bb6c1f82d0831d3fcbf35e3a6788eb7027d193b8de129  assets/banner.svg
+70d7aee1fe5e1e61dc7154c0aa4e33c818ff30ebbbe5d78fb5f52ba09fad1e04  assets/sizes.png
+e6a92bb1337fb4997c5d657e12d0b34208d0240e006aa3eb58bce6d3a9b53bec  assets/sizes.svg
+f5689263059364546fe23824634183d8781defb4691f36d951fff63558f16afc  config.json
+13350ec5c2ffa334de1dd6fb11e318fb1eeb5b80791506768546c3e43607539d  evaluation/bn_frozen.json
+79b140c2feac29957461d9ee879db331ea30885dee334675694169315f0d7535  evaluation/examples.json
+8d7b75ab880b89c749765cba386fcf0c3a290cdb6c1d2c9491be4a341a5453bc  evaluation/gguf_independent_validation.json
+4f898f14ad3912b8ecfa2025476f17ddad7aaa307bcac381620ffa17f2652a27  evaluation/gguf_parity.json
+bf7eba31593b49e8223d56726022d6e68627ad437661003771144995b5cba07f  evaluation/gguf_sizes.json
+392f25d66278a10a0e4f0093539d8f117727737bfd1b3b2d0e4dc31cfb5c792d  evaluation/hindi_existing_results.json
+8e29063e276abfa4320d1fc299b3a5560ec2bbada77bb7147a8fba97563b3acd  evaluation/indicxlit_storage.json
+157f7694918ba21b4e5268edbcf54f3fa5f1416506234ef5b78f1b9a502755da  evaluation/kn_frozen.json
+8a9b660df627f538d67decc52b16b5349dc416d0cb0b25fb0b793f7752f54763  evaluation/latency.json
+50f1594f15182c2ade9a10d4b971418932e98d7413a40590d2fc96f30ff6b80b  evaluation/multiindic_test.json
+25b2cc588a557f77ee852427fe06c77b108cad74fb6082d299b72cc1e1d58fe0  evaluation/release_summary.json
+a8912129bb1143d06e7f8761b515516b6e503d2f36951e684ee2f602b7911285  evaluation/sizes.json
+4798b6de7e820251a93924186934287d70407705df242573910c74b60d85e431  evaluation/ta_frozen.json
+eb327b20a375b534bad337dbf90b4e054b58a788fc17a34b142668ccbdae0cca  evaluation/te_frozen.json
+7a0b78e133bb3ed773f6a8bae5e3f1af4b75f5158de4e814efadeaa0002ed580  gguf/aksharalp-bn-word-f16.gguf
+8311c77889381ce559d483b965027c2d409787ac090b146ae1af08060e4a4552  gguf/aksharalp-bn-word-q4.gguf
+446ff8f003c3680e9098df94727f8eb67a6a05a7c418888fb0bfeeb2742e098e  gguf/aksharalp-hi-sentence-f16.gguf
+8ecc83bc618b9c67173328a34f0f40e5c83ce77a993a6279acdeae6d724690db  gguf/aksharalp-hi-sentence-q4.gguf
+5ff23bc8d131b560ee29c79ee16d15294600149200948433d76f29f4b8c3e496  gguf/aksharalp-hi-sentence-q8.gguf
+3b6670429a226a39d357160691c634160667728bae8cfc7a711067424a2d224e  gguf/aksharalp-hi-word-f16.gguf
+737d14197514a2625eb9505eb0a22ebd62cd2a4b764c65761f4084599bb6eb36  gguf/aksharalp-hi-word-q4.gguf
+c9d273c8339864bd37c2b051cab60d2741c1d71e56e8c27d00a981ff28ef3864  gguf/aksharalp-kn-word-f16.gguf
+0e000f7fc0e52bbcb1f05753a46e6de50d1a1bdd6bb0ef46303361b9c39138ed  gguf/aksharalp-kn-word-q4.gguf
+3baf8764ecc77250666eb3f8eb83f2d7a33763e13227e2d6db8272868dd5a864  gguf/aksharalp-ta-word-f16.gguf
+f270e77e6355d53dc140f7cc7282551f5126302bcfbdbb17d802861510896c04  gguf/aksharalp-ta-word-q4.gguf
+3c556cec413db8f1485a1ee9ea237406f967937019e0d9d6076c3ae41b7a6231  gguf/aksharalp-te-word-f16.gguf
+56b9f956b69e4fcf9e8d3f23c9391a091208651e90b06e2bde6d7f46d7cb6a60  gguf/aksharalp-te-word-q4.gguf
+9348b0d8b522e9e3e45843a95c47087a3331f811a26c52813fa177a506f5cdcf  gguf_read.py
+edb2ad0bd7de4e87bd4d49ac406a9ef01011451bf23ac82d4c063bb9cce1dc2b  models/bn/dict.tsv.gz
+7007c674102f7d242b71832c877ea1d58b6b83393ac1ea2d2814f93dec499cda  models/bn/inventory.json
+e0bd564ad61f6dcef8eca6bf3d64f8b231146c9885aa4c5b282a2b2420606c4f  models/bn/params.json
+7d85a15252ea42676289a216bb775c2e48fea96634b32d2f2f0dce214951ada3  models/bn/training_report.json
+9f099f0b9524f446debf1d3f0ae1dc7ed1e88625c33f6dbb7b45e0348bf76376  models/bn/xlit.q4
+1c87e62a4249a63c333942c1c8736eb2c89f9ee3148ef8fc422784ba6ce21e43  models/bn/xlit.q4.gz
+73bd5436b77db4ee433c564521f19a2b78cb6a45c429f274d43bef909708347f  models/hi/dict/lex.bin
+3b4dbff703f395e97b4503646fde7782bab0f3d79b9731ef703df6b961e38f55  models/hi/dict/lex.bin.gz
+254e34abe7c1e6ed82f74d5cbf175aba1aedbee316363fe2a67e06291ba589ee  models/hi/dict/lm.bin
+ab3f6b4135862b5bbf42815cc1219274eed711b5a7dfc66f09fc7b0fe32ceed6  models/hi/dict/lm.bin.gz
+6d60eaf2b4e89a847a750a226680021460e454c6d1a06070a56e8f8664500565  models/hi/dict/params.json
+bc7dd23ba83ec6bc5841b03fa7b8e3cb934d0322dda652863537ce03958ea79d  models/hi/dict/params.json.gz
+620ab5ebf213fc4b8469b88338368b34616390154f72f64f3dcd49a550f01472  models/hi/manifest.json
+acb2bd2c0c12813d18c9696058260d57ea504db76db6d2d0810054a9e65a875e  models/hi/manifest.json.gz
+1dc58564cd649278c819641a65141e9e96a42f3fc7fba555b9072628674c2935  models/hi/params.json
+8e42f8aec0204bca50891244478d94ac3ba1801f8cc774c475b84c5581377c73  models/hi/params.json.gz
+2c90f300c0114354fcf0c5a12300b9ba98ce71b4580b54f0b870a9395e869227  models/hi/sentence/nlm.q4
+b8f3850506d93f3b0531beb37362cf1ad7c02a9e1d8c81ebc128efee9e44c17e  models/hi/sentence/nlm.q8
+7face18513d342c59b005f3ea57a0e86310dc863778a828e7743cea0d5c68526  models/hi/sentence/nlm.q8.gz
+652eafe0c1e2b46fec85663825621284f0a5911f2d7cc6e13b1182ce47cd9803  models/hi/xlit.q4
+191eb28f75f29c45ec29f044031a6ae5aedd6a80edc3f3a9d1ae78a53c04fb9d  models/hi/xlit.q4.gz
+97035264d30eddb5810467dc5bbf088614221ea6ba2c3fff349c6bbd196bb644  models/kn/dict.tsv.gz
+cc2085a1f34c929678658159bc22eaa7641ce4aecada94fa3336dc82e1e553a3  models/kn/inventory.json
+6076d9126ffce66630b634e47ac5b38d03cda3bce7ab4d07162aeb0cbc925356  models/kn/params.json
+bfae38dd1fb1c799e9005750b3e7ed93860a2be8051489a3cd0ba9ead17ae30f  models/kn/training_report.json
+3788fd545c297febb7dbbdbdd780e831042395b4ec99260b95c963607afc8664  models/kn/xlit.q4
+bedc5a698dc5c83a964f03586bc0ded67e88fa0b644cdd82fafc21f1e50925d9  models/kn/xlit.q4.gz
+4d14f514020e7737144c22991e9324a29734b5ab203bef3b68ec6d9f51e7cddb  models/ta/dict.tsv.gz
+e1ea48ffbd41b725a435c611d0ebd1da454904bc90e2368eb2fbaecee9f1870b  models/ta/inventory.json
+f4520a8320f02573e6b8b6530a372c980903fbf757999540698957e8eec58385  models/ta/params.json
+0c93e563d879dfef97ca0d36ae6c6ce4ca21dd417741be90478b7191214b0165  models/ta/training_report.json
+5dd185a033db79835fede66ab41a25f5897f24ad5d7c8ecc9864ca6ec167fdc0  models/ta/xlit.q4
+c15be889468520e6d5ee8dfbc4c77cad781a79f454d4681026fc082ae217afc2  models/ta/xlit.q4.gz
+20612a52ccf39c816d3068e78e9e9a3aa9aed9156f10de45ebbd543363943640  models/te/dict.tsv.gz
+ff20b9635ecef52f692a53807be1c6a5acf3922212d2bc14f7dd9283002bc690  models/te/inventory.json
+653f897efe1af78bb52149f2a18dc804b49ecac9c93add867871e64f56e07470  models/te/params.json
+da8bc7af7e481e995e3ada79f21585bdbcf7df8fc751fdf151d8a774f8f0f343  models/te/training_report.json
+aea094e46a3c61eb9487d1aaba3824d3d9dc60f3f1cf6b6f99cadf04fd4318f9  models/te/xlit.q4
+3513df30b5d3bfa07085baa19f0c9c1076e85268a5df88fcc4b492630b1b4eb2  models/te/xlit.q4.gz
+037d1f4e40f61457e4e33a9b8a034d2509316b723f27907849d55f0fb9cf23cf  multiindic.mjs
+a0bc14ac9629a0bb6b69fe94609eb922a9b71bbd327b17553581b91648d970c2  nlm.mjs
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  python/bhc/__init__.py
+af77db2406b19466f538ebc35be31b6ffcf23de2d8447e48247a8ba58c1c2697  python/bhc/agree.py
+aebfdd405693b5096744cbab03a30662e440308583302e9b5b54575ce0498231  python/bhc/casual.py
+b07229b0d484048f6f75fd8304dc62e2a05d2dbc450289bfb57f6efd669c9ffc  python/bhc/lexlm.py
+5b7c49b45309ac3f324906bd2546c35cfc4d5a199a9311a9d845233337a41b39  python/bhc/lextrie.py
+ba9c1c4b326328cbbc65085b379835fac522d116e1ea163625bfcadf5c0de723  python/bhc/nlm.py
+39e674d163ebcb7e7adcf02dd267fd6c9b188be1346486f2695736240c4f42ff  python/bhc/skel.py
+09617bd0f9b90a4179628dd40d786a30ba6a3a2e043d3993ecf36cf223524483  python/bhc/text.py
+67de6da4dc780c3128ca5c2da4461d1e0616d3f1c65619cc62ef59699a28c2d2  python/bhc/translit2.py
+e93f9cc572853158f7414a46ac24815aea40c76e0939a2b79a6034542aeb2615  python/bhc/translit3.py
+8eb7235292a0b42a4cdfc07c548809fe312e2eec2c622317ff40e0c203ff5766  python/bhc/xlit_model.py
+1a577ff4b9075b6ee73debed2955840d30a5796a846ba74af58561866681307f  python/bhc/xlit_q.py
+9348b0d8b522e9e3e45843a95c47087a3331f811a26c52813fa177a506f5cdcf  python/gguf_read.py
+e22e0dfcd2cee31d13d5eee303e07dea973032bfdfbd09b2c4060d69b0f12367  python/multiindic.py
+72cabbf383d3b033df0eb37be00bb7079f51b0d08d793f0ac9d7eec4709dffbb  python/requirements.txt
+de4599eedf54731c4d61f2bee9afb0d3776aaa5a49516fa4d2bf720bc2fdbb7a  sent.mjs
+f62025ba5b1c6b7a0e3559a0b6a6dc6f53773f188f8be118f9d0e6083b4e5211  tools/build_gguf.py
+9fcbb1848e3a46daaf9d9b317f240a1d04d7442a7242af99df7a23d24fd48940  tools/gguf_format.py
+dcd348d0f8f26b3316123228ae534d37155bfe552520141a38aa55b14707f0d9  v3.mjs

aksharalp.mjs

@@ -0,0 +1,822 @@
+/*!
+ * BhashaCore JS runtime — romanized Hindi / Hinglish -> Devanagari, fully offline (browser, Web Worker, Node >= 18).
+ * Zero dependencies. Pure-JS port of the reference Python runtime (bhc/translit2.py, bhc/xlit_q.py, bhc/lexlm.py,
+ * bhc/lextrie.py, bhc/lid.py, bhc/hinglish.py) over the exported asset formats:
+ *   xlit.q4 | xlit.q8 | xlit.int8   char Transformer (pre-LN enc/dec), 4/8-bit grouped or per-row INT8 weights
+ *   lex.bin                         Devanagari lexicon (LX4 alphabetical front-coded + 4-bit bands, or v2 bucketed)
+ *   lm.bin                          pruned backoff word-bigram LM (varint CSR + 4-bit codebooks)
+ *   params.json                     decoding params + reranker (linear w or MLP w1/b1/w2/b2; mu / sd)
+ *   lid.int8                        char-CNN + BiGRU token / sentence language ID (optional)
+ *   loan.tsv, natural_hi.tsv        Hinglish loanword lexicon, Natural-Hindi glossary (optional)
+ * Bundle layout: a small CORE (model + params.json) that works standalone, plus an optional dictionary pack in
+ * dict/ (lex.bin + lm.bin + params.json overriding the core params), fetched on first use, cached with the Cache API
+ * when available, and attached with engine.loadDict(); if it cannot be fetched the engine stays in core-only mode.
+ *
+ * AksharaLP by Sourojit Dhua — https://huggingface.co/sourojitdotcom/AksharaLP-1M-v2
+ * Copyright 2026 Sourojit Dhua. Licensed under the Apache License, Version 2.0; commercial use permitted.
+ * Attribution required (Apache-2.0 §4(d)): keep this notice and the NOTICE file in redistributions and derivatives:
+ *   "Transliteration by AksharaLP (Sourojit Dhua) — https://huggingface.co/sourojitdotcom/AksharaLP-1M-v2"
+ */
+
+// ------------------------------------------------------------------ vocab
+const PAD = 0, BOS = 1, EOS = 2, UNK = 3, MAXLEN = 32, V = 132;
+const NEG = -Infinity;
+const encSrc = (w) => {
+  const s = w.toLowerCase().slice(0, MAXLEN - 1), ids = [];
+  for (const c of s) { const k = c.charCodeAt(0); ids.push(c.length === 1 && k >= 97 && k <= 122 ? k - 97 + 4 : UNK); }
+  ids.push(EOS); return ids;
+};
+const t2i = (ch) => { const k = ch.codePointAt(0); return k >= 0x900 && k < 0x980 ? k - 0x900 + 4 : -1; };
+const encTgt = (w) => { const ids = []; for (const c of [...w].slice(0, MAXLEN - 2)) { const i = t2i(c); ids.push(i < 0 ? UNK : i); } ids.push(EOS); return ids; };
+const decTgt = (ids) => { let s = ""; for (const i of ids) if (i > UNK) s += String.fromCharCode(0x900 + i - 4); return s; };
+
+// ------------------------------------------------------------------ binary helpers
+function f16(h) {
+  const s = (h & 0x8000) ? -1 : 1, e = (h >> 10) & 0x1f, f = h & 0x3ff;
+  if (e === 0) return s * f * 5.960464477539063e-8;
+  if (e === 31) return f ? NaN : s * Infinity;
+  return s * (1 + f / 1024) * Math.pow(2, e - 15);
+}
+function readHeader(buf) {
+  const u8 = new Uint8Array(buf), dv = new DataView(u8.buffer, u8.byteOffset, u8.byteLength);
+  const n = dv.getUint32(0, true);
+  const meta = JSON.parse(new TextDecoder().decode(u8.subarray(4, 4 + n)));
+  const data = u8.subarray(4 + n);
+  return { meta, data, dv: new DataView(data.buffer, data.byteOffset, data.byteLength) };
+}
+const prod = (a) => a.reduce((x, y) => x * y, 1);
+const f16arr = (dv, off, n) => { const o = new Float32Array(n); for (let i = 0; i < n; i++) o[i] = f16(dv.getUint16(off + 2 * i, true)); return o; };
+
+/** Parse xlit.int8 / xlit.q4 / xlit.q8 / lid.int8 into {cfg, extra, T: {name: {shape, data: Float32Array}}}. */
+export function parseWeights(buf) {
+  const { meta, data, dv } = readHeader(buf); const T = {};
+  for (const t of meta.tensors) {
+    const [k, shape, kind, off] = t; const cnt = prod(shape); let w;
+    if (kind === "f16") w = f16arr(dv, off, cnt);
+    else if (kind === "i8") {                                   // per-row symmetric INT8 (+ fp16 row scale)
+      const r = shape[0], c = cnt / r, sc = f16arr(dv, off + cnt, r); w = new Float32Array(cnt);
+      for (let i = 0; i < r; i++) for (let j = 0; j < c; j++) { const q = (data[off + i * c + j] << 24) >> 24; w[i * c + j] = q * sc[i]; }
+    } else {                                                    // q4 / q8: group-wise symmetric (G inputs per fp16 scale)
+      const bits = kind.length > 1 ? parseInt(kind.slice(1)) : meta.bits, G = meta.G, ln = t[4];
+      const [r, c] = shape, ng = Math.ceil(c / G), sc = f16arr(dv, off + ln, r * ng); w = new Float32Array(cnt);
+      for (let i = 0; i < cnt; i++) {
+        let q;
+        if (bits === 4) { const b = data[off + (i >> 1)]; q = ((i & 1) ? (b & 15) : (b >> 4)) - 8; }
+        else q = (data[off + i] << 24) >> 24;
+        const row = (i / c) | 0, col = i - row * c; w[i] = q * sc[row * ng + ((col / G) | 0)];
+      }
+    }
+    T[k] = { shape, data: w };
+  }
+  return { cfg: meta.cfg, extra: meta.extra || {}, T };
+}
+
+// ------------------------------------------------------------------ math kernels
+function linear(x, xo, W, b, out, oo, nin, nout) {           // out[oo..] = W x + b ; W row-major [nout, nin]
+  for (let o = 0; o < nout; o++) {
+    let s = b ? b[o] : 0; const r = o * nin;
+    for (let i = 0; i < nin; i++) s += W[r + i] * x[xo + i];
+    out[oo + o] = s;
+  }
+}
+function layerNorm(x, xo, g, b, out, oo, d) {
+  let m = 0; for (let i = 0; i < d; i++) m += x[xo + i]; m /= d;
+  let v = 0; for (let i = 0; i < d; i++) { const t = x[xo + i] - m; v += t * t; } v /= d;
+  const r = 1 / Math.sqrt(v + 1e-5);
+  for (let i = 0; i < d; i++) out[oo + i] = (x[xo + i] - m) * r * g[i] + b[i];
+}
+function erf(x) {                                              // |err| < 1.2e-7 (Numerical Recipes erfc)
+  const z = Math.abs(x), t = 1 / (1 + 0.5 * z);
+  const r = t * Math.exp(-z * z - 1.26551223 + t * (1.00002368 + t * (0.37409196 + t * (0.09678418 + t * (-0.18628806 +
+    t * (0.27886807 + t * (-1.13520398 + t * (1.48851587 + t * (-0.82215223 + t * 0.17087277)))))))));
+  return x >= 0 ? 1 - r : r - 1;
+}
+const gelu = (x) => 0.5 * x * (1 + erf(x / Math.SQRT2));
+const sigmoid = (x) => 1 / (1 + Math.exp(-x));
+
+// ------------------------------------------------------------------ char Transformer
+export class XlitModel {
+  constructor(w) {
+    const c = w.cfg; this.V = w.T["temb.weight"].shape[0];
+    const start = {hi:0x900,bn:0x980,te:0xc00,kn:0xc80,ta:0xb80}[c.vocab || "hi"];
+    this.target = Array.from({length:128}, (_,i)=>String.fromCharCode(start+i));
+    if (c.vocab === "bn") this.target.push("\u0964");
+    this.decTgt = ids => ids.filter(i=>i>3).map(i=>this.target[i-4]).join("");
+    this.d = c.d; this.h = c.h; this.dk = c.d / c.h; this.ff = c.ff; this.ne = c.enc; this.nd = c.dec;
+    const g = (k) => { const t = w.T[k]; if (!t) throw new Error("missing tensor " + k); return t.data; };
+    this.semb = g("semb.weight"); this.temb = g("temb.weight"); this.spos = g("spos.weight"); this.tpos = g("tpos.weight");
+    const att = (p) => ({ q: g(p + ".q.weight"), qb: g(p + ".q.bias"), kv: g(p + ".kv.weight"), kvb: g(p + ".kv.bias"), o: g(p + ".o.weight"), ob: g(p + ".o.bias") });
+    const ln = (p) => ({ g: g(p + ".weight"), b: g(p + ".bias") });
+    const ffn = (p) => ({ w1: g(p + ".ff.0.weight"), b1: g(p + ".ff.0.bias"), w2: g(p + ".ff.2.weight"), b2: g(p + ".ff.2.bias") });
+    this.enc = []; for (let i = 0; i < c.enc; i++) { const p = "enc." + i; this.enc.push({ n1: ln(p + ".n1"), att: att(p + ".att"), n2: ln(p + ".n2"), ff: ffn(p) }); }
+    this.dec = []; for (let i = 0; i < c.dec; i++) { const p = "dec." + i; this.dec.push({ n1: ln(p + ".n1"), sa: att(p + ".sa"), n2: ln(p + ".n2"), ca: att(p + ".ca"), n3: ln(p + ".n3"), ff: ffn(p) }); }
+    this.en = ln("en"); this.dn = ln("dn"); this.scale = Math.sqrt(c.d);
+    this.nParams = Object.values(w.T).reduce((a, t) => a + t.data.length, 0);
+    const d = this.d; this._y = new Float32Array(d); this._a = new Float32Array(d); this._f = new Float32Array(this.ff); this._q = new Float32Array(d); this._kv = new Float32Array(2 * d);
+    this._sc = new Float64Array(MAXLEN);
+  }
+  _ffn(L, x, xo, n) { // x[xo] += FF(LN(x))
+    const d = this.d, y = this._y, f = this._f, a = this._a;
+    layerNorm(x, xo, n.g, n.b, y, 0, d); linear(y, 0, L.w1, L.b1, f, 0, d, this.ff);
+    for (let i = 0; i < this.ff; i++) f[i] = gelu(f[i]);
+    linear(f, 0, L.w2, L.b2, a, 0, this.ff, d); for (let i = 0; i < d; i++) x[xo + i] += a[i];
+  }
+  /** attention of one query q (len d) over K/V rows (stride 2d in kvbuf: K at +0, V at +d) -> out (len d) */
+  _attend(q, kvbuf, kvo, n, out) {
+    const { h, dk, d } = this, sc = n <= MAXLEN ? this._sc : new Float64Array(n), inv = 1 / Math.sqrt(dk);
+    for (let hh = 0; hh < h; hh++) {
+      let mx = -Infinity;
+      for (let j = 0; j < n; j++) { let s = 0; const ko = kvo + j * 2 * d + hh * dk; for (let i = 0; i < dk; i++) s += q[hh * dk + i] * kvbuf[ko + i]; s *= inv; sc[j] = s; if (s > mx) mx = s; }
+      let z = 0; for (let j = 0; j < n; j++) { sc[j] = Math.exp(sc[j] - mx); z += sc[j]; }
+      for (let i = 0; i < dk; i++) { let s = 0; for (let j = 0; j < n; j++) s += sc[j] * kvbuf[kvo + j * 2 * d + d + hh * dk + i]; out[hh * dk + i] = s / z; }
+    }
+  }
+  /** encode one word -> {S, cross: per decoder layer K/V of memory (Float32Array S*2d)} */
+  encode(ids) {
+    const d = this.d, S = ids.length, x = new Float32Array(S * d), y = new Float32Array(S * d), kv = new Float32Array(S * 2 * d), att = new Float32Array(d), o = this._a;
+    for (let t = 0; t < S; t++) for (let i = 0; i < d; i++) x[t * d + i] = this.semb[ids[t] * d + i] * this.scale + this.spos[t * d + i];
+    for (const L of this.enc) {
+      for (let t = 0; t < S; t++) { layerNorm(x, t * d, L.n1.g, L.n1.b, y, t * d, d); linear(y, t * d, L.att.kv, L.att.kvb, kv, t * 2 * d, d, 2 * d); }
+      const nx = new Float32Array(x);
+      for (let t = 0; t < S; t++) {
+        linear(y, t * d, L.att.q, L.att.qb, this._q, 0, d, d); this._attend(this._q, kv, 0, S, att);
+        linear(att, 0, L.att.o, L.att.ob, o, 0, d, d); for (let i = 0; i < d; i++) nx[t * d + i] += o[i];
+      }
+      for (let t = 0; t < S; t++) this._ffn(L.ff, nx, t * d, L.n2);
+      x.set(nx);
+    }
+    const mem = new Float32Array(S * d); for (let t = 0; t < S; t++) layerNorm(x, t * d, this.en.g, this.en.b, mem, t * d, d);
+    const cross = this.dec.map((L) => { const c = new Float32Array(S * 2 * d); for (let t = 0; t < S; t++) linear(mem, t * d, L.ca.kv, L.ca.kvb, c, t * 2 * d, d, 2 * d); return c; });
+    return { S, cross };
+  }
+  newCache(K) { const d = this.d; return this.dec.map(() => new Float32Array(K * MAXLEN * 2 * d)); }
+  /** one decoder step for beam rows `rows` (indices into cache) with tokens tok[row] at position t -> logprobs[row] (Float64Array V) */
+  step(enc, cache, rows, tok, t, out) {
+    const V = this.V, d = this.d, x = new Float32Array(d), y = this._y, att = new Float32Array(d), o = this._a;
+    for (const r of rows) {
+      for (let i = 0; i < d; i++) x[i] = this.temb[tok[r] * d + i] * this.scale + this.tpos[t * d + i];
+      for (let l = 0; l < this.nd; l++) {
+        const L = this.dec[l], C = cache[l], base = r * MAXLEN * 2 * d;
+        layerNorm(x, 0, L.n1.g, L.n1.b, y, 0, d);
+        linear(y, 0, L.sa.kv, L.sa.kvb, C, base + t * 2 * d, d, 2 * d);
+        linear(y, 0, L.sa.q, L.sa.qb, this._q, 0, d, d); this._attend(this._q, C, base, t + 1, att);
+        linear(att, 0, L.sa.o, L.sa.ob, o, 0, d, d); for (let i = 0; i < d; i++) x[i] += o[i];
+        layerNorm(x, 0, L.n2.g, L.n2.b, y, 0, d);
+        linear(y, 0, L.ca.q, L.ca.qb, this._q, 0, d, d); this._attend(this._q, enc.cross[l], 0, enc.S, att);
+        linear(att, 0, L.ca.o, L.ca.ob, o, 0, d, d); for (let i = 0; i < d; i++) x[i] += o[i];
+        this._ffn(L.ff, x, 0, L.n3);
+      }
+      layerNorm(x, 0, this.dn.g, this.dn.b, y, 0, d);
+      const lp = out[r]; let mx = -Infinity;
+      for (let v = 0; v < V; v++) { let s = 0; for (let i = 0; i < d; i++) s += y[i] * this.temb[v * d + i]; lp[v] = s; if (s > mx) mx = s; }
+      let z = 0; for (let v = 0; v < V; v++) z += Math.exp(lp[v] - mx); const lz = mx + Math.log(z);
+      for (let v = 0; v < V; v++) lp[v] -= lz;
+    }
+  }
+  /** beam search for one word (identical procedure to bhc.xlit_model.beam_search_fast for one batch row) */
+  beam(enc, S, { beam: K = 8, nbest = 8, trie = null } = {}) {
+    const V = this.V, d = this.d, cache = this.newCache(K), out = Array.from({ length: K }, () => new Float64Array(V));
+    let score = new Array(K).fill(NEG); score[0] = 0; let tok = new Array(K).fill(BOS); let hist = Array.from({ length: K }, () => []);
+    let nodes = trie ? new Array(K).fill(0) : null; const done = []; const L = Math.min(MAXLEN - 1, S + 8);
+    const tmp = new Float32Array(MAXLEN * 2 * d);
+    for (let t = 0; t < L; t++) {
+      const rows = []; for (let k = 0; k < K; k++) if (score[k] > NEG) rows.push(k);
+      this.step(enc, cache, rows, tok, t, out);
+      const cand = [];                                          // (total, beam, v)
+      for (const k of rows) {
+        const lp = out[k]; let allow = null;
+        if (trie) { const n = nodes[k]; if (n === null) continue; allow = trie.allowed(n); }
+        if (allow) { for (const v of allow) if (v !== PAD && v !== BOS && v !== UNK) cand.push([score[k] + lp[v], k, v]); }
+        else { cand.push([score[k] + lp[EOS], k, EOS]); for (let v = 4; v < V; v++) cand.push([score[k] + lp[v], k, v]); }
+      }
+      cand.sort((a, b) => b[0] - a[0] || (a[1] * V + a[2]) - (b[1] * V + b[2]));
+      const top = cand.slice(0, 2 * K);
+      const ns = new Array(K).fill(NEG), nt = new Array(K).fill(EOS), par = new Array(K).fill(0), nn = trie ? new Array(K).fill(null) : null;
+      let j = 0;
+      for (const [sc, b, v] of top) {
+        if (!(sc > NEG)) continue;
+        if (v === EOS) { if (done.length < nbest) done.push([this.decTgt(hist[b]), sc]); continue; }
+        if (j < K) { ns[j] = sc; nt[j] = v; par[j] = b; if (trie) nn[j] = trie.child(nodes[b], v); j++; }
+      }
+      let alive = true;
+      if (done.length >= nbest && (j === 0 || ns[0] < Math.min(...done.map((x) => x[1])))) alive = false;
+      else if (!j) alive = false;
+      if (!alive) break;
+      for (const C of cache) {                                   // reorder self-attention caches by parent beam
+        const n = (t + 1) * 2 * d, olds = [];
+        for (let k = 0; k < j; k++) { const src = par[k] * MAXLEN * 2 * d; olds.push(C.slice(src, src + n)); }
+        for (let k = 0; k < j; k++) C.set(olds[k], k * MAXLEN * 2 * d);
+      }
+      hist = Array.from({ length: K }, (_, k) => (k < j ? hist[par[k]].concat([nt[k]]) : []));
+      score = ns; tok = nt; nodes = nn;
+    }
+    const best = new Map();
+    for (const [s, sc] of done) if (!best.has(s) || sc > best.get(s)) best.set(s, sc);
+    return [...best.entries()].sort((a, b) => b[1] - a[1]).slice(0, nbest);
+  }
+  /** teacher-forced ln P(tgt | src) (as bhc.xlit_model.score_pairs) */
+  scorePair(enc, tgt) {
+    const V = this.V, ids = [...tgt].slice(0,MAXLEN-2).map(c=>{const i=this.target.indexOf(c);return i<0?UNK:i+4;}).concat([EOS]), cache = this.newCache(1), out = [new Float64Array(V)]; let tok = [BOS], s = 0;
+    for (let t = 0; t < ids.length; t++) { this.step(enc, cache, [0], tok, t, out); s += out[0][ids[t]]; tok = [ids[t]]; }
+    return s;
+  }
+}
+
+// ------------------------------------------------------------------ lexicon + bigram LM
+function readVarints(b, n, pos) {
+  const out = new Uint32Array(n);
+  for (let i = 0; i < n; i++) { let v = 0, sh = 0, c; do { c = b[pos++]; v += (c & 0x7f) * 2 ** sh; sh += 7; } while (c >= 0x80); out[i] = v; }
+  return [out, pos];
+}
+const unpack4 = (b, n) => { const o = new Uint8Array(n); for (let i = 0; i < n; i++) o[i] = (i & 1) ? b[i >> 1] & 15 : b[i >> 1] >> 4; return o; };
+
+export class LexLM {
+  constructor(lexBuf, lmBuf) {
+    if (!lexBuf) {                                                              // core-only: empty lexicon, no LM
+      this.words = []; this.sortedWords = []; this.uni = new Float32Array(0); this.N = 0; this.unk = 0; this.alpha = true; this.hasLM = false; return;
+    }
+    const b = new Uint8Array(lexBuf), dv = new DataView(b.buffer, b.byteOffset, b.byteLength); const words = [];
+    const decodeFC = (P, S, nWords) => { let prev = [], j = 0, pi = 0;
+      while (words.length < nWords) { const k = P ? P[pi++] : S[j++]; const cur = prev.slice(0, k);
+        for (;;) { const c = S[j++], ch = c & 0x7f; if (ch !== 0x7f) cur.push(ch); if (c & 0x80) break; }
+        words.push(String.fromCharCode(...cur.map((x) => 0x900 + x))); prev = cur; } };
+    if (b[0] === 0x4c && b[1] === 0x58 && b[2] === 0x34 && b[3] === 0) {      // "LX4\0"
+      const N = dv.getUint32(4, true), nfc = dv.getUint32(8, true), NB = dv.getUint32(16, true); this.unk = dv.getFloat32(12, true);
+      let pos = 20; const ucb = new Float32Array(NB); for (let i = 0; i < NB; i++) ucb[i] = dv.getFloat32(pos + 4 * i, true); pos += 4 * NB;
+      const band = unpack4(b.subarray(pos, pos + ((N + 1) >> 1)), N); pos += (N + 1) >> 1;
+      this.uni = new Float32Array(N); for (let i = 0; i < N; i++) this.uni[i] = ucb[band[i]];
+      decodeFC(b.subarray(pos, pos + N), b.subarray(pos + N, pos + N + nfc), N); this.N = N; this.alpha = true;
+    } else {                                                                     // v2 (frequency-bucketed)
+      const N = dv.getUint32(0, true), nb = dv.getUint32(4, true), nfc = dv.getUint32(8, true); this.unk = dv.getFloat32(12, true);
+      let [bn, pos] = readVarints(b, nb, 16); const blp = []; for (let i = 0; i < nb; i++) blp.push(f16(dv.getUint16(pos + 2 * i, true))); pos += 2 * nb;
+      this.uni = new Float32Array(N); let w = 0; for (let i = 0; i < nb; i++) for (let k = 0; k < bn[i]; k++) this.uni[w++] = blp[i];
+      decodeFC(null, b.subarray(pos, pos + nfc), N); this.N = N; this.alpha = false;
+    }
+    this.words = words;
+    let sorted = this.alpha; for (let i = 1; sorted && i < words.length; i++) if (!(words[i - 1] < words[i])) sorted = false;
+    if (!sorted) { this.map = new Map(); words.forEach((w, i) => this.map.set(w, i)); }
+    this.sortedWords = sorted ? words : [...new Set(words)].sort();
+    this.hasLM = false;
+    if (lmBuf) this._loadLM(new Uint8Array(lmBuf));
+  }
+  id(w) {
+    if (this.map) { const i = this.map.get(w); return i === undefined ? null : i; }
+    let lo = 0, hi = this.words.length; while (lo < hi) { const m = (lo + hi) >> 1; if (this.words[m] < w) lo = m + 1; else hi = m; }
+    return lo < this.words.length && this.words[lo] === w ? lo : null;
+  }
+  _loadLM(b) {
+    const dv = new DataView(b.buffer, b.byteOffset, b.byteLength), N = this.N;
+    const nbg = dv.getUint32(4, true), nh = dv.getUint32(16, true); let pos = 20;
+    const cb = [], cbb = []; for (let i = 0; i < 16; i++) cb.push(dv.getFloat32(pos + 4 * i, true)); pos += 64;
+    for (let i = 0; i < 16; i++) cbb.push(dv.getFloat32(pos + 4 * i, true)); pos += 64;
+    let cnt, delta; [cnt, pos] = readVarints(b, N + 1, pos); [delta, pos] = readVarints(b, nbg, pos);
+    const q = unpack4(b.subarray(pos, pos + ((nbg + 1) >> 1)), nbg); pos += (nbg + 1) >> 1;
+    const qb = unpack4(b.subarray(pos, pos + ((nh + 1) >> 1)), nh);
+    this.off = new Uint32Array(N + 2); this.succ = new Uint32Array(nbg); this.bq = q; this.cb = Float32Array.from(cb);
+    this.bow = new Float32Array(N + 1); let i = 0, hi = 0;
+    for (let h = 0; h <= N; h++) {
+      this.off[h] = i; const n = cnt[h]; if (!n) continue;
+      this.bow[h] = cbb[qb[hi++]]; let acc = -1;
+      for (let k = 0; k < n; k++) { acc = acc < 0 ? delta[i] : acc + delta[i] + 1; this.succ[i++] = acc; }
+    }
+    this.off[N + 1] = i; this.hasLM = nbg > 0;
+  }
+  unigram(wid) { return wid === null ? this.unk : wid === this.N ? Math.log(0.04) : this.uni[wid]; }
+  /** ln P(wid | prev); prev null = sentence start; wid null = OOV; wid N = </s> */
+  logp(prev, wid) {
+    const h = prev === null ? this.N : prev;
+    if (wid !== null && this.hasLM) {
+      let lo = this.off[h], hi = this.off[h + 1];
+      while (lo < hi) { const m = (lo + hi) >> 1; if (this.succ[m] < wid) lo = m + 1; else hi = m; }
+      if (lo < this.off[h + 1] && this.succ[lo] === wid) return this.cb[this.bq[lo]];
+    }
+    return (this.hasLM ? this.bow[h] : 0) + this.unigram(wid);
+  }
+}
+
+/** lazy lexicon trie over the sorted word list (same node semantics as bhc/lextrie.py) */
+class LazyTrie {
+  constructor(sortedWords) { this.w = sortedWords; this.nodes = [[0, sortedWords.length, 0]]; this.ids = new Map([["0,0," + sortedWords.length, 0]]); this.cache = new Map(); this.allow = new Map(); }
+  _bisect(x, lo, hi) { const w = this.w; while (lo < hi) { const m = (lo + hi) >> 1; if (w[m] < x) lo = m + 1; else hi = m; } return lo; }
+  children(n) {
+    let c = this.cache.get(n); if (c) return c;
+    const [lo, hi, d] = this.nodes[n], w = this.w; c = new Map(); let pos = lo;
+    if (pos < hi && w[pos].length === d) pos++;
+    while (pos < hi) {
+      const ch = w[pos].charCodeAt(d), pre = w[pos].slice(0, d);
+      const end = this._bisect(pre + String.fromCharCode(ch + 1), pos, hi);
+      const key = pos + "," + (d + 1) + "," + end; let nid = this.ids.get(key);
+      if (nid === undefined) { nid = this.nodes.length; this.nodes.push([pos, end, d + 1]); this.ids.set(key, nid); }
+      c.set(ch - 0x900 + 4, nid); pos = end;
+    }
+    if (this.cache.size > 400000) { this.cache.clear(); this.allow.clear(); }
+    this.cache.set(n, c); return c;
+  }
+  final(n) { const [lo, , d] = this.nodes[n]; return lo < this.w.length && this.w[lo].length === d; }
+  allowed(n) { let a = this.allow.get(n); if (!a) { a = [...this.children(n).keys()]; if (this.final(n)) a.push(EOS); this.allow.set(n, a); } return a; }
+  child(n, v) { return this.children(n).get(v); }
+}
+
+// ------------------------------------------------------------------ language ID (char-CNN + BiGRU)
+const LID_CH = {}; [..."abcdefghijklmnopqrstuvwxyz'"].forEach((c, i) => (LID_CH[c] = i + 3));
+const WMAX = 16, MODES3 = ["english", "romanized_hindi", "hinglish"];
+const RE_LATIN_APOS = /^[A-Za-z]+(?:'[A-Za-z]+)?$/, RE_DEVA = /^[\u0900-\u0963\u0971-\u097F]+$/;
+function whash(w, n) { let h = 2166136261; for (const ch of new TextEncoder().encode(w.toLowerCase())) h = Math.imul(h ^ ch, 16777619) >>> 0; return h % n; }
+
+export class LID {
+  constructor(w) {
+    this.cfg = w.cfg; this.x = w.extra || {}; const g = (k) => w.T[k] && w.T[k].data;
+    Object.assign(this, { ce: g("ce.weight"), c1: g("c1.weight"), c1b: g("c1.bias"), c2: g("c2.weight"), c2b: g("c2.bias"), wemb: g("wemb.weight"),
+      wih: g("gru.weight_ih_l0"), whh: g("gru.weight_hh_l0"), bih: g("gru.bias_ih_l0"), bhh: g("gru.bias_hh_l0"),
+      wihR: g("gru.weight_ih_l0_reverse"), whhR: g("gru.weight_hh_l0_reverse"), bihR: g("gru.bias_ih_l0_reverse"), bhhR: g("gru.bias_hh_l0_reverse"),
+      tw: g("tok.weight"), tb: g("tok.bias"), s0: g("sent.0.weight"), s0b: g("sent.0.bias"), s2: g("sent.2.weight"), s2b: g("sent.2.bias") });
+    this.nParams = Object.values(w.T).reduce((a, t) => a + t.data.length, 0);
+  }
+  _wvec(word) {
+    const { ce: E, cf } = this.cfg; const ids = new Array(WMAX).fill(0); const w = [...word.toLowerCase()].slice(0, WMAX - 2);
+    ids[0] = 1; w.forEach((c, i) => (ids[i + 1] = LID_CH[c] ?? 2)); ids[w.length + 1] = 2;
+    const x = new Float32Array(E * WMAX); for (let t = 0; t < WMAX; t++) for (let i = 0; i < E; i++) x[i * WMAX + t] = this.ce[ids[t] * E + i];
+    const conv = (inp, cin, W, b) => { const o = new Float32Array(cf * WMAX);
+      for (let oc = 0; oc < cf; oc++) for (let t = 0; t < WMAX; t++) { let s = b[oc];
+        for (let ic = 0; ic < cin; ic++) for (let k = 0; k < 3; k++) { const tt = t + k - 1; if (tt >= 0 && tt < WMAX) s += W[(oc * cin + ic) * 3 + k] * inp[ic * WMAX + tt]; }
+        o[oc * WMAX + t] = s > 0 ? s : 0; } return o; };
+    const h1 = conv(x, E, this.c1, this.c1b), h2 = conv(h1, cf, this.c2, this.c2b);
+    const wh = this.cfg.wh, wd = this.cfg.wd, v = new Float32Array(cf + (wh ? wd : 0));
+    for (let oc = 0; oc < cf; oc++) { let m = -Infinity; for (let t = 0; t < WMAX; t++) { const val = ids[t] === 0 ? -1e4 : h2[oc * WMAX + t] + h1[oc * WMAX + t]; if (val > m) m = val; } v[oc] = m; }
+    if (wh) { const r = whash(word, wh); for (let i = 0; i < wd; i++) v[cf + i] = this.wemb[r * wd + i]; }
+    return v;
+  }
+  _gru(xs, Wih, Whh, bih, bhh, rev) {
+    const H = this.cfg.hid, nin = xs[0].length, T = xs.length, out = new Array(T); let h = new Float32Array(H);
+    const gi = new Float32Array(3 * H), gh = new Float32Array(3 * H);
+    for (let s = 0; s < T; s++) {
+      const t = rev ? T - 1 - s : s; linear(xs[t], 0, Wih, bih, gi, 0, nin, 3 * H); linear(h, 0, Whh, bhh, gh, 0, H, 3 * H);
+      const nh = new Float32Array(H);
+      for (let i = 0; i < H; i++) { const r = sigmoid(gi[i] + gh[i]), z = sigmoid(gi[H + i] + gh[H + i]), n = Math.tanh(gi[2 * H + i] + r * gh[2 * H + i]); nh[i] = (1 - z) * n + z * h[i]; }
+      h = nh; out[t] = h;
+    }
+    return out;
+  }
+  /** Latin words -> {pen: P(EN) per word, ps: [english, romanized_hindi, hinglish]} */
+  run(words) {
+    if (!words.length) return { pen: [], ps: null };
+    const H = this.cfg.hid, xs = words.map((w) => this._wvec(w)), T = xs.length, nin = xs[0].length;
+    const f = this._gru(xs, this.wih, this.whh, this.bih, this.bhh, false), r = this._gru(xs, this.wihR, this.whhR, this.bihR, this.bhhR, true);
+    const pen = [], hm = new Float32Array(2 * H), hx = new Float32Array(2 * H).fill(-1e4), z = new Float32Array(2 * H + nin), lo = new Float32Array(2);
+    for (let t = 0; t < T; t++) {
+      for (let i = 0; i < H; i++) { z[i] = f[t][i]; z[H + i] = r[t][i]; } z.set(xs[t], 2 * H);
+      linear(z, 0, this.tw, this.tb, lo, 0, 2 * H + nin, 2); const m = Math.max(lo[0], lo[1]), e0 = Math.exp(lo[0] - m), e1 = Math.exp(lo[1] - m); pen.push(e0 / (e0 + e1));
+      for (let i = 0; i < 2 * H; i++) { hm[i] += z[i] / T; if (z[i] > hx[i]) hx[i] = z[i]; }
+    }
+    const sin = new Float32Array(4 * H + 1); sin.set(hm, 0); sin.set(hx, 2 * H); sin[4 * H] = pen.reduce((a, b) => a + b, 0) / T;
+    const h1 = new Float32Array(32); linear(sin, 0, this.s0, this.s0b, h1, 0, 4 * H + 1, 32); for (let i = 0; i < 32; i++) h1[i] = Math.max(0, h1[i]);
+    const s = new Float32Array(3); linear(h1, 0, this.s2, this.s2b, s, 0, 32, 3); const m = Math.max(...s); const e = [...s].map((v) => Math.exp(v - m)); const Z = e.reduce((a, b) => a + b, 0);
+    return { pen, ps: e.map((v) => v / Z) };
+  }
+  tag(tokens) {
+    const lat = tokens.filter((t) => RE_LATIN_APOS.test(t)); const { pen } = this.run(lat); let k = 0; const th = this.x.tok_th ?? 0.5;
+    return tokens.map((t) => (RE_DEVA.test(t) ? "DEV" : RE_LATIN_APOS.test(t) ? (pen[k++] >= th ? "EN" : "HI") : "OTHER"));
+  }
+  mode(tokens) {
+    const nd = tokens.filter((t) => RE_DEVA.test(t)).length, nl = tokens.filter((t) => RE_LATIN_APOS.test(t)).length;
+    if (nd && !nl) return "devanagari";
+    if (nd && nl) return nd / (nd + nl) >= (this.x.dev_frac ?? 0.5) ? "devanagari" : "mixed_script";
+    if (!nl) return "english";
+    const { ps } = this.run(tokens.filter((t) => RE_LATIN_APOS.test(t))); const bias = this.x.mode_bias || [0, 0, 0];
+    let best = 0, bv = -Infinity; ps.forEach((p, i) => { const v = Math.log(p + 1e-9) + bias[i]; if (v > bv) { bv = v; best = i; } });
+    return MODES3[best];
+  }
+}
+
+// ------------------------------------------------------------------ tokenizer + policy tables
+export const TOKEN_RE = /(https?:\/\/\S+|www\.\S+|[\p{L}\p{N}_.+-]+@[\p{L}\p{N}_-]+\.[\p{L}\p{N}_.]+|[@#][\p{L}\p{N}_]+|[A-Za-z]+(?:'[A-Za-z]+)?|[\u0900-\u0963\u0971-\u097F]+|[0-9\u0966-\u096F]+(?:[.,:][0-9\u0966-\u096F]+)*|[\u0964\u0965]|[^\sA-Za-z0-9\u0900-\u097F])/gu;
+const RE_LATIN = /^[A-Za-z]+$/;
+const RE_ACRO = /^[b-df-hj-np-tv-z]{2,7}$/;
+const LETTER = Object.fromEntries([..."abcdefghijklmnopqrstuvwxyz"].map((c, i) => [c, ("\u090f \u092c\u0940 \u0938\u0940 \u0921\u0940 \u0908 \u090f\u092b \u091c\u0940 \u090f\u091a \u0906\u0908 \u091c\u0947 \u0915\u0947 \u090f\u0932 \u090f\u092e \u090f\u0928 " +
+  "\u0913 \u092a\u0940 \u0915\u094d\u092f\u0942 \u0906\u0930 \u090f\u0938 \u091f\u0940 \u092f\u0942 \u0935\u0940 \u0921\u092c\u094d\u0932\u094d\u092f\u0942 \u090f\u0915\u094d\u0938 \u0935\u093e\u0908 \u091c\u0947\u0921").split(" ")[i]]));
+// Hinglish policy letter names (bhc/hinglish baseline): f -> एफ़, z -> ज़ेड
+const LETTERS_H = { ...LETTER, f: "\u090f\u092b\u093c", z: "\u091c\u093c\u0947\u0921" };
+const VAR_SWAPS = [["\u0949", "\u094b"], ["\u094b", "\u0949"], ["\u093e", "\u0949"], ["\u0949", "\u093e"], ["\u0903", ""], ["\u092b\u093c", "\u092b"], ["\u092b", "\u092b\u093c"],
+  ["\u091c\u093c", "\u091c"], ["\u091c", "\u091c\u093c"], ["\u0901", "\u0902"], ["\u0902", "\u0901"]];
+export const FEATS = ["sc", "scn", "inlex", "uni", "free0", "infree", "incon", "aw", "visarga", "nukta", "chandra", "acro", "variant", "frank",
+  "skel", "skmatch", "lev", "sc_rel", "scn_rel", "uni_rel", "lenr", "visr", "hfin", "acro_nv",
+  "ck1", "ck2", "ckr", "abbr", "abbr_r", "novow"];   // v2 casual features (append-only; older params use the first mu.length)
+const BRANDS = new Set(`whatsapp google meet paytm amazon flipkart youtube instagram facebook gmail excel uber ola swiggy zomato
+netflix linkedin github python java jira slack zoom iphone android windows microsoft apple samsung twitter telegram
+chrome firefox safari office365 outlook teams spotify hotstar jio airtel phonepe gpay`.split(/\s+/));
+const KEEP_RE = /^(https?:\/\/|www\.|[@#])|@.+\.|_|\d.*[A-Za-z]|[A-Za-z].*\d|^[a-z]+[A-Z]/;
+const isUpper = (t) => /[A-Z]/.test(t) && t === t.toUpperCase();
+const DEFAULTS = { beam: 8, nbest: 8, cbeam: 8, variants: true, acro: true, w: null, mu: null, sd: null, w_lm: 0.8, oov_pen_s: 4.0, w_ch_s: 1.0, use_lm: true,
+  skel: false, skel_k: 200, skel_m: 16, skel_wu: 0.03, visr: false, acro_v: false,
+  casual: false, cas_top: 300000, cas_n1: 6, cas_n2: 4, abbr: false, squeeze: 0, k1gate: 99, k2gate: 99 };
+const parseTSV = (txt) => { const m = new Map(); for (const l of txt.split("\n")) { const i = l.indexOf("\t"); if (i > 0) m.set(l.slice(0, i).trim(), l.slice(i + 1).trim()); } return m; };
+const now = () => (typeof performance !== "undefined" ? performance.now() : Date.now());
+
+// Sizes are checked against the manifest, never against Content-Length (a CDN redirect or an auth error page has its
+// own, unrelated length); a wrong-size body (e.g. an HTML/plain-text error page served with 200) is rejected.
+const checkSize = (b, expect, url) => { if (expect && b.length !== expect) throw new Error(`fetch ${url}: got ${b.length} bytes, manifest says ${expect}`); return b; };
+async function fetchBytes(base, name, fetchImpl, expect = 0) {
+  const url = base.endsWith("/") ? base + name : base + "/" + name;
+  const r = await fetchImpl(url); if (!r.ok) throw new Error(`fetch ${url}: ${r.status}`);
+  if (name.endsWith(".gz") && typeof DecompressionStream !== "undefined") return new Uint8Array(await new Response(r.body.pipeThrough(new DecompressionStream("gzip"))).arrayBuffer());
+  return checkSize(new Uint8Array(await r.arrayBuffer()), expect, url);
+}
+
+// ------------------------------------------------------------------ consonant-skeleton index (bhc/skel.py; tables generated from it)
+const SK_DEV = {"\u0915": "K", "\u0916": "K", "\u0917": "G", "\u0918": "G", "\u091a": "C", "\u091b": "C", "\u091c": "J", "\u091d": "J", "\u091f": "T", "\u0920": "T", "\u0924": "T", "\u0925": "T", "\u0921": "D", "\u0922": "D", "\u0926": "D", "\u0927": "D", "\u0919": "N", "\u091e": "N", "\u0923": "N", "\u0928": "N", "\u0929": "N", "\u092e": "N", "\u0902": "N", "\u0901": "N", "\u092a": "P", "\u092b": "P", "\u092c": "B", "\u092d": "B", "\u0930": "R", "\u0931": "R", "\u090b": "R", "\u0943": "R", "\u0932": "L", "\u0933": "L", "\u0935": "V", "\u0936": "S", "\u0937": "S", "\u0938": "S", "\u0939": "H", "\u0903": "H"};
+const SK_ROM1 = {"k": "K", "q": "K", "g": "G", "j": "J", "z": "J", "t": "T", "d": "D", "n": "N", "m": "N", "p": "P", "f": "P", "b": "B", "r": "R", "l": "L", "v": "V", "w": "V", "s": "S", "h": "H"};
+const SK_ASP = "bcdgjkpstz";
+const DR_C = {"\u0915": "k", "\u0916": "kh", "\u0917": "g", "\u0918": "gh", "\u0919": "n", "\u091a": "ch", "\u091b": "chh", "\u091c": "j", "\u091d": "jh", "\u091e": "n", "\u091f": "t", "\u0920": "th", "\u0921": "d", "\u0922": "dh", "\u0923": "n", "\u0924": "t", "\u0925": "th", "\u0926": "d", "\u0927": "dh", "\u0928": "n", "\u092a": "p", "\u092b": "ph", "\u092c": "b", "\u092d": "bh", "\u092e": "m", "\u092f": "y", "\u0930": "r", "\u0932": "l", "\u0935": "v", "\u0936": "sh", "\u0937": "sh", "\u0938": "s", "\u0939": "h", "\u0933": "l"};
+const DR_V = {"\u0905": "a", "\u0906": "aa", "\u0907": "i", "\u0908": "ee", "\u0909": "u", "\u090a": "oo", "\u090b": "ri", "\u090f": "e", "\u0910": "ai", "\u0913": "o", "\u0914": "au", "\u0911": "o"};
+const DR_M = {"\u093e": "a", "\u093f": "i", "\u0940": "ee", "\u0941": "u", "\u0942": "oo", "\u0943": "ri", "\u0947": "e", "\u0948": "ai", "\u094b": "o", "\u094c": "au", "\u0949": "o", "\u0945": "e"};
+const NUKTA_ROM = { k: "q", kh: "kh", g: "g", j: "z", ph: "f", d: "r", dh: "rh" };
+export function devSkel(w) { let o = ""; for (const ch of w) { const c = SK_DEV[ch]; if (c && o[o.length - 1] !== c) o += c; } return o; }
+export function romSkels(w, maxk = 4) {
+  w = w.toLowerCase(); let outs = [""]; let i = 0; const n = w.length;
+  while (i < n) {
+    const ch = w[i]; let step = 1, opts;
+    if ((ch === "t" || ch === "s") && w.startsWith("ion", i + 1)) { opts = ch === "t" ? ["SN", "TN"] : ["SN"]; step = 4; }
+    else if (ch === "c") {
+      const nx = i + 1 < n ? w[i + 1] : "";
+      if (nx === "h") { opts = ["C", "K"]; step = 2; while (i + step < n && (w[i + step] === "c" || w[i + step] === "h")) step++; }
+      else if (nx === "e" || nx === "i" || nx === "y") opts = ["S", "K"];
+      else if (nx === "k") { opts = ["K"]; step = 2; }
+      else opts = ["K", "S"];
+    } else if (ch === "x") opts = ["KS"];
+    else if (SK_ASP.includes(ch) && i + 1 < n && w[i + 1] === "h") { opts = [SK_ROM1[ch] || ""]; step = 2; while (i + step < n && w[i + step] === "h") step++; }
+    else opts = [SK_ROM1[ch] || ""];
+    const nw = [];
+    for (const o of outs) for (const q of opts) { let s = o; for (const c of q) if (!s || s[s.length - 1] !== c) s += c; nw.push(s); }
+    outs = [...new Set(nw)].slice(0, maxk); i += step;
+  }
+  return outs;
+}
+export function devRom(w) {
+  const out = [], a = [...w], L = a.length;
+  for (let j = 0; j < L; j++) {
+    const ch = a[j];
+    if (DR_C[ch] !== undefined) {
+      out.push(DR_C[ch]); let nx = j + 1 < L ? a[j + 1] : "";
+      if (nx === "\u093c") nx = j + 2 < L ? a[j + 2] : "";
+      if (nx && DR_M[nx] === undefined && nx !== "\u094d" && j + 1 < L - 1) out.push("a");
+    } else if (DR_V[ch] !== undefined) out.push(DR_V[ch]);
+    else if (DR_M[ch] !== undefined) out.push(DR_M[ch]);
+    else if (ch === "\u0902" || ch === "\u0901") out.push("n");
+    else if (ch === "\u0903") out.push("h");
+    else if (ch === "\u093c" && out.length) { const l = out[out.length - 1]; out[out.length - 1] = NUKTA_ROM[l] ?? l; }
+  }
+  return out.join("");
+}
+export function lev(a, b) {
+  if (a.length < b.length) [a, b] = [b, a];
+  let prev = Array.from({ length: b.length + 1 }, (_, j) => j);
+  for (let i = 1; i <= a.length; i++) {
+    const cur = [i];
+    for (let j = 1; j <= b.length; j++) cur.push(Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (a[i - 1] !== b[j - 1] ? 1 : 0)));
+    prev = cur;
+  }
+  return prev[b.length];
+}
+class SkelIndex {
+  constructor(lm) {
+    const b = new Map(), N = lm.words.length;
+    for (let i = 0; i < N; i++) { const k = devSkel(lm.words[i]); if (k) { let v = b.get(k); if (!v) b.set(k, (v = [])); v.push(i); } }
+    for (const v of b.values()) v.sort((x, y) => (lm.uni[y] - lm.uni[x]) || (x - y));
+    this.b = b;
+  }
+  lookup(roman, topk = 24) {
+    const seen = [], S = new Set();
+    for (const k of romSkels(roman)) { const v = this.b.get(k); if (!v) continue; for (const i of v.slice(0, topk)) if (!S.has(i)) { S.add(i); seen.push(i); } }
+    return seen;
+  }
+}
+// ------------------------------------------------------------------ casual / chat-romanization keys (bhc/casual.py; keep 1:1)
+const CAS_DC = {"\u0915": "k", "\u0916": "k", "\u0917": "g", "\u0918": "g", "\u091a": "c", "\u091b": "c", "\u091c": "j", "\u091d": "j", "\u091f": "t", "\u0920": "t", "\u0924": "t", "\u0925": "t", "\u0921": "d", "\u0922": "d", "\u0926": "d", "\u0927": "d", "\u0923": "n", "\u0928": "n", "\u0929": "n", "\u092a": "p", "\u092b": "p", "\u092c": "b", "\u092d": "b", "\u092e": "m", "\u092f": "y", "\u0930": "r", "\u0931": "r", "\u0932": "l", "\u0933": "l", "\u0935": "v", "\u0936": "s", "\u0937": "s", "\u0938": "s", "\u0939": "h", "\u0958": "k", "\u0959": "k", "\u095a": "g", "\u095b": "j", "\u095c": "R", "\u095d": "R", "\u095e": "p", "\u095f": "y"};
+const CAS_DV = {"\u0905": "a", "\u0906": "a", "\u093e": "a", "\u0907": "i", "\u0908": "i", "\u093f": "i", "\u0940": "i", "\u0909": "u", "\u090a": "u", "\u0941": "u", "\u0942": "u", "\u090f": "e", "\u0910": "e", "\u0947": "e", "\u0948": "e", "\u0945": "e", "\u090d": "e", "\u0913": "o", "\u0914": "o", "\u0911": "o", "\u094b": "o", "\u094c": "o", "\u0949": "o", "\u090b": "r", "\u0943": "r"};
+export const ABBR = {"h": ["\u0939\u0948", "\u0939\u0948\u0902"], "hy": ["\u0939\u0948"], "hn": ["\u0939\u093e\u0901", "\u0939\u0948\u0902"], "hm": ["\u0939\u092e", "\u0939\u092e\u094d\u092e"], "k": ["\u0915\u0947", "\u0915\u0940", "\u0913\u0915\u0947"], "ki": ["\u0915\u0940", "\u0915\u093f"], "n": ["\u0928\u093e", "\u0928"], "ni": ["\u0928\u0939\u0940\u0902", "\u0928\u0940"], "nai": ["\u0928\u0939\u0940\u0902", "\u0928\u0908"], "nhi": ["\u0928\u0939\u0940\u0902"], "nhe": ["\u0928\u0939\u0940\u0902"], "nahi": ["\u0928\u0939\u0940\u0902"], "kr": ["\u0915\u0930"], "krna": ["\u0915\u0930\u0928\u093e"], "kro": ["\u0915\u0930\u094b"], "tm": ["\u0924\u0941\u092e"], "tu": ["\u0924\u0942"], "mai": ["\u092e\u0948\u0902", "\u092e\u0947\u0902"], "me": ["\u092e\u0947\u0902", "\u092e\u0948\u0902"], "m": ["\u092e\u0948\u0902", "\u092e\u0947\u0902"], "thk": ["\u0920\u0940\u0915"], "tk": ["\u0920\u0940\u0915", "\u0924\u0915"], "yr": ["\u092f\u093e\u0930"], "yrr": ["\u092f\u093e\u0930"], "plz": ["\u092a\u094d\u0932\u0940\u091c\u093c"], "pls": ["\u092a\u094d\u0932\u0940\u091c\u093c"], "plzz": ["\u092a\u094d\u0932\u0940\u091c\u093c"], "tq": ["\u0925\u0948\u0902\u0915 \u092f\u0942"], "ty": ["\u0925\u0948\u0902\u0915 \u092f\u0942"], "thx": ["\u0925\u0948\u0902\u0915\u094d\u0938"], "thnx": ["\u0925\u0948\u0902\u0915\u094d\u0938"], "thnks": ["\u0925\u0948\u0902\u0915\u094d\u0938"], "gn": ["\u0917\u0941\u0921 \u0928\u093e\u0907\u091f"], "gm": ["\u0917\u0941\u0921 \u092e\u0949\u0930\u094d\u0928\u093f\u0902\u0917"], "ok": ["\u0913\u0915\u0947"], "okk": ["\u0913\u0915\u0947"], "kk": ["\u0913\u0915\u0947"], "sry": ["\u0938\u0949\u0930\u0940"], "bro": ["\u092c\u094d\u0930\u094b"], "msg": ["\u092e\u0948\u0938\u0947\u091c"], "bday": ["\u092c\u0930\u094d\u0925\u0921\u0947"], "clg": ["\u0915\u0949\u0932\u0947\u091c"], "ofc": ["\u0911\u092b\u093c\u093f\u0938"], "pic": ["\u092a\u093f\u0915"], "bhai": ["\u092d\u093e\u0908"], "bht": ["\u092c\u0939\u0941\u0924"], "bhot": ["\u092c\u0939\u0941\u0924"], "bohot": ["\u092c\u0939\u0941\u0924"], "pta": ["\u092a\u0924\u093e"], "kb": ["\u0915\u092c"], "jb": ["\u091c\u092c"], "sb": ["\u0938\u092c"], "mtlb": ["\u092e\u0924\u0932\u092c"], "kch": ["\u0915\u0941\u091b"], "kuch": ["\u0915\u0941\u091b"]};
+const CAS_VOW = new Set(["a", "e", "i", "o", "u"]), CAS_ASP = new Set([..."kgjtdpbc"]);
+function casFin(s) { let o = ""; for (let i = 0; i < s.length; i++) { const ch = s[i]; if (ch === "a" && i > 0) continue; if (o && o[o.length - 1] === ch) continue; o += ch; } return o; }
+export function k2Of(k1) {
+  if (!k1) return "";
+  let s = k1[0]; for (const c of k1.slice(1)) if (!CAS_VOW.has(c)) s += c;
+  if (s && CAS_VOW.has(s[0])) s = s.slice(1) || s;
+  let o = ""; for (const ch of s) if (!(o && o[o.length - 1] === ch)) o += ch; return o;
+}
+export function devKeys(w) {
+  const a = [...w], raw = [];
+  for (let i = 0; i < a.length; i++) {
+    const ch = a[i];
+    if (ch === "\u093c") { if (raw.length && raw[raw.length - 1] === "d" && i > 0 && (a[i - 1] === "\u0921" || a[i - 1] === "\u0922")) raw[raw.length - 1] = "R"; continue; }
+    if (CAS_DC[ch] !== undefined) raw.push(CAS_DC[ch]); else if (CAS_DV[ch] !== undefined) raw.push(CAS_DV[ch]);
+  }
+  const s = raw.join(""); if (!s) return [];
+  const ks = [...new Set([casFin(s.split("R").join("d")), casFin(s.split("R").join("r"))])].filter((k) => k);
+  return ks.sort();
+}
+export function romKeys(t, cap = 12) {
+  t = t.toLowerCase().replace(/[^a-z]/g, ""); if (!t) return [[], []];
+  t = t.replace(/(.)\1{2,}/g, "$1");
+  t = t.split("chh").join("c").split("cch").join("c").split("ch").join("c").split("sh").join("s");
+  t = t.split("x").join("ks").split("q").join("k").split("z").join("j").split("f").join("p").split("w").join("v");
+  const alts = [], L = t.length, V = (c) => CAS_VOW.has(c); let i = 0;
+  while (i < L) {
+    const c = t[i], nx = i + 1 < L ? t[i + 1] : "", prev = i > 0 ? t[i - 1] : "";
+    if (c === "h" && CAS_ASP.has(prev) && i > 0) { alts.push(["", "h"]); i += 1; continue; }
+    if (nx && (c === "a" || c === "e") && (nx === "i" || nx === "y") && (i + 2 >= L || !V(t[i + 2]))) {
+      if (c === "a" && nx === "i") { alts.push(["e", "ai"]); i += 2; continue; }
+      if (c === "a" && nx === "y" && i + 2 >= L) { alts.push(["e"]); i += 2; continue; }
+      if (c === "e" && nx === "y" && i + 2 >= L) { alts.push(["e"]); i += 2; continue; }
+    }
+    if (nx && (c === "a" || c === "o") && "uvw".includes(nx) && (i + 2 >= L || !V(t[i + 2])) && c + nx !== "av") { alts.push(["o", c + "u"]); i += 2; continue; }
+    if (c === "o" && nx === "u") { alts.push(["o", "u"]); i += 2; continue; }
+    if (c === "e" && nx === "e") { alts.push(["i"]); i += 2; continue; }
+    if (c === "o" && nx === "o") { alts.push(["u", "o"]); i += 2; continue; }
+    if (c === "e" && nx === "i" && (i + 2 >= L || !V(t[i + 2]))) { alts.push(["e"]); i += 2; continue; }
+    if (c === "e" && nx === "h" && i + 2 < L && !V(t[i + 2]) && i > 0) { alts.push(["e", ""]); i += 1; continue; }
+    if (nx && c === "i" && "aeou".includes(nx) && i > 0) { alts.push(["y", "i"]); i += 1; continue; }
+    if (c === "y" && i === L - 1 && prev && !V(prev)) { alts.push(["e", "i"]); i += 1; continue; }
+    if (c === "n" && V(prev) && i > 0 && (i === L - 1 || (nx && !V(nx) && nx !== "y"))) { alts.push(["n", ""]); i += 1; continue; }
+    if ((c === "i" || c === "u") && prev === "r" && i >= 2 && !V(t[i - 2])) { alts.push([c, ""]); i += 1; continue; }
+    alts.push([c]); i += 1;
+  }
+  let outs = [""];
+  for (const a of alts) { const nxt = []; for (const o of outs) for (const x of a) nxt.push(o + x); outs = [...new Set(nxt)].slice(0, 64); }
+  const k1 = [], seen = new Set();
+  for (const o of outs) { const k = casFin(o); if (k && !seen.has(k)) { seen.add(k); k1.push(k); } }
+  const k1c = k1.slice(0, cap); const k2 = [...new Set(k1c.map(k2Of).filter((k) => k))];
+  return [k1c, k2];
+}
+export function squeezeIn(t, mode = 0) {
+  if (!mode) return t;
+  if (mode === 1) return t.replace(/([a-z])\1{2,}/g, "$1");
+  return t.replace(/([a-z])\1{2,}/g, (m, c) => (/[aeo]/.test(c) ? c + c : c));
+}
+class CasualIndex {
+  constructor(lm, topN = 300000, perKey = 8) {
+    const N = lm.words.length, uni = lm.uni, order = Array.from({ length: N }, (_, i) => i);
+    order.sort((x, y) => (uni[y] - uni[x]) || (x - y));
+    this.k1 = new Map(); this.k2 = new Map();
+    const put = (m, k, i) => { let b = m.get(k); if (!b) m.set(k, (b = [])); if (b.length < perKey && !b.includes(i)) b.push(i); };
+    for (let r = 0; r < Math.min(N, topN); r++) { const i = order[r]; for (const k of devKeys(lm.words[i])) { put(this.k1, k, i); const k2 = k2Of(k); if (k2) put(this.k2, k2, i); } }
+  }
+  lookup(roman, n1 = 6, n2 = 4) {
+    const [k1, k2] = romKeys(roman), r1 = [], r2 = [];
+    for (const k of k1) for (const i of this.k1.get(k) || []) if (!r1.includes(i)) r1.push(i);
+    for (const k of k2) for (const i of this.k2.get(k) || []) if (!r2.includes(i)) r2.push(i);
+    return [r1.slice(0, n1 * 2), r2.slice(0, n2 * 2)];
+  }
+}
+const VIS_RE = /[aeiou]h$/;
+
+// ------------------------------------------------------------------ the engine
+export class BhashaCore {
+  /**
+   * Load a bundle (directory with manifest.json) from a URL (browser / worker) or via a custom fetch (Node: file loader).
+   * dict: true (default) = also load the dictionary pack and wait for it; false = core only (call loadDict() later).
+   * cache: Cache API cache name for the dictionary pack (browser / worker; default "aksharalp-dict"), or false.
+   */
+  static async load(base, { fetch: fetchImpl = globalThis.fetch, onProgress, dict = true, cache = "aksharalp-dict" } = {}) {
+    const t0 = now(); const txt = (b) => new TextDecoder().decode(b);
+    let man; try { man = JSON.parse(txt(await fetchBytes(base, "manifest.json", fetchImpl))); } catch { man = { files: { model: "xlit.q4", lex: "lex.bin", lm: "lm.bin", params: "params.json", lid: "lid.int8", loan: "loan.tsv", natural: "natural_hi.tsv" } }; }
+    const F = man.files, SZ = man.sizes || {}, get = async (n, opt) => { if (!n) return null; try { const b = await fetchBytes(base, n, fetchImpl, SZ[n] && SZ[n].bytes); onProgress && onProgress(n); return b; } catch (e) { if (opt) return null; throw e; } };
+    const [mb, lexb, lmb, pb, lidb, loanb, natb] = await Promise.all([get(F.model), get(F.lex, true), get(F.lm, true), get(F.params, true), get(F.lid, true), get(F.loan, true), get(F.natural, true)]);
+    const e = new BhashaCore({ model: mb, lex: lexb, lm: lmb, params: pb ? JSON.parse(txt(pb)) : {}, lid: lidb, loan: loanb ? txt(loanb) : null, natural: natb ? txt(natb) : null, manifest: man });
+    e.base = base; e.fetchImpl = fetchImpl;
+    if (dict && man.packs && man.packs.dict) await e.loadDict({ onProgress, cache });
+    e.loadMs = now() - t0; return e;
+  }
+  /** Fetch (or read from cache) the dictionary pack and attach it. Returns true on success; on any failure the engine
+   *  keeps working in core-only mode and the error is kept in this.dictError. */
+  async loadDict({ onProgress, cache = "aksharalp-dict" } = {}) {
+    const pk = this.manifest && this.manifest.packs && this.manifest.packs.dict; if (!pk) return false;
+    const dir = pk.dir.endsWith("/") ? pk.dir : pk.dir + "/", fetchImpl = this.fetchImpl || globalThis.fetch;
+    const useCache = cache && typeof caches !== "undefined";
+    const get = async (n) => {
+      if (!n) return null;
+      const ver = (pk.sizes && pk.sizes[n] && pk.sizes[n].sha256) || "", expect = (pk.sizes && pk.sizes[n] && pk.sizes[n].bytes) || 0;
+      if (useCache) {
+        try {
+          const url = new URL(dir + n, new URL(this.base.endsWith("/") ? this.base : this.base + "/", globalThis.location && globalThis.location.href)).href;
+          const c = await caches.open(cache), key = url + "?v=" + ver.slice(0, 16), hit = await c.match(key);
+          if (hit) { const hb = new Uint8Array(await hit.arrayBuffer()); if (!expect || hb.length === expect) { onProgress && onProgress(n, "cache"); return hb; } await c.delete(key); }
+          const r = await fetchImpl(url); if (!r.ok) throw new Error(`fetch ${url}: ${r.status}`);
+          const b = checkSize(new Uint8Array(await r.arrayBuffer()), expect, url); try { await c.put(key, new Response(b)); } catch (_) {}   // stored as a fresh, non-redirected response
+          onProgress && onProgress(n, "network"); return b;
+        } catch (e) { if (String(e).includes("fetch ")) throw e; }
+      }
+      const b = await fetchBytes(this.base, dir + n, fetchImpl, expect); onProgress && onProgress(n, "network"); return b;
+    };
+    try {
+      const [lexb, lmb, pb] = await Promise.all([get(pk.files.lex), get(pk.files.lm), get(pk.files.params)]);
+      this.attachDict({ lex: lexb, lm: lmb, params: pb ? JSON.parse(new TextDecoder().decode(pb)) : {} });
+      this.dictInfo = { name: pk.name || "dict", words: this.lm.N, bytes: pk.bytes, gz: pk.gz }; return true;
+    } catch (e) { this.dictError = String(e && e.message || e); return false; }
+  }
+  /** attach an already-fetched dictionary pack: {lex, lm, params} (params override the core params) */
+  attachDict({ lex, lm = null, params = {} }) {
+    this.p = { ...DEFAULTS, ...this.coreParams, ...params };
+    this._setLex(lex, lm); this.cache = new Map(); this.hasDict = this.lm.N > 0;
+  }
+  _setLex(lex, lm) {
+    this.lm = new LexLM(lex, lm && this.p.use_lm ? lm : null);
+    if (!this.lm.N) { this.p.cbeam = 0; this.p.skel = false; }
+    this.cix = this.p.casual && this.lm.N ? new CasualIndex(this.lm, this.p.cas_top) : null;
+    // the trie never contains the empty string (as in Python's LazyTrie), otherwise "" would take a constrained-beam slot
+    const sw = this.lm.sortedWords, tw = sw.length && sw[0] === "" ? sw.slice(1) : sw;
+    this.trie = this.p.cbeam ? new LazyTrie(tw) : null;
+    this.skix = this.p.skel ? new SkelIndex(this.lm) : null;
+  }
+  constructor({ model, lex = null, lm = null, params = {}, lid = null, loan = null, natural = null, manifest = null }) {
+    this.coreParams = params; this.p = { ...DEFAULTS, ...params }; this.manifest = manifest;
+    this.model = new XlitModel(parseWeights(model));
+    this._setLex(lex, lm); this.hasDict = this.lm.N > 0; this.dictInfo = null; this.dictError = null;
+    this.lid = lid ? new LID(parseWeights(lid)) : null;
+    this.loan = loan ? parseTSV(loan) : new Map();
+    this.natural = natural ? parseTSV(natural) : new Map();
+    this.cache = new Map();
+  }
+  info() {
+    const s = this.manifest && this.manifest.sizes ? this.manifest.sizes : {};
+    const used = Object.values(this.manifest ? this.manifest.files : {}).filter((f) => f && s[f]);
+    return { params: this.model.nParams + (this.lid ? this.lid.nParams : 0), xlitParams: this.model.nParams, lidParams: this.lid ? this.lid.nParams : 0,
+      lexWords: this.lm.N, bytes: used.reduce((a, f) => a + s[f].bytes, 0), gz: used.reduce((a, f) => a + s[f].gz, 0), hasLID: !!this.lid, hasLM: this.lm.hasLM,
+      hasDict: this.hasDict, dict: this.dictInfo, dictError: this.dictError, hasLoan: this.loan.size > 0, hasNatural: this.natural.size > 0 };
+  }
+
+  /** candidate rows with reranker features for one (lower-cased) Latin word: [[devanagari, fv, wid], ...] */
+  candFeats(word) {
+    const p = this.p, w = squeezeIn(word.toLowerCase(), p.squeeze); const hit = this.cache.get(w); if (hit) return hit;
+    const m = this.model, ids = encSrc(w), enc = m.encode(ids), S = ids.length;
+    const free = m.beam(enc, S, { beam: p.beam, nbest: p.nbest });
+    const con = p.cbeam && this.trie ? m.beam(enc, S, { beam: p.beam, nbest: p.nbest, trie: this.trie }) : [];
+    const d = new Map();
+    free.forEach(([s, sc], i) => { if (s) d.set(s, { sc, infree: 1, frank: i, free0: +(i === 0), beamEntry: 1 }); });
+    for (const [s, sc] of con) if (s) { if (!d.has(s)) d.set(s, { sc, infree: 0, frank: p.nbest, free0: 0, beamEntry: 1 }); d.get(s).incon = 1; }
+    if (p.variants) {
+      const top = [...d.keys()].sort((a, b) => d.get(b).sc - d.get(a).sc).slice(0, 4);
+      for (const s of top) for (const [a, b] of VAR_SWAPS) if (s.includes(a)) {
+        const v = s.split(a).join(b); if (!d.has(v) && this.lm.id(v) !== null) d.set(v, { sc: m.scorePair(enc, v), infree: 0, frank: p.nbest, free0: 0, variant: 1 });
+      }
+    }
+    if (p.acro && (RE_ACRO.test(w) || (p.acro_v && w.length >= 2 && w.length <= 4))) {
+      const v = [...w].map((c) => LETTER[c]).join("");
+      if (!d.has(v)) d.set(v, { sc: m.scorePair(enc, v), infree: 0, frank: p.nbest, free0: 0, acro: 1 }); else d.get(v).acro = 1;
+    }
+    if (p.visr && VIS_RE.test(w)) {                    // Aksharantar convention: literal word-final h -> visarga
+      const top = [...d.keys()].filter((x) => d.get(x).beamEntry).sort((x, y) => d.get(y).sc - d.get(x).sc).slice(0, 2);
+      for (const s of top) {
+        const v = s.endsWith("\u0939") ? s.slice(0, -1) + "\u0903" : (!s.endsWith("\u0903") ? s + "\u0903" : null);
+        if (v && !d.has(v)) d.set(v, { sc: m.scorePair(enc, v), infree: 0, frank: p.nbest, free0: 0, visr: 1 });
+      }
+    }
+    if (this.skix) {
+      const L = w.length + 1, words = this.lm.words, uni = this.lm.uni;
+      const ids = this.skix.lookup(w, p.skel_k).map((i) => [i, lev(w, devRom(words[i])) / L - p.skel_wu * uni[i]])
+        .sort((x, y) => x[1] - y[1]).slice(0, p.skel_m);
+      for (const [i] of ids) { const v = words[i]; if (!d.has(v)) d.set(v, { sc: m.scorePair(enc, v), infree: 0, frank: p.nbest, free0: 0, skel: 1 }); else d.get(v).skel = 1; }
+    }
+    if (this.cix) {
+      const [r1, r2] = this.cix.lookup(w, p.cas_n1, p.cas_n2), words = this.lm.words, nv = (w.match(/[aeiou]/g) || []).length;
+      const ab = new Set(p.abbr && Object.prototype.hasOwnProperty.call(ABBR, w) ? ABBR[w] : []), s1 = new Set(r1.slice(0, p.cas_n1).map((i) => words[i]));
+      for (const [key, ids, n] of [["ck1", r1, p.cas_n1], ["ck2", r2, p.cas_n2]]) ids.slice(0, n).forEach((i, rk) => {
+        const v = words[i];
+        if (!d.has(v)) {
+          if (!ab.has(v) && ((s1.has(v) && nv > p.k1gate) || (!s1.has(v) && nv > p.k2gate))) return;   // vowel gate
+          d.set(v, { sc: m.scorePair(enc, v), infree: 0, frank: p.nbest, free0: 0 });
+        }
+        const e = d.get(v); e[key] = 1; e.ckr = Math.min(e.ckr ?? 1, rk / n);
+      });
+    }
+    if (p.abbr && Object.prototype.hasOwnProperty.call(ABBR, w)) ABBR[w].forEach((v, rk) => {
+      if (!d.has(v)) d.set(v, { sc: m.scorePair(enc, v), infree: 0, frank: p.nbest, free0: 0 });
+      const e = d.get(v); e.abbr = 1; e.abbr_r = rk;
+    });
+    const rows = [], sks = new Set(romSkels(w)), Lw = w.length + 1, vis = VIS_RE.test(w), novow = !/[aeiou]/.test(w);
+    for (const [s, e] of d) {
+      const wid = this.lm.id(s), L = [...s].length;
+      rows.push([s, [e.sc, e.sc / (L + 1), +(wid !== null), wid !== null ? this.lm.unigram(wid) : 0, e.free0 || 0, e.infree || 0, e.incon || 0,
+        +(s.includes("\u0949") || s.includes("\u0911")), +s.includes("\u0903"), +s.includes("\u093c"), +s.includes("\u0901"), e.acro || 0, e.variant || 0, (e.frank ?? p.nbest) / p.nbest,
+        e.skel || 0, +sks.has(devSkel(s)), lev(w, devRom(s)) / Lw, 0, 0, 0, L / Lw, e.visr || 0, +(vis && s.endsWith("\u0903")), +(!!e.acro && novow),
+        e.ck1 || 0, e.ck2 || 0, e.ckr ?? 1, e.abbr || 0, e.abbr_r ?? 3, +!/[aeiouy]/.test(w)], wid]);
+    }
+    if (rows.length) {
+      let msc = -Infinity, mscn = -Infinity, mu = null;
+      for (const [, fv] of rows) { msc = Math.max(msc, fv[0]); mscn = Math.max(mscn, fv[1]); if (fv[2]) mu = mu === null ? fv[3] : Math.max(mu, fv[3]); }
+      for (const [, fv] of rows) { fv[17] = fv[0] - msc; fv[18] = fv[1] - mscn; fv[19] = fv[2] ? fv[3] - (mu ?? 0) : 0; }
+    }
+    if (this.cache.size > 20000) this.cache.clear();
+    this.cache.set(w, rows); return rows;
+  }
+  score(fv) {
+    const p = this.p; if (!p.w && !p.w1) return fv[0] + 0.2 * (fv[2] ? fv[3] : -25.0);
+    const n = p.mu.length, z = new Array(n); for (let i = 0; i < n; i++) z[i] = (fv[i] - p.mu[i]) / p.sd[i];
+    if (p.w1) {                                      // 1-hidden-layer tanh MLP reranker (fit on VALID only)
+      let s = p.b2 || 0;
+      for (let h = 0; h < p.w1.length; h++) { const r = p.w1[h]; let a = p.b1[h]; for (let i = 0; i < n; i++) a += r[i] * z[i]; s += p.w2[h] * Math.tanh(a); }
+      return s;
+    }
+    let s = 0; for (let i = 0; i < Math.min(n, p.w.length); i++) s += p.w[i] * z[i]; return s;
+  }
+  /** word mode: top-k Devanagari candidates for a romanized word */
+  word(w, k = 8) {
+    if (!RE_LATIN.test(w)) return [{ text: w, score: 0 }];
+    const rows = this.candFeats(w).map(([s, fv]) => ({ text: s, score: this.score(fv) }));
+    return rows.sort((a, b) => b.score - a.score).slice(0, k);
+  }
+  /** sentence Viterbi over candidate lattices with the bigram LM (bhc/translit2.py decode_slots) */
+  decodeSlots(slots, context = true) {
+    const p = this.p, lm = this.lm, lat = [];
+    for (const [kind, t] of slots) {
+      if (kind === "x") {
+        let sc = this.candFeats(t).map(([s, fv, wid]) => [s, this.score(fv), wid]);
+        if (sc.length) {
+          const mx = Math.max(...sc.map((x) => x[1])), lse = mx + Math.log(sc.reduce((a, x) => a + Math.exp(x[1] - mx), 0));
+          sc = sc.map(([s, v, wid]) => [s, p.w_ch_s * (v - lse), wid]).sort((a, b) => b[1] - a[1]).slice(0, 12);
+        }
+        lat.push(sc.length ? sc : [[t, 0, null]]);
+      } else {
+        const wid = RE_DEVA.test(t) ? lm.id(t) ?? -1 : -2; lat.push([[t, 0, wid]]);
+      }
+    }
+    const post = lat.map((o) => o.map(([s, v]) => [s, v]));      // per-slot candidate list (for UI chips)
+    if (!context || !p.use_lm || !lm.hasLM) return { out: lat.map((o) => o.reduce((a, b) => (b[1] > a[1] ? b : a))[0]), lat: post };
+    let beams = [[0, null, []]];
+    for (const opts of lat) {
+      if (opts[0][2] === -2 && opts.length === 1) { beams = beams.map(([s, h, ch]) => [s, h, ch.concat([opts[0][0]])]); continue; }
+      const best = new Map();
+      for (const [s, h, ch] of beams) for (const [cand, csc, wid] of opts) {
+        const w = wid === null || wid === -1 ? null : wid;
+        const tot = s + csc + p.w_lm * (lm.logp(h, w) - (w === null ? p.oov_pen_s : 0));
+        if (!best.has(cand) || tot > best.get(cand)[0]) best.set(cand, [tot, w, ch.concat([cand])]);
+      }
+      beams = [...best.values()].sort((a, b) => b[0] - a[0]).slice(0, 16);
+    }
+    let bb = null, bv = -Infinity;
+    for (const b of beams) { const v = b[0] + p.w_lm * lm.logp(b[1], lm.N); if (v > bv) { bv = v; bb = b; } }
+    return { out: bb[2], lat: post };
+  }
+  /** plain Track-B sentence conversion (bhc/translit2.py convert) */
+  convertPlain(text) {
+    const spans = [...text.matchAll(TOKEN_RE)].map((m) => [m.index, m.index + m[0].length, m[0]]);
+    const { out } = this.decodeSlots(spans.map(([, , t]) => (RE_LATIN.test(t) ? ["x", t] : ["fix", t])));
+    if (out.length && out[out.length - 1] === ".") out[out.length - 1] = "\u0964";
+    let s = "", pos = 0; spans.forEach(([a, b], i) => { s += text.slice(pos, a) + out[i]; pos = b; }); return s + text.slice(pos);
+  }
+
+  /**
+   * Full pipeline: tokenise -> language ID -> policy (keep / acronym / loanword / glossary) -> contextual decoding.
+   * @param {string} text
+   * @param {{mode?: 'conversational'|'natural'|'strict', k?: number, context?: boolean, overrides?: Object<number,string>}} opt
+   *        context: false = word-by-word (no sentence LM)
+   *        overrides: {tokenStartOffset: chosenDevanagari} pins a user-picked suggestion (re-decodes the rest in context)
+   * @returns {{output: string, tokens: Array<{text,start,end,output,lang,source,candidates:Array<{text,score}>}>, sentence: string|null, ms: number}}
+   */
+  transliterate(text, { mode = "conversational", k = 5, overrides = {}, context = true } = {}) {
+    const t0 = now(); const policy = mode === "strict" ? "strict" : "conversational";
+    const spans = [...text.matchAll(TOKEN_RE)].map((m) => [m.index, m.index + m[0].length, m[0]]);
+    const toks = spans.map((s) => s[2]);
+    const tags = this.lid ? this.lid.tag(toks) : toks.map((t) => (RE_DEVA.test(t) ? "DEV" : RE_LATIN_APOS.test(t) ? (this.loan.has(t.toLowerCase()) ? "EN" : "HI") : "OTHER"));
+    const slots = [], src = []; let prevBrand = false;
+    for (let i = 0; i < toks.length; i++) {
+      const t = toks[i], tag = tags[i], nxt = toks[i + 1] || "", low = t.toLowerCase(), ov = overrides[spans[i][0]];
+      if (ov !== undefined) { slots.push(["fix", ov]); src.push("user"); prevBrand = false; continue; }
+      if (!RE_LATIN_APOS.test(t) || KEEP_RE.test(t) || nxt === "(") {
+        if (t === "." && (i === toks.length - 1 || /^[A-Z]/.test(toks[i + 1] || ""))) slots.push(["fix", "\u0964"]); else slots.push(["fix", t]);
+        src.push(RE_DEVA.test(t) ? "devanagari" : KEEP_RE.test(t) ? "keep" : "punct"); prevBrand = false; continue;
+      }
+      const isBrand = BRANDS.has(low) || (prevBrand && /^[A-Z]/.test(t)); prevBrand = isBrand;
+      if (policy === "strict" && isBrand) { slots.push(["fix", t]); src.push("brand"); continue; }
+      if (mode === "natural" && tag === "EN" && this.natural.has(low)) { slots.push(["fix", this.natural.get(low)]); src.push("glossary"); continue; }
+      if (isUpper(t) && t.length >= 2 && t.length <= 5 && !this.loan.has(low)) { slots.push(["fix", [...low].map((c) => LETTERS_H[c]).join("")]); src.push("acronym"); continue; }
+      if ((tag === "EN" || isBrand) && this.loan.has(low)) { slots.push(["fix", this.loan.get(low)]); src.push("loanword"); continue; }
+      slots.push(["x", t]); src.push("model");
+    }
+    const { out, lat } = this.decodeSlots(slots, context);
+    let output = "", pos = 0; spans.forEach(([a, b], i) => { output += text.slice(pos, a) + out[i]; pos = b; }); output = (output + text.slice(pos)).trim();
+    const tokens = spans.map(([a, b, t], i) => {
+      let cands = [];
+      if (slots[i][0] === "x") {
+        const mx = lat[i][0] ? lat[i][0][1] : 0;
+        cands = lat[i].slice(0, k).map(([s, v]) => ({ text: s, score: Math.exp(v - mx) }));
+      } else if (RE_LATIN_APOS.test(t) && !KEEP_RE.test(t)) {
+        cands = [{ text: out[i], score: 1 }];
+        for (const c of this.word(t, k)) if (c.text !== out[i]) cands.push({ text: c.text, score: null });
+        if (out[i] !== t) cands.push({ text: t, score: null });
+        cands = cands.slice(0, k);
+      }
+      const isW = RE_LATIN_APOS.test(t) || RE_DEVA.test(t);
+      return { text: t, start: a, end: b, output: out[i], lang: isW ? tags[i] : "OTHER", source: src[i], candidates: cands };
+    });
+    let sentence = null; if (this.lid && toks.length) try { sentence = this.lid.mode(toks); } catch { sentence = null; }
+    return { output, tokens, sentence, ms: now() - t0 };
+  }
+}
+
+/** Node helper: load a bundle from a local directory. */
+export async function loadFromDir(dir, opts = {}) {
+  const fs = await import("node:fs/promises"), path = await import("node:path");
+  const f = async (u) => { try { const b = await fs.readFile(path.resolve(u)); return new Response(b); } catch { return new Response(null, { status: 404 }); } };
+  return BhashaCore.load(dir, { fetch: f, cache: false, ...opts });
+}
+export { BhashaCore as AksharaLP };
+export default BhashaCore;

config.json

@@ -0,0 +1,27 @@
+{
+  "name": "AksharaLP-MultiIndic-8M",
+  "version": "1.0.0",
+  "format": "aksharalp-multiindic/1",
+  "languages": [
+    "hi",
+    "bn",
+    "te",
+    "kn",
+    "ta"
+  ],
+  "parameters": {
+    "hi": 988928,
+    "bn": 989056,
+    "te": 988928,
+    "kn": 988928,
+    "ta": 988928,
+    "word_total": 4944768,
+    "hindi_sentence_lm": 2770752,
+    "total": 7715520
+  },
+  "sentence_languages": [
+    "hi"
+  ],
+  "license": "cc-by-nc-4.0",
+  "requires_bundled_decoder": true
+}

evaluation/bn_frozen.json

@@ -0,0 +1,74 @@
+{
+  "files": {
+    "xlit.q4": "9f099f0b9524f446debf1d3f0ae1dc7ed1e88625c33f6dbb7b45e0348bf76376",
+    "dict.tsv.gz": "edb2ad0bd7de4e87bd4d49ac406a9ef01011451bf23ac82d4c063bb9cce1dc2b",
+    "params.json": "e0bd564ad61f6dcef8eca6bf3d64f8b231146c9885aa4c5b282a2b2420606c4f"
+  },
+  "fit": "VALID only; frozen before test",
+  "config": {
+    "format": "aksharalp-multiindic/1",
+    "language": "bn",
+    "beam": 8,
+    "nbest": 8,
+    "model": "xlit.q4",
+    "dictionary": "dict.tsv.gz",
+    "dictionary_words": 500000,
+    "dictionary_total": 29467775,
+    "unk": -17.198807885289952,
+    "reranker": {
+      "type": "linear",
+      "features": [
+        "sc",
+        "scn",
+        "inlex",
+        "uni",
+        "free0",
+        "rank",
+        "sc_rel",
+        "uni_rel"
+      ],
+      "w": [
+        0.7719883322715759,
+        0.7239015102386475,
+        2.2432072162628174,
+        0.3994755744934082,
+        -0.1507813185453415,
+        -0.07777873426675797,
+        0.7293219566345215,
+        0.42816251516342163
+      ],
+      "mu": [
+        -5.078212233229818,
+        -0.8147631729600909,
+        0.31161147274689055,
+        -4.2673892678692775,
+        0.13265511803554703,
+        0.4336724409822265,
+        -3.7508561308261763,
+        -0.6986245748643944
+      ],
+      "sd": [
+        2.216960295304686,
+        0.6165648295196909,
+        0.46315298671618116,
+        6.490492456336254,
+        0.33920261805997143,
+        0.28614764292216904,
+        2.341120778815271,
+        1.748762448987244
+      ],
+      "fit_split": "all Aksharantar VALID",
+      "fit_n": 11188,
+      "l2": 0.001,
+      "seed": 0
+    }
+  },
+  "inventory": {
+    "parameters": 989056,
+    "model_raw": 589040,
+    "model_gz9": 540107,
+    "dict_raw": 14068340,
+    "dict_gz9": 2508610,
+    "dict_words": 500000
+  }
+}

evaluation/examples.json

@@ -0,0 +1,274 @@
+{
+  "hi": [
+    {
+      "input": "namaste",
+      "output": "नमस्ते",
+      "top8": [
+        "नमस्ते",
+        "नमसते",
+        "नाश्ते",
+        "नामस्ते",
+        "नमास्ते",
+        "नमस्तें",
+        "नास्ते",
+        "नम्स्ते"
+      ],
+      "ms": 405.55157300000064
+    },
+    {
+      "input": "kaise",
+      "output": "कैसे",
+      "top8": [
+        "कैसे",
+        "केसे",
+        "कइसे",
+        "कैस",
+        "कईसे",
+        "किसे",
+        "काइसे",
+        "कैशे"
+      ],
+      "ms": 146.41421700000137
+    },
+    {
+      "input": "dhanyavaad",
+      "output": "धन्यवाद",
+      "top8": [
+        "धन्यवाद",
+        "धन्यावाद",
+        "धन्यवाद्",
+        "धान्यवाद",
+        "धनयवाद",
+        "धन्यववाद",
+        "धन्यवादी",
+        "धन्येवाद"
+      ],
+      "ms": 234.02473099999952
+    },
+    {
+      "input": "mera naam rahul hai",
+      "output": "मेरा नाम राहुल है",
+      "top8": null,
+      "ms": 623.3726830000014
+    }
+  ],
+  "hi_sentence": [
+    {
+      "input": "main theek hoon",
+      "output": "मैं ठीक हूं"
+    },
+    {
+      "input": "ghar main kaun hai",
+      "output": "घर में कौन है"
+    }
+  ],
+  "bn": [
+    {
+      "input": "nomoskar",
+      "output": "নমস্কার",
+      "top8": [
+        "নমস্কার",
+        "নমষ্কার",
+        "নোমস্কার",
+        "নোমসকার",
+        "নমসকার",
+        "নমস্কর",
+        "নোমস্কর"
+      ],
+      "ms": 62.31514000000061
+    },
+    {
+      "input": "ami",
+      "output": "আমি",
+      "top8": [
+        "আমি",
+        "অমি",
+        "এমি",
+        "আমী",
+        "অামি",
+        "এএমই",
+        "এএম"
+      ],
+      "ms": 42.25174700000025
+    },
+    {
+      "input": "bhalo",
+      "output": "ভালো",
+      "top8": [
+        "ভালো",
+        "ভাল",
+        "ভালও",
+        "ভলো",
+        "বলো",
+        "ভল",
+        "ভা",
+        "ভলও"
+      ],
+      "ms": 36.55845999999838
+    },
+    {
+      "input": "tumi kemon acho",
+      "output": "তুমি কেমন আছো",
+      "top8": null,
+      "ms": 95.42537099999936
+    }
+  ],
+  "te": [
+    {
+      "input": "namaskaram",
+      "output": "నమస్కారం",
+      "top8": [
+        "నమస్కారం",
+        "నమస్కరం",
+        "నామస్కారం",
+        "నమస్కారమ్",
+        "నామస్కరం",
+        "నమాస్కారం",
+        "నమస్కారంా",
+        "నమస్కారంో"
+      ],
+      "ms": 57.50473300000158
+    },
+    {
+      "input": "nenu",
+      "output": "నేను",
+      "top8": [
+        "నేను",
+        "నెను",
+        "నేనూ",
+        "నను",
+        "నే",
+        "ఏను",
+        "నేన",
+        "నె"
+      ],
+      "ms": 27.787594999999783
+    },
+    {
+      "input": "manchi",
+      "output": "మంచి",
+      "top8": [
+        "మంచి",
+        "మాంచి",
+        "మంచీ",
+        "మన్చి",
+        "మం",
+        "మంఛి",
+        "మంచ"
+      ],
+      "ms": 34.4755350000014
+    },
+    {
+      "input": "meeru ela unnaru",
+      "output": "మీరు ఎలా ఉన్నారు",
+      "top8": null,
+      "ms": 103.91343400000005
+    }
+  ],
+  "kn": [
+    {
+      "input": "namaskara",
+      "output": "ನಮಸ್ಕಾರ",
+      "top8": [
+        "ನಮಸ್ಕಾರ",
+        "ನಮಸ್ಕರ",
+        "ನಮಸ್ಕರಾ",
+        "ನಮಾಸ್ಕರ",
+        "ನಮಸಕರ",
+        "ನಾಮಸ್ಕರ",
+        "ನಮಸಕಾರ",
+        "ನಮಸ್ಕಕರ"
+      ],
+      "ms": 46.95310299999983
+    },
+    {
+      "input": "nanu",
+      "output": "ನನು",
+      "top8": [
+        "ನನು",
+        "ನಂ",
+        "ನರು",
+        "ನಣು",
+        "ನಮು",
+        "ನಾನ",
+        "ಣನು",
+        "ನ"
+      ],
+      "ms": 27.71208200000001
+    },
+    {
+      "input": "chennagide",
+      "output": "ಚೆನ್ನಾಗಿದೆ",
+      "top8": [
+        "ಚೆನ್ನಾಗಿದೆ",
+        "ಚನ್ನಾಗಿದೆ",
+        "ಚೆನ್ನಗಿದೆ",
+        "ಚೆಣ್ಣಾಗಿದೆ",
+        "ಚೆನ್ನಾಗಿಡೆ",
+        "ಚೇನ್ನಾಗಿದೆ",
+        "ಚೆನ್ನಾಗಿದೇ",
+        "ಚೆನ್ನಾಗಿದಾ"
+      ],
+      "ms": 61.05912199999875
+    },
+    {
+      "input": "neevu hegiddira",
+      "output": "ನೀವು ಹೇಗಿದ್ದೀರಾ",
+      "top8": null,
+      "ms": 84.23852799999986
+    }
+  ],
+  "ta": [
+    {
+      "input": "vanakkam",
+      "output": "வணக்கம்",
+      "top8": [
+        "வணக்கம்",
+        "வனக்கம்",
+        "வண",
+        "வணாக்கம்",
+        "வாணக்கம்",
+        "வானக்கம்",
+        "வநக்கம்",
+        "வணக்கம"
+      ],
+      "ms": 61.89169000000038
+    },
+    {
+      "input": "naan",
+      "output": "ணன்",
+      "top8": [
+        "ணன்",
+        "னா",
+        "னன்",
+        "ணண்",
+        "ணண",
+        "ணன",
+        "ண",
+        "ன"
+      ],
+      "ms": 24.088208000001032
+    },
+    {
+      "input": "nandri",
+      "output": "நன்றி",
+      "top8": [
+        "நன்றி",
+        "நந்திரி",
+        "நந்த்ரி",
+        "நான்றி",
+        "னன்றி",
+        "நண்ட்ரி",
+        "நனத்ரி",
+        "ந"
+      ],
+      "ms": 41.495796000001064
+    },
+    {
+      "input": "neenga eppadi irukeenga",
+      "output": "நீங்க எப்படி இருகீங்க",
+      "top8": null,
+      "ms": 139.49065300000075
+    }
+  ]
+}

evaluation/gguf_independent_validation.json

@@ -0,0 +1,54 @@
+{
+  "aksharalp-hi-word-q4.gguf": {
+    "tensors": 1,
+    "metadata_keys": 18
+  },
+  "aksharalp-hi-sentence-q4.gguf": {
+    "tensors": 1,
+    "metadata_keys": 18
+  },
+  "aksharalp-ta-word-f16.gguf": {
+    "tensors": 94,
+    "metadata_keys": 16
+  },
+  "aksharalp-kn-word-q4.gguf": {
+    "tensors": 1,
+    "metadata_keys": 17
+  },
+  "aksharalp-bn-word-f16.gguf": {
+    "tensors": 94,
+    "metadata_keys": 16
+  },
+  "aksharalp-te-word-f16.gguf": {
+    "tensors": 94,
+    "metadata_keys": 16
+  },
+  "aksharalp-kn-word-f16.gguf": {
+    "tensors": 94,
+    "metadata_keys": 16
+  },
+  "aksharalp-ta-word-q4.gguf": {
+    "tensors": 1,
+    "metadata_keys": 17
+  },
+  "aksharalp-te-word-q4.gguf": {
+    "tensors": 1,
+    "metadata_keys": 17
+  },
+  "aksharalp-hi-sentence-f16.gguf": {
+    "tensors": 12,
+    "metadata_keys": 16
+  },
+  "aksharalp-bn-word-q4.gguf": {
+    "tensors": 1,
+    "metadata_keys": 17
+  },
+  "aksharalp-hi-word-f16.gguf": {
+    "tensors": 94,
+    "metadata_keys": 17
+  },
+  "aksharalp-hi-sentence-q8.gguf": {
+    "tensors": 1,
+    "metadata_keys": 18
+  }
+}

evaluation/gguf_parity.json

@@ -0,0 +1,262 @@
+{
+  "sample": "First 40 alphabetic ASCII roman words per full Aksharantar VALID; first 12 Dakshina VALID sentences with <=12 input words. No test used.",
+  "word": {
+    "hi": {
+      "max_abs_f16_weight_delta": 0.00091552734375,
+      "q4": {
+        "n": 40,
+        "same_top1": 40,
+        "same_top8_order": 40,
+        "same_top8_set": 40
+      },
+      "f16": {
+        "n": 40,
+        "same_top1": 40,
+        "same_top8_order": 40,
+        "same_top8_set": 40
+      }
+    },
+    "bn": {
+      "max_abs_f16_weight_delta": 0.0009765625,
+      "q4": {
+        "n": 40,
+        "same_top1": 40,
+        "same_top8_order": 40,
+        "same_top8_set": 40
+      },
+      "f16": {
+        "n": 40,
+        "same_top1": 40,
+        "same_top8_order": 40,
+        "same_top8_set": 40
+      }
+    },
+    "te": {
+      "max_abs_f16_weight_delta": 0.0009765625,
+      "q4": {
+        "n": 40,
+        "same_top1": 40,
+        "same_top8_order": 40,
+        "same_top8_set": 40
+      },
+      "f16": {
+        "n": 40,
+        "same_top1": 40,
+        "same_top8_order": 40,
+        "same_top8_set": 40
+      }
+    },
+    "kn": {
+      "max_abs_f16_weight_delta": 0.0009765625,
+      "q4": {
+        "n": 40,
+        "same_top1": 40,
+        "same_top8_order": 40,
+        "same_top8_set": 40
+      },
+      "f16": {
+        "n": 40,
+        "same_top1": 40,
+        "same_top8_order": 40,
+        "same_top8_set": 40
+      }
+    },
+    "ta": {
+      "max_abs_f16_weight_delta": 0.0009765625,
+      "q4": {
+        "n": 40,
+        "same_top1": 40,
+        "same_top8_order": 40,
+        "same_top8_set": 40
+      },
+      "f16": {
+        "n": 40,
+        "same_top1": 40,
+        "same_top8_order": 40,
+        "same_top8_set": 40
+      }
+    }
+  },
+  "sentence": {
+    "n": 12,
+    "source_search_outputs": [
+      [
+        "विलुप्ति",
+        "की",
+        "कगार",
+        "पर",
+        "गुणकारी",
+        "तिखुर"
+      ],
+      [
+        "शाहनामा",
+        "नौरोज",
+        "के",
+        "त्योहार",
+        "को",
+        "महान",
+        "जमशेद",
+        "के",
+        "शासनकाल",
+        "से",
+        "जोड़ता",
+        "है",
+        "."
+      ],
+      [
+        "मेहरोत्रा",
+        ",",
+        "दर",
+        "न",
+        "."
+      ],
+      [
+        "दुर्गा",
+        "भाभी"
+      ],
+      [
+        "स्वामी",
+        "बृहस्पति"
+      ],
+      [
+        "बॉलीवुड",
+        ",",
+        "हिंदी",
+        "चलचित्र",
+        "उद्योग",
+        "भी",
+        "मुंबई",
+        "में",
+        "ही",
+        "स्थित",
+        "है",
+        "."
+      ],
+      [
+        "क्योंकि",
+        "ऐसे",
+        "शब्द",
+        "भाषा",
+        "के",
+        "मूल",
+        "शब्द",
+        "होते",
+        "हैं",
+        "."
+      ],
+      [
+        "अंडे",
+        "के",
+        "छिलके",
+        "अन्य",
+        "एकांकी",
+        "तथा",
+        "बीज",
+        "नाटक",
+        "-",
+        "मोहन",
+        "राकेश"
+      ],
+      [
+        "तीसरा",
+        "सप्तक",
+        "(",
+        "कविता",
+        "संग्रह",
+        ")",
+        ","
+      ],
+      [
+        "राह",
+        "चलते",
+        "एक",
+        "वैद्य",
+        "को",
+        "शंकर",
+        "मिल",
+        "जाता",
+        "है",
+        "."
+      ],
+      [
+        "अति",
+        "विशिष्ट",
+        "सेवा",
+        "मेडल"
+      ],
+      [
+        "सारे",
+        "ललहते",
+        "खेत",
+        ",",
+        "पेड़",
+        "-",
+        "पौधे",
+        ",",
+        "महल",
+        "एयर",
+        "झोपड़ियां",
+        "-",
+        "सब",
+        "की",
+        "सब",
+        "उजाड़",
+        "जाएंगी",
+        "."
+      ]
+    ],
+    "q8": {
+      "same_decoder_outputs": 12,
+      "max_abs_sequence_score_delta": 0.0,
+      "max_abs_weight_delta": 0.0
+    },
+    "q4": {
+      "same_decoder_outputs": 11,
+      "max_abs_sequence_score_delta": 6.343046009540558,
+      "max_abs_weight_delta": 0.2186431884765625
+    },
+    "f16": {
+      "same_decoder_outputs": 12,
+      "max_abs_sequence_score_delta": 0.008701682090759277,
+      "max_abs_weight_delta": 0.0009307861328125
+    }
+  },
+  "js_vs_python": {
+    "hi": {
+      "n": 40,
+      "same_top1": 40,
+      "same_top8_order": 40,
+      "same_top8_set": 40
+    },
+    "bn": {
+      "n": 40,
+      "same_top1": 40,
+      "same_top8_order": 40,
+      "same_top8_set": 40
+    },
+    "te": {
+      "n": 40,
+      "same_top1": 40,
+      "same_top8_order": 40,
+      "same_top8_set": 40
+    },
+    "kn": {
+      "n": 40,
+      "same_top1": 40,
+      "same_top8_order": 40,
+      "same_top8_set": 40
+    },
+    "ta": {
+      "n": 40,
+      "same_top1": 40,
+      "same_top8_order": 40,
+      "same_top8_set": 40
+    }
+  },
+  "js_sentence_vs_python": {
+    "n": 12,
+    "same_output": 12,
+    "mismatches": [],
+    "policy": "Both runtimes use plain reference token conversion; older UI brand/acronym heuristics are not used."
+  }
+}

evaluation/gguf_sizes.json

@@ -0,0 +1,67 @@
+{
+  "aksharalp-hi-word-q4.gguf": {
+    "raw": 632416,
+    "gzip9": 562579,
+    "sha256": "737d14197514a2625eb9505eb0a22ebd62cd2a4b764c65761f4084599bb6eb36"
+  },
+  "aksharalp-hi-word-f16.gguf": {
+    "raw": 2026048,
+    "gzip9": 1348288,
+    "sha256": "3b6670429a226a39d357160691c634160667728bae8cfc7a711067424a2d224e"
+  },
+  "aksharalp-bn-word-q4.gguf": {
+    "raw": 591136,
+    "gzip9": 541220,
+    "sha256": "8311c77889381ce559d483b965027c2d409787ac090b146ae1af08060e4a4552"
+  },
+  "aksharalp-bn-word-f16.gguf": {
+    "raw": 1984864,
+    "gzip9": 1321177,
+    "sha256": "7a0b78e133bb3ed773f6a8bae5e3f1af4b75f5158de4e814efadeaa0002ed580"
+  },
+  "aksharalp-te-word-q4.gguf": {
+    "raw": 591008,
+    "gzip9": 542744,
+    "sha256": "56b9f956b69e4fcf9e8d3f23c9391a091208651e90b06e2bde6d7f46d7cb6a60"
+  },
+  "aksharalp-te-word-f16.gguf": {
+    "raw": 1984608,
+    "gzip9": 1329379,
+    "sha256": "3c556cec413db8f1485a1ee9ea237406f967937019e0d9d6076c3ae41b7a6231"
+  },
+  "aksharalp-kn-word-q4.gguf": {
+    "raw": 591008,
+    "gzip9": 540642,
+    "sha256": "0e000f7fc0e52bbcb1f05753a46e6de50d1a1bdd6bb0ef46303361b9c39138ed"
+  },
+  "aksharalp-kn-word-f16.gguf": {
+    "raw": 1984608,
+    "gzip9": 1319622,
+    "sha256": "c9d273c8339864bd37c2b051cab60d2741c1d71e56e8c27d00a981ff28ef3864"
+  },
+  "aksharalp-ta-word-q4.gguf": {
+    "raw": 591008,
+    "gzip9": 542094,
+    "sha256": "f270e77e6355d53dc140f7cc7282551f5126302bcfbdbb17d802861510896c04"
+  },
+  "aksharalp-ta-word-f16.gguf": {
+    "raw": 1984608,
+    "gzip9": 1328348,
+    "sha256": "3baf8764ecc77250666eb3f8eb83f2d7a33763e13227e2d6db8272868dd5a864"
+  },
+  "aksharalp-hi-sentence-q8.gguf": {
+    "raw": 3488192,
+    "gzip9": 2981426,
+    "sha256": "5ff23bc8d131b560ee29c79ee16d15294600149200948433d76f29f4b8c3e496"
+  },
+  "aksharalp-hi-sentence-q4.gguf": {
+    "raw": 2110912,
+    "gzip9": 1596808,
+    "sha256": "8ecc83bc618b9c67173328a34f0f40e5c83ce77a993a6279acdeae6d724690db"
+  },
+  "aksharalp-hi-sentence-f16.gguf": {
+    "raw": 5790336,
+    "gzip9": 5159347,
+    "sha256": "446ff8f003c3680e9098df94727f8eb67a6a05a7c418888fb0bfeeb2742e098e"
+  }
+}

evaluation/hindi_existing_results.json

@@ -0,0 +1,43 @@
+{
+  "word": {
+    "ours": {
+      "n": 9991,
+      "top1": 70.04,
+      "top1_ci95": [
+        69.16,
+        70.9
+      ],
+      "top3": 88.31,
+      "top8": 93.63
+    },
+    "indicxlit_full": {
+      "top1": 69.33,
+      "top1_ci95": [
+        68.44,
+        70.23
+      ],
+      "top3": 85.4,
+      "top8": 88.29
+    },
+    "paired_ours_minus_indicxlit": {
+      "top1": [
+        0.71,
+        -0.04,
+        1.4
+      ]
+    }
+  },
+  "sentence": {
+    "n": 4976,
+    "baseline_word_accuracy": 89.02,
+    "word_accuracy": 89.44,
+    "baseline_sentence_em": 25.7,
+    "sentence_em": 26.73,
+    "baseline_cer": 4.07,
+    "cer": 3.93,
+    "baseline_wer": 11.2,
+    "wer": 10.78,
+    "source": "https://huggingface.co/sourojitdotcom/AksharaLP-Sentence-2.8M",
+    "not_retested": true
+  }
+}

evaluation/indicxlit_storage.json

@@ -0,0 +1,27 @@
+{
+  "parameters": 11487744,
+  "fp32_parameter_bytes": 45950976,
+  "dictionaries": {
+    "hi": {
+      "raw": 358118856,
+      "zip_compressed": 42255573
+    },
+    "bn": {
+      "raw": 403487845,
+      "zip_compressed": 45529651
+    },
+    "te": {
+      "raw": 761940466,
+      "zip_compressed": 84164136
+    },
+    "kn": {
+      "raw": 1045504407,
+      "zip_compressed": 114183601
+    },
+    "ta": {
+      "raw": 1044620827,
+      "zip_compressed": 104546002
+    }
+  },
+  "note": "Published dictionary JSON archive member bytes, without repacking; ZIP is not gzip-9. FP32 parameter bytes excludes serialization overhead. IndicXlit supports more languages than this bundle."
+}

evaluation/kn_frozen.json

@@ -0,0 +1,74 @@
+{
+  "files": {
+    "xlit.q4": "3788fd545c297febb7dbbdbdd780e831042395b4ec99260b95c963607afc8664",
+    "dict.tsv.gz": "97035264d30eddb5810467dc5bbf088614221ea6ba2c3fff349c6bbd196bb644",
+    "params.json": "6076d9126ffce66630b634e47ac5b38d03cda3bce7ab4d07162aeb0cbc925356"
+  },
+  "fit": "VALID only; frozen before test",
+  "config": {
+    "format": "aksharalp-multiindic/1",
+    "language": "kn",
+    "beam": 8,
+    "nbest": 8,
+    "model": "xlit.q4",
+    "dictionary": "dict.tsv.gz",
+    "dictionary_words": 500000,
+    "dictionary_total": 23038668,
+    "unk": -16.952684623026045,
+    "reranker": {
+      "type": "linear",
+      "features": [
+        "sc",
+        "scn",
+        "inlex",
+        "uni",
+        "free0",
+        "rank",
+        "sc_rel",
+        "uni_rel"
+      ],
+      "w": [
+        0.7762413620948792,
+        0.47590601444244385,
+        1.8370015621185303,
+        0.37460821866989136,
+        -0.14645907282829285,
+        -0.39526867866516113,
+        0.7561540603637695,
+        0.40543532371520996
+      ],
+      "mu": [
+        -5.910224747519752,
+        -0.7570656360672512,
+        0.21765999600123598,
+        -2.8589205075172495,
+        0.12688805278368503,
+        0.43655597360815745,
+        -4.8503386403114055,
+        -0.3713378327997356
+      ],
+      "sd": [
+        2.385781122067686,
+        0.5672953341562836,
+        0.4126559674266211,
+        5.516374375872727,
+        0.3328485249185093,
+        0.2863515606523184,
+        2.4442090407344486,
+        1.2445700746967023
+      ],
+      "fit_split": "all Aksharantar VALID",
+      "fit_n": 6981,
+      "l2": 0.001,
+      "seed": 0
+    }
+  },
+  "inventory": {
+    "parameters": 988928,
+    "model_raw": 588904,
+    "model_gz9": 539543,
+    "dict_raw": 16786145,
+    "dict_gz9": 2684083,
+    "dict_words": 500000
+  }
+}

evaluation/latency.json

@@ -0,0 +1,43 @@
+{
+  "device": "Intel(R) Xeon(R) Processor",
+  "node": "v24.22.1",
+  "platform": "linux",
+  "method": "100 evenly spaced Aksharantar VALID words per language; 5 warmups; cache cleared per word; beam 8 with dictionary; Node single JS thread; load time excluded",
+  "languages": {
+    "hi": {
+      "n": 100,
+      "p50_ms": 207.02,
+      "p95_ms": 319.99,
+      "mean_ms": 217.33,
+      "load_ms": 7697.54
+    },
+    "bn": {
+      "n": 100,
+      "p50_ms": 51.61,
+      "p95_ms": 74,
+      "mean_ms": 52.41,
+      "load_ms": 0.02
+    },
+    "te": {
+      "n": 100,
+      "p50_ms": 51.77,
+      "p95_ms": 87.87,
+      "mean_ms": 54.86,
+      "load_ms": 398.74
+    },
+    "kn": {
+      "n": 100,
+      "p50_ms": 55.35,
+      "p95_ms": 100.77,
+      "mean_ms": 59.86,
+      "load_ms": 380.85
+    },
+    "ta": {
+      "n": 100,
+      "p50_ms": 60.98,
+      "p95_ms": 103.39,
+      "mean_ms": 63.09,
+      "load_ms": 451.39
+    }
+  }
+}

evaluation/multiindic_test.json

@@ -0,0 +1,318 @@
+{
+  "protocol": "Frozen q4 beam-8 + eight-feature linear reranker fitted on all VALID; one final test per artifact; any-reference NFC and zero-width stripped; 2000 bootstrap replicates, seed 0 (scorer defaults)",
+  "languages": {
+    "bn": {
+      "ours": {
+        "n": 13367,
+        "top1": 64.64,
+        "top1_ci95": [
+          63.83,
+          65.44
+        ],
+        "top3": 75.66,
+        "top3_ci95": [
+          74.9,
+          76.38
+        ],
+        "top8": 77.97,
+        "top8_ci95": [
+          77.23,
+          78.66
+        ]
+      },
+      "indicxlit_full": {
+        "n": 13367,
+        "top1": 70.52,
+        "top1_ci95": [
+          69.76,
+          71.34
+        ],
+        "top3": 82.25,
+        "top3_ci95": [
+          81.61,
+          82.89
+        ],
+        "top8": 85.44,
+        "top8_ci95": [
+          84.86,
+          86.06
+        ]
+      },
+      "paired_ours_minus_indicxlit": {
+        "top1": [
+          -5.88,
+          -6.52,
+          -5.22
+        ],
+        "top3": [
+          -6.59,
+          -7.18,
+          -5.97
+        ],
+        "top8": [
+          -7.47,
+          -8.05,
+          -6.89
+        ]
+      },
+      "frozen_sha256": {
+        "xlit.q4": "9f099f0b9524f446debf1d3f0ae1dc7ed1e88625c33f6dbb7b45e0348bf76376",
+        "dict.tsv.gz": "edb2ad0bd7de4e87bd4d49ac406a9ef01011451bf23ac82d4c063bb9cce1dc2b",
+        "params.json": "e0bd564ad61f6dcef8eca6bf3d64f8b231146c9885aa4c5b282a2b2420606c4f"
+      },
+      "test_rows_sha256": "de2dc6a45db144a32074c586b2b92f6af07e9b3b4fef495150d62c59c31c9dfd",
+      "baseline_predictions_sha256": "e3e72158035ae04c46691f75eb85df3b59959e9d801cd9099af26148cf7a5c9e",
+      "baseline_missing_empty_inputs": 1,
+      "baseline_note": "Existing IndicXlit PyTorch port beam-8 + full dictionary predictions, rescored on exactly the same rows; no new baseline inference.",
+      "size": {
+        "parameters": 989056,
+        "model_raw": 589040,
+        "model_gz9": 540107,
+        "dict_raw": 14068340,
+        "dict_gz9": 2508610,
+        "dict_words": 500000
+      }
+    },
+    "te": {
+      "ours": {
+        "n": 9936,
+        "top1": 78.74,
+        "top1_ci95": [
+          77.93,
+          79.53
+        ],
+        "top3": 86.87,
+        "top3_ci95": [
+          86.2,
+          87.52
+        ],
+        "top8": 88.63,
+        "top8_ci95": [
+          88.0,
+          89.25
+        ]
+      },
+      "indicxlit_full": {
+        "n": 9936,
+        "top1": 82.42,
+        "top1_ci95": [
+          81.65,
+          83.14
+        ],
+        "top3": 90.63,
+        "top3_ci95": [
+          90.04,
+          91.18
+        ],
+        "top8": 92.75,
+        "top8_ci95": [
+          92.21,
+          93.26
+        ]
+      },
+      "paired_ours_minus_indicxlit": {
+        "top1": [
+          -3.67,
+          -4.27,
+          -3.07
+        ],
+        "top3": [
+          -3.76,
+          -4.27,
+          -3.21
+        ],
+        "top8": [
+          -4.13,
+          -4.59,
+          -3.62
+        ]
+      },
+      "frozen_sha256": {
+        "xlit.q4": "aea094e46a3c61eb9487d1aaba3824d3d9dc60f3f1cf6b6f99cadf04fd4318f9",
+        "dict.tsv.gz": "20612a52ccf39c816d3068e78e9e9a3aa9aed9156f10de45ebbd543363943640",
+        "params.json": "653f897efe1af78bb52149f2a18dc804b49ecac9c93add867871e64f56e07470"
+      },
+      "test_rows_sha256": "698de748186eb92a43baed29e1e0aa9a15acbd5b20a79b3a4fd7db4b336ceda1",
+      "baseline_predictions_sha256": "6c05dfc219ebc012fd96bac5f5b9586fd5ac35d0856a36af28c7362b04a31c82",
+      "baseline_note": "Existing IndicXlit PyTorch port beam-8 + full dictionary predictions, rescored on exactly the same rows; no new baseline inference.",
+      "size": {
+        "parameters": 988928,
+        "model_raw": 588904,
+        "model_gz9": 541242,
+        "dict_raw": 15989177,
+        "dict_gz9": 2680527,
+        "dict_words": 500000
+      }
+    },
+    "kn": {
+      "ours": {
+        "n": 11165,
+        "top1": 77.37,
+        "top1_ci95": [
+          76.61,
+          78.13
+        ],
+        "top3": 85.8,
+        "top3_ci95": [
+          85.16,
+          86.46
+        ],
+        "top8": 87.68,
+        "top8_ci95": [
+          87.08,
+          88.32
+        ]
+      },
+      "indicxlit_full": {
+        "n": 11165,
+        "top1": 82.61,
+        "top1_ci95": [
+          81.92,
+          83.29
+        ],
+        "top3": 90.68,
+        "top3_ci95": [
+          90.16,
+          91.2
+        ],
+        "top8": 92.41,
+        "top8_ci95": [
+          91.94,
+          92.9
+        ]
+      },
+      "paired_ours_minus_indicxlit": {
+        "top1": [
+          -5.24,
+          -5.83,
+          -4.67
+        ],
+        "top3": [
+          -4.87,
+          -5.39,
+          -4.39
+        ],
+        "top8": [
+          -4.73,
+          -5.19,
+          -4.26
+        ]
+      },
+      "frozen_sha256": {
+        "xlit.q4": "3788fd545c297febb7dbbdbdd780e831042395b4ec99260b95c963607afc8664",
+        "dict.tsv.gz": "97035264d30eddb5810467dc5bbf088614221ea6ba2c3fff349c6bbd196bb644",
+        "params.json": "6076d9126ffce66630b634e47ac5b38d03cda3bce7ab4d07162aeb0cbc925356"
+      },
+      "test_rows_sha256": "5bfe000c741316d700f4caae046ed05c4cdcaf485ef31707f6254bcc4c9abbd9",
+      "baseline_predictions_sha256": "928c3375ffdbca7811a36284ac4afb70b8b5192d28aa35f88a6291f941373407",
+      "baseline_note": "Existing IndicXlit PyTorch port beam-8 + full dictionary predictions, rescored on exactly the same rows; no new baseline inference.",
+      "size": {
+        "parameters": 988928,
+        "model_raw": 588904,
+        "model_gz9": 539543,
+        "dict_raw": 16786145,
+        "dict_gz9": 2684083,
+        "dict_words": 500000
+      }
+    },
+    "ta": {
+      "ours": {
+        "n": 11405,
+        "top1": 71.78,
+        "top1_ci95": [
+          71.01,
+          72.61
+        ],
+        "top3": 80.05,
+        "top3_ci95": [
+          79.37,
+          80.78
+        ],
+        "top8": 82.61,
+        "top8_ci95": [
+          81.95,
+          83.31
+        ]
+      },
+      "indicxlit_full": {
+        "n": 11405,
+        "top1": 75.95,
+        "top1_ci95": [
+          75.2,
+          76.73
+        ],
+        "top3": 84.21,
+        "top3_ci95": [
+          83.56,
+          84.89
+        ],
+        "top8": 87.26,
+        "top8_ci95": [
+          86.66,
+          87.87
+        ]
+      },
+      "paired_ours_minus_indicxlit": {
+        "top1": [
+          -4.16,
+          -4.75,
+          -3.59
+        ],
+        "top3": [
+          -4.16,
+          -4.66,
+          -3.63
+        ],
+        "top8": [
+          -4.65,
+          -5.12,
+          -4.16
+        ]
+      },
+      "frozen_sha256": {
+        "xlit.q4": "5dd185a033db79835fede66ab41a25f5897f24ad5d7c8ecc9864ca6ec167fdc0",
+        "dict.tsv.gz": "4d14f514020e7737144c22991e9324a29734b5ab203bef3b68ec6d9f51e7cddb",
+        "params.json": "f4520a8320f02573e6b8b6530a372c980903fbf757999540698957e8eec58385"
+      },
+      "test_rows_sha256": "464c11fec7cfa71680a01c57899aeb691d88f1a871b1c7976184b0a21e2d75a6",
+      "baseline_predictions_sha256": "0c2272e9423eb22e3fa04037a9973a777d50070cb9f3bcd03d4037865fd71bc3",
+      "baseline_note": "Existing IndicXlit PyTorch port beam-8 + full dictionary predictions, rescored on exactly the same rows; no new baseline inference.",
+      "size": {
+        "parameters": 988928,
+        "model_raw": 588904,
+        "model_gz9": 540818,
+        "dict_raw": 18105290,
+        "dict_gz9": 2588634,
+        "dict_words": 500000
+      }
+    },
+    "hi": {
+      "ours": {
+        "n": 9991,
+        "top1": 70.04,
+        "top1_ci95": [
+          69.16,
+          70.9
+        ],
+        "top3": 88.31,
+        "top8": 93.63
+      },
+      "indicxlit_full": {
+        "top1": 69.33,
+        "top1_ci95": [
+          68.44,
+          70.23
+        ],
+        "top3": 85.4,
+        "top8": 88.29
+      },
+      "paired_ours_minus_indicxlit": {
+        "top1": [
+          0.71,
+          -0.04,
+          1.4
+        ]
+      }
+    }
+  }
+}

evaluation/release_summary.json

@@ -0,0 +1,51 @@
+{
+  "name": "AksharaLP-MultiIndic-8M",
+  "parameters": 7715520,
+  "word_parameters": 4944768,
+  "sizes": {
+    "languages": {
+      "hi": {
+        "raw": 16221771,
+        "gzip9": 7905041
+      },
+      "bn": {
+        "raw": 14658574,
+        "gzip9": 3049313
+      },
+      "te": {
+        "raw": 16579274,
+        "gzip9": 3222365
+      },
+      "kn": {
+        "raw": 17376245,
+        "gzip9": 3224222
+      },
+      "ta": {
+        "raw": 18695389,
+        "gzip9": 3130045
+      }
+    },
+    "shared_js_and_config": {
+      "raw": 68781,
+      "gzip9": 23186
+    },
+    "word_only": {
+      "raw": 83600034,
+      "gzip9": 20554172
+    },
+    "sentence_addon": {
+      "raw": 3256675,
+      "gzip9": 2928839
+    },
+    "with_sentence": {
+      "raw": 86856709,
+      "gzip9": 23483011
+    },
+    "note": "Canonical payload bytes and sum of per-file gzip -9 equivalents; MB=1,000,000 bytes. Excludes documentation, Python reference, fonts, charts, GGUF alternatives and duplicate compressed companions. Dictionaries are frequency data, not neural parameters. Runtime computes float32 after dequantization; RAM is not file size."
+  },
+  "gguf_files": 13,
+  "gguf_total": {
+    "raw": 24350752,
+    "gzip9": 19113674
+  }
+}

evaluation/sizes.json

@@ -0,0 +1,41 @@
+{
+  "languages": {
+    "hi": {
+      "raw": 16221771,
+      "gzip9": 7905041
+    },
+    "bn": {
+      "raw": 14658574,
+      "gzip9": 3049313
+    },
+    "te": {
+      "raw": 16579274,
+      "gzip9": 3222365
+    },
+    "kn": {
+      "raw": 17376245,
+      "gzip9": 3224222
+    },
+    "ta": {
+      "raw": 18695389,
+      "gzip9": 3130045
+    }
+  },
+  "shared_js_and_config": {
+    "raw": 68781,
+    "gzip9": 23186
+  },
+  "word_only": {
+    "raw": 83600034,
+    "gzip9": 20554172
+  },
+  "sentence_addon": {
+    "raw": 3256675,
+    "gzip9": 2928839
+  },
+  "with_sentence": {
+    "raw": 86856709,
+    "gzip9": 23483011
+  },
+  "note": "Canonical payload bytes and sum of per-file gzip -9 equivalents; MB=1,000,000 bytes. Excludes documentation, Python reference, fonts, charts, GGUF alternatives and duplicate compressed companions. Dictionaries are frequency data, not neural parameters. Runtime computes float32 after dequantization; RAM is not file size."
+}

evaluation/ta_frozen.json

@@ -0,0 +1,74 @@
+{
+  "files": {
+    "xlit.q4": "5dd185a033db79835fede66ab41a25f5897f24ad5d7c8ecc9864ca6ec167fdc0",
+    "dict.tsv.gz": "4d14f514020e7737144c22991e9324a29734b5ab203bef3b68ec6d9f51e7cddb",
+    "params.json": "f4520a8320f02573e6b8b6530a372c980903fbf757999540698957e8eec58385"
+  },
+  "fit": "VALID only; frozen before test",
+  "config": {
+    "format": "aksharalp-multiindic/1",
+    "language": "ta",
+    "beam": 8,
+    "nbest": 8,
+    "model": "xlit.q4",
+    "dictionary": "dict.tsv.gz",
+    "dictionary_words": 500000,
+    "dictionary_total": 20232595,
+    "unk": -16.82280552518196,
+    "reranker": {
+      "type": "linear",
+      "features": [
+        "sc",
+        "scn",
+        "inlex",
+        "uni",
+        "free0",
+        "rank",
+        "sc_rel",
+        "uni_rel"
+      ],
+      "w": [
+        0.9623245596885681,
+        0.5414344072341919,
+        1.828786015510559,
+        0.003193225245922804,
+        -0.12186560034751892,
+        -0.003971157129853964,
+        0.9330659508705139,
+        0.5227358937263489
+      ],
+      "mu": [
+        -6.01308456526665,
+        -0.7381095081752549,
+        0.22912272701362366,
+        -2.9371551977749273,
+        0.1254346462976686,
+        0.43728267685116573,
+        -4.8824671233512555,
+        -0.38838364800108277
+      ],
+      "sd": [
+        2.449121527790252,
+        0.5435378258591739,
+        0.42026937018709404,
+        5.483689004938199,
+        0.3312121045929875,
+        0.28639834850311835,
+        2.5225466225846818,
+        1.2477123699412511
+      ],
+      "fit_split": "all Aksharantar VALID",
+      "fit_n": 8802,
+      "l2": 0.001,
+      "seed": 0
+    }
+  },
+  "inventory": {
+    "parameters": 988928,
+    "model_raw": 588904,
+    "model_gz9": 540818,
+    "dict_raw": 18105290,
+    "dict_gz9": 2588634,
+    "dict_words": 500000
+  }
+}

evaluation/te_frozen.json

@@ -0,0 +1,74 @@
+{
+  "files": {
+    "xlit.q4": "aea094e46a3c61eb9487d1aaba3824d3d9dc60f3f1cf6b6f99cadf04fd4318f9",
+    "dict.tsv.gz": "20612a52ccf39c816d3068e78e9e9a3aa9aed9156f10de45ebbd543363943640",
+    "params.json": "653f897efe1af78bb52149f2a18dc804b49ecac9c93add867871e64f56e07470"
+  },
+  "fit": "VALID only; frozen before test",
+  "config": {
+    "format": "aksharalp-multiindic/1",
+    "language": "te",
+    "beam": 8,
+    "nbest": 8,
+    "model": "xlit.q4",
+    "dictionary": "dict.tsv.gz",
+    "dictionary_words": 500000,
+    "dictionary_total": 24117810,
+    "unk": -16.998461171144285,
+    "reranker": {
+      "type": "linear",
+      "features": [
+        "sc",
+        "scn",
+        "inlex",
+        "uni",
+        "free0",
+        "rank",
+        "sc_rel",
+        "uni_rel"
+      ],
+      "w": [
+        0.8893349170684814,
+        0.42266765236854553,
+        1.647674560546875,
+        0.01816709153354168,
+        -0.09264160692691803,
+        -0.17859779298305511,
+        0.8579440116882324,
+        0.4983164668083191
+      ],
+      "mu": [
+        -5.577303872867701,
+        -0.7360878454967777,
+        0.22874146800712822,
+        -3.057939543170047,
+        0.12881207760330857,
+        0.43559396119834565,
+        -4.512450547449157,
+        -0.4069890521202824
+      ],
+      "sd": [
+        2.345458571867553,
+        0.5172883363068768,
+        0.4200233908567531,
+        5.716765533673338,
+        0.33499283014943054,
+        0.28628677366288935,
+        2.4278868542564216,
+        1.326799761380314
+      ],
+      "fit_split": "all Aksharantar VALID",
+      "fit_n": 7662,
+      "l2": 0.001,
+      "seed": 0
+    }
+  },
+  "inventory": {
+    "parameters": 988928,
+    "model_raw": 588904,
+    "model_gz9": 541242,
+    "dict_raw": 15989177,
+    "dict_gz9": 2680527,
+    "dict_words": 500000
+  }
+}

gguf/aksharalp-bn-word-f16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a0b78e133bb3ed773f6a8bae5e3f1af4b75f5158de4e814efadeaa0002ed580
+size 1984864

gguf/aksharalp-bn-word-q4.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8311c77889381ce559d483b965027c2d409787ac090b146ae1af08060e4a4552
+size 591136

gguf/aksharalp-hi-sentence-f16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:446ff8f003c3680e9098df94727f8eb67a6a05a7c418888fb0bfeeb2742e098e
+size 5790336

gguf/aksharalp-hi-sentence-q4.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ecc83bc618b9c67173328a34f0f40e5c83ce77a993a6279acdeae6d724690db
+size 2110912

gguf/aksharalp-hi-sentence-q8.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ff23bc8d131b560ee29c79ee16d15294600149200948433d76f29f4b8c3e496
+size 3488192

gguf/aksharalp-hi-word-f16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b6670429a226a39d357160691c634160667728bae8cfc7a711067424a2d224e
+size 2026048

gguf/aksharalp-hi-word-q4.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:737d14197514a2625eb9505eb0a22ebd62cd2a4b764c65761f4084599bb6eb36
+size 632416

gguf/aksharalp-kn-word-f16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9d273c8339864bd37c2b051cab60d2741c1d71e56e8c27d00a981ff28ef3864
+size 1984608

gguf/aksharalp-kn-word-q4.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e000f7fc0e52bbcb1f05753a46e6de50d1a1bdd6bb0ef46303361b9c39138ed
+size 591008

gguf/aksharalp-ta-word-f16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3baf8764ecc77250666eb3f8eb83f2d7a33763e13227e2d6db8272868dd5a864
+size 1984608

gguf/aksharalp-ta-word-q4.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f270e77e6355d53dc140f7cc7282551f5126302bcfbdbb17d802861510896c04
+size 591008

gguf/aksharalp-te-word-f16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c556cec413db8f1485a1ee9ea237406f967937019e0d9d6076c3ae41b7a6231
+size 1984608

gguf/aksharalp-te-word-q4.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56b9f956b69e4fcf9e8d3f23c9391a091208651e90b06e2bde6d7f46d7cb6a60
+size 591008

gguf_read.py

@@ -0,0 +1,383 @@
+"""Minimal, dependency-light reader for the GGUF files shipped with AksharaLP-MultiIndic-8M.
+
+GGUF here is used as what it is: a portable, single-file tensor container -- a header, typed
+key/value metadata, a tensor table, and aligned little-endian tensor data. It is NOT a
+llama.cpp-loadable model: AksharaLP is a custom character-level encoder-decoder transformer with a
+bespoke decoding stack (lexicon-trie candidate beam, hand-built candidate features, gates, MLP
+reranker), and llama.cpp has no such architecture. This reader is the decoder-side tool: it returns
+the weights our reference implementation (python/bhc/xlit_*.py) expects.
+
+Only numpy is required. No torch, no llama.cpp, no gguf/gguf-py.
+
+    python gguf_read.py aksharalp-1m-v2-f16.gguf          # summary
+    python gguf_read.py --json aksharalp-1m-v2-q4.gguf     # metadata + tensor table
+
+    import gguf_read
+    w = gguf_read.load("aksharalp-1m-v2-f16.gguf")          # -> Weights(cfg, state_dict, metadata)
+    w.state_dict["semb.weight"].shape                      # (30, 128) float32
+
+Two container layouts are understood, both GGUF v3:
+
+  * aksharalp-1m-v2-q4.gguf   one I8 tensor, ``aksharalp.q4.blob``: the byte-exact xlit.q4 file
+                              (4-byte header length, JSON tensor table, then the payload). Decoded
+                              here exactly as bhc/xlit_q.py:load_q does.
+  * aksharalp-1m-v2-f16.gguf  every weight under its state_dict name at F16.
+"""
+import json
+import struct
+import sys
+
+import numpy as np
+
+MAGIC = b"GGUF"
+SUPPORTED_VERSIONS = (3,)
+
+# GGUF metadata value types
+GGUF_UINT8, GGUF_INT8, GGUF_UINT16, GGUF_INT16 = 0, 1, 2, 3
+GGUF_UINT32, GGUF_INT32, GGUF_FLOAT32, GGUF_BOOL = 4, 5, 6, 7
+GGUF_STRING, GGUF_ARRAY, GGUF_UINT64, GGUF_INT64, GGUF_FLOAT64 = 8, 9, 10, 11, 12
+
+# GGML tensor types we can read: F32, F16, I8 (the three this model uses)
+_GGML_F32, _GGML_F16, _GGML_I8 = 0, 1, 24
+_TENSOR_DTYPES = {_GGML_F32: np.dtype("<f4"), _GGML_F16: np.dtype("<f2"), _GGML_I8: np.dtype("i1")}
+_TENSOR_NAMES = {_GGML_F32: "F32", _GGML_F16: "F16", _GGML_I8: "I8"}
+_KV_FMT = {GGUF_UINT8: "<B", GGUF_INT8: "<b", GGUF_UINT16: "<H", GGUF_INT16: "<h",
+           GGUF_UINT32: "<I", GGUF_INT32: "<i", GGUF_FLOAT32: "<f", GGUF_BOOL: "?",
+           GGUF_UINT64: "<Q", GGUF_INT64: "<q", GGUF_FLOAT64: "<d"}
+
+
+class GGUFError(ValueError):
+    """Raised for anything that is not a well-formed GGUF v3 file we can read."""
+
+
+class _Cursor:
+    def __init__(self, buf, pos=0):
+        self.buf, self.pos = buf, pos
+
+    def take(self, n):
+        if n < 0 or self.pos + n > len(self.buf):
+            raise GGUFError("truncated file: wanted %d bytes at %d, file is %d" % (n, self.pos, len(self.buf)))
+        b = self.buf[self.pos:self.pos + n]
+        self.pos += n
+        return b
+
+    def unpack(self, fmt):
+        n = struct.calcsize(fmt)
+        return struct.unpack(fmt, self.take(n))[0]
+
+    def string(self):
+        """GGUF strings: u64 length + UTF-8 bytes (not NUL-terminated)."""
+        n = self.unpack("Q")
+        return self.take(n).decode("utf-8")
+
+
+class Tensor:
+    """One tensor: name, logical shape (GGML order reversed back to row-major), dtype, raw bytes."""
+
+    __slots__ = ("name", "shape", "type_id", "offset", "data")
+
+    def __init__(self, name, shape, type_id, offset, data):
+        self.name, self.shape, self.type_id, self.offset, self.data = name, shape, type_id, offset, data
+
+    @property
+    def dtype(self):
+        return _TENSOR_DTYPES[self.type_id]
+
+    @property
+    def type_name(self):
+        return _TENSOR_NAMES[self.type_id]
+
+    def array(self):
+        return np.frombuffer(self.data, dtype=self.dtype).reshape(self.shape)
+
+    def __repr__(self):
+        return "<Tensor %s %s %s>" % (self.name, self.type_name, tuple(self.shape))
+
+
+class GGUF:
+    """Parsed GGUF file: header fields, typed metadata, tensor table, tensor data."""
+
+    def __init__(self, path):
+        self.path = str(path)
+        with open(self.path, "rb") as f:
+            self.buf = f.read()
+        self._parse()
+
+    # ---------------------------------------------------------------- parsing
+    def _parse(self):
+        c = _Cursor(self.buf)
+        self.magic = c.take(4)
+        if self.magic != MAGIC:
+            raise GGUFError("%s: not a GGUF file (magic %r)" % (self.path, self.magic))
+        self.version = c.unpack("I")
+        if self.version not in SUPPORTED_VERSIONS:
+            raise GGUFError("%s: GGUF version %d not supported (this reader knows %s)"
+                            % (self.path, self.version, SUPPORTED_VERSIONS))
+        self.tensor_count = c.unpack("Q")
+        self.kv_count = c.unpack("Q")
+
+        self.metadata = {}
+        for _ in range(self.kv_count):
+            key = c.string()
+            self.metadata[key] = self._value(c)
+
+        self.alignment = int(self.metadata.get("general.alignment", 32))
+        if self.alignment <= 0 or self.alignment & (self.alignment - 1):
+            raise GGUFError("%s: general.alignment %d is not a power of two" % (self.path, self.alignment))
+
+        infos = []
+        for _ in range(self.tensor_count):
+            name = c.string()
+            n_dims = c.unpack("I")
+            if n_dims > 8:
+                raise GGUFError("%s: tensor %s declares %d dims" % (self.path, name, n_dims))
+            # GGML stores dims fastest-varying first; reverse to get the row-major shape.
+            dims = [c.unpack("Q") for _ in range(n_dims)]
+            type_id = c.unpack("I")
+            offset = c.unpack("Q")
+            infos.append((name, list(reversed(dims)), type_id, offset))
+
+        pad = (-c.pos) % self.alignment
+        self.data_start = c.pos + pad
+        if self.data_start > len(self.buf):
+            raise GGUFError("%s: tensor info block overruns the file" % self.path)
+
+        self.tensors = {}
+        self.order = []
+        for name, shape, type_id, offset in infos:
+            if type_id not in _TENSOR_DTYPES:
+                raise GGUFError("%s: tensor %s has GGML type %d, which this reader does not decode "
+                                "(only F32/F16/I8 are used by these files)" % (self.path, name, type_id))
+            if name in self.tensors:
+                raise GGUFError("%s: duplicate tensor name %r" % (self.path, name))
+            numel = int(np.prod(shape, dtype=np.int64)) if shape else 1
+            nbytes = numel * _TENSOR_DTYPES[type_id].itemsize
+            start = self.data_start + offset
+            end = start + nbytes
+            if start < self.data_start or end > len(self.buf):
+                raise GGUFError("%s: tensor %s spans [%d, %d) outside the data section [%d, %d)"
+                                % (self.path, name, start, end, self.data_start, len(self.buf)))
+            if offset % self.alignment:
+                raise GGUFError("%s: tensor %s offset %d is not aligned to %d"
+                                % (self.path, name, offset, self.alignment))
+            self.tensors[name] = Tensor(name, shape, type_id, offset, self.buf[start:end])
+            self.order.append(name)
+
+    def _value(self, c, vtype=None):
+        if vtype is None:
+            vtype = c.unpack("I")
+        if vtype == GGUF_STRING:
+            return c.string()
+        if vtype == GGUF_ARRAY:
+            sub = c.unpack("I")
+            n = c.unpack("Q")
+            return [self._value(c, sub) for _ in range(n)]
+        if vtype in _KV_FMT:
+            return c.unpack(_KV_FMT[vtype])
+        raise GGUFError("%s: unknown metadata value type %d" % (self.path, vtype))
+
+    # ---------------------------------------------------------------- checks
+    def validate(self):
+        """Structural checks; returns a list of human-readable results, raises GGUFError on failure."""
+        notes = []
+        if len(self.tensors) != self.tensor_count:
+            raise GGUFError("tensor count mismatch: header says %d, parsed %d" % (self.tensor_count, len(self.tensors)))
+        notes.append("magic GGUF, version %d" % self.version)
+        notes.append("%d tensors, %d metadata keys" % (self.tensor_count, len(self.metadata)))
+        notes.append("alignment %d, data section at byte %d" % (self.alignment, self.data_start))
+        declared = self.metadata.get("general.architecture")
+        if declared is None:
+            raise GGUFError("general.architecture is missing")
+        for name in self.order:
+            t = self.tensors[name]
+            if t.offset + len(t.data) > len(self.buf) - self.data_start:
+                raise GGUFError("tensor %s overruns the data section" % name)
+        notes.append("all tensor offsets aligned and inside the data section")
+        arch = self.metadata.get("general.architecture")
+        notes.append("general.architecture = %r (custom; not a llama.cpp architecture)" % arch)
+        return notes
+
+    # ---------------------------------------------------------------- weights
+    def state_dict(self):
+        """Return {state_dict name: float32 ndarray} for the reference XlitTransformer."""
+        cfg, sd = self._weights()
+        return cfg, sd
+
+    def _weights(self):
+        if "aksharalp.nlm.blob" in self.tensors:
+            return _weights_from_nlm_blob(bytes(self.tensors["aksharalp.nlm.blob"].data))
+        if _BLOB_TENSOR in self.tensors:
+            return _weights_from_q4_blob(bytes(self.tensors[_BLOB_TENSOR].data))
+        cfg = _cfg_from_metadata(self.metadata)
+        sd = {}
+        for name in self.order:
+            if name.startswith(RERANKER_PREFIX):
+                continue            # reranker matrices are carried separately; see reranker()
+            a = self.tensors[name].array()
+            sd[name] = np.ascontiguousarray(a, dtype=np.float32)
+        return cfg, sd
+
+    def reranker(self):
+        """{('core'|'pack', 'w1'|'b1'|'w2'|'b2'): float32 ndarray} from the F16 variant, else {}."""
+        out = {}
+        for name in self.order:
+            if not name.startswith(RERANKER_PREFIX):
+                continue
+            _, part, param = name.split(".")
+            out[(part, param)] = np.ascontiguousarray(self.tensors[name].array(), dtype=np.float32)
+        return out
+
+
+_BLOB_TENSOR = "aksharalp.q4.blob"
+RERANKER_PREFIX = "reranker."
+
+
+def _cfg_from_metadata(md):
+    if "aksharalp.config_json" in md: return json.loads(md["aksharalp.config_json"])
+    def g(key, default=None):
+        v = md.get("aksharalp." + key, default)
+        if v is None:
+            raise GGUFError("metadata is missing aksharalp.%s" % key)
+        return v
+    cfg = {"d": int(g("d_model")), "h": int(g("n_heads")), "enc": int(g("encoder_layers")),
+           "dec": int(g("decoder_layers")), "ff": int(g("ff_dim")), "p": float(g("dropout", 0.1))}
+    return cfg
+
+
+def _weights_from_q4_blob(blob):
+    """Decode the carried xlit.q4 bytes exactly as python/bhc/xlit_q.py:load_q does."""
+    n = int.from_bytes(blob[:4], "little")
+    meta = json.loads(blob[4:4 + n])
+    data = blob[4 + n:]
+    bits_default, G = int(meta["bits"]), int(meta["G"])
+    cfg = dict(meta["cfg"])
+    sd = {}
+    for k, shape, kind, off, ln in meta["tensors"]:
+        cnt = int(np.prod(shape))
+        if kind.startswith("q"):
+            bits = int(kind[1:]) if len(kind) > 1 else bits_default
+            if bits == 4:
+                raw = np.frombuffer(data, np.uint8, ln, off)
+                q = np.stack([raw >> 4, raw & 15], 1).reshape(-1)[:cnt].astype(np.int16) - 8
+            elif bits == 8:
+                q = np.frombuffer(data, np.int8, ln, off).astype(np.int16)
+            else:
+                raise GGUFError("unsupported q width %d for %s" % (bits, k))
+            r, c = shape
+            ng = (c + G - 1) // G
+            sc = np.frombuffer(data, np.float16, r * ng, off + ln).astype(np.float32).reshape(r, ng)
+            w = q.reshape(r, c).astype(np.float32) * np.repeat(sc, G, 1)[:, :c]
+            sd[k] = w
+        elif kind == "f16":
+            sd[k] = np.frombuffer(data, np.float16, cnt, off).astype(np.float32).reshape(shape).copy()
+        else:
+            raise GGUFError("unsupported tensor kind %r for %s" % (kind, k))
+    return cfg, sd
+
+
+class Weights:
+    """Weights + config, in the shape the reference implementation consumes.
+
+    ``state_dict`` maps state_dict names (as in python/bhc/xlit_q.py:load_q) to float32 ndarrays;
+    ``cfg`` is the XlitTransformer config. To build the torch model the reference uses:
+
+        import torch
+        from bhc.xlit_model import XlitTransformer
+        m = XlitTransformer(**w.cfg)
+        m.load_state_dict({k: torch.from_numpy(v) for k, v in w.state_dict.items()})
+        m.eval()
+    """
+
+    def __init__(self, cfg, state_dict, metadata=None, source=None, reranker=None):
+        self.cfg, self.state_dict, self.metadata, self.source = cfg, state_dict, metadata or {}, source
+        self.reranker = reranker or {}
+
+    @property
+    def n_params(self):
+        return int(sum(v.size for v in self.state_dict.values()))
+
+    def __repr__(self):
+        return "<Weights %s: %d tensors, %d params>" % (self.source, len(self.state_dict), self.n_params)
+
+
+def load(path):
+    """Parse either GGUF variant and return Weights (cfg + state_dict of float32 arrays)."""
+    g = GGUF(path)
+    cfg, sd = g.state_dict()
+    return Weights(cfg, sd, g.metadata, g.path, g.reranker())
+
+
+def tensors(path):
+    """Yield (name, shape, type_name, float32 array) for every tensor in the file."""
+    g = GGUF(path)
+    for name in g.order:
+        t = g.tensors[name]
+        yield name, tuple(t.shape), t.type_name, t.array()
+
+
+def q4_blob(path):
+    """Bytes of the carried xlit.q4 file from the compact GGUF (byte-identical to the original)."""
+    g = GGUF(path)
+    if _BLOB_TENSOR not in g.tensors:
+        raise GGUFError("%s carries no %s tensor (that is the F16 variant)" % (path, _BLOB_TENSOR))
+    return bytes(g.tensors[_BLOB_TENSOR].data)
+
+
+def main(argv):
+    args = [a for a in argv[1:] if not a.startswith("-")]
+    flags = {a for a in argv[1:] if a.startswith("-")}
+    if not args:
+        print(__doc__)
+        return 2
+    g = GGUF(args[0])
+    if "--json" in flags:
+        print(json.dumps({"version": g.version, "tensors": g.tensor_count,
+                          "metadata": g.metadata,
+                          "tensor_table": [{"name": n, "shape": g.tensors[n].shape,
+                                            "type": g.tensors[n].type_name,
+                                            "offset": g.tensors[n].offset} for n in g.order]},
+                         indent=1, ensure_ascii=False))
+        return 0
+    for line in g.validate():
+        print(line)
+    print("general.name = %r" % g.metadata.get("general.name"))
+    print("-- tensors --")
+    for n in g.order:
+        t = g.tensors[n]
+        print("  %-28s %-4s %-18s %d B" % (n, t.type_name, tuple(t.shape), len(t.data)))
+    w = load(args[0])
+    print("-- weights --")
+    print("cfg = %s" % (w.cfg,))
+    print("%d tensors, %d parameters" % (len(w.state_dict), w.n_params))
+    return 0
+
+
+
+
+
+def _weights_from_nlm_blob(blob):
+    n=int.from_bytes(blob[:4],"little");md=json.loads(blob[4:4+n]);data=blob[4+n:];W={};G=md["G"]
+    for k,shape,kind,off,ln in md["tensors"]:
+        count=int(np.prod(shape))
+        if kind in ("f16","f32"):
+            W[k]=np.frombuffer(data,np.float16 if kind=="f16" else np.float32,count,off).astype(np.float32).reshape(shape).copy()
+        else:
+            r,c=shape;ng=(c+G-1)//G
+            if kind=="q8":q=np.frombuffer(data,np.int8,r*c,off).astype(np.float32).reshape(r,c);end=off+r*c
+            else:
+                nb=(r*c+1)//2;u=np.frombuffer(data,np.uint8,nb,off);q=(np.stack([u>>4,u&15],1).reshape(-1)[:r*c].astype(np.int16)-8).astype(np.float32).reshape(r,c);end=off+nb
+            end+=(-end)%4;sc=np.frombuffer(data,np.float16,r*ng,end).astype(np.float32).reshape(r,ng);W[k]=q*np.repeat(sc,G,1)[:,:c]
+    return md["cfg"],W
+
+def load_word_model(path):
+    import torch
+    from bhc.xlit_model import XlitTransformer, set_vocab
+    w=load(path);set_vocab(w.cfg.get("vocab","hi"));m=XlitTransformer(**w.cfg);m.load_state_dict({k:torch.from_numpy(v.copy()) for k,v in w.state_dict.items()});return m.eval()
+
+def load_sentence_lm(path):
+    from bhc.nlm import NLM
+    w=load(path);vocab=json.loads(w.metadata["aksharalp.tokenizer_json"])
+    return NLM(w.cfg,w.state_dict,vocab,w.cfg.get("norm","self"))
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

models/bn/dict.tsv.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edb2ad0bd7de4e87bd4d49ac406a9ef01011451bf23ac82d4c063bb9cce1dc2b
+size 2508610

models/bn/inventory.json

@@ -0,0 +1,8 @@
+{
+  "parameters": 989056,
+  "model_raw": 589040,
+  "model_gz9": 540107,
+  "dict_raw": 14068340,
+  "dict_gz9": 2508610,
+  "dict_words": 500000
+}

models/bn/params.json

@@ -0,0 +1,58 @@
+{
+  "format": "aksharalp-multiindic/1",
+  "language": "bn",
+  "beam": 8,
+  "nbest": 8,
+  "model": "xlit.q4",
+  "dictionary": "dict.tsv.gz",
+  "dictionary_words": 500000,
+  "dictionary_total": 29467775,
+  "unk": -17.198807885289952,
+  "reranker": {
+    "type": "linear",
+    "features": [
+      "sc",
+      "scn",
+      "inlex",
+      "uni",
+      "free0",
+      "rank",
+      "sc_rel",
+      "uni_rel"
+    ],
+    "w": [
+      0.7719883322715759,
+      0.7239015102386475,
+      2.2432072162628174,
+      0.3994755744934082,
+      -0.1507813185453415,
+      -0.07777873426675797,
+      0.7293219566345215,
+      0.42816251516342163
+    ],
+    "mu": [
+      -5.078212233229818,
+      -0.8147631729600909,
+      0.31161147274689055,
+      -4.2673892678692775,
+      0.13265511803554703,
+      0.4336724409822265,
+      -3.7508561308261763,
+      -0.6986245748643944
+    ],
+    "sd": [
+      2.216960295304686,
+      0.6165648295196909,
+      0.46315298671618116,
+      6.490492456336254,
+      0.33920261805997143,
+      0.28614764292216904,
+      2.341120778815271,
+      1.748762448987244
+    ],
+    "fit_split": "all Aksharantar VALID",
+    "fit_n": 11188,
+    "l2": 0.001,
+    "seed": 0
+  }
+}

models/bn/training_report.json

@@ -0,0 +1,31 @@
+{
+ "pairs": 1835520,
+ "counts": {
+  "aks-Dakshina": 76330,
+  "aks-AK-Freq": 7365,
+  "aks-Existing": 5478,
+  "aks-Wikidata": 18249,
+  "aks-Samanantar": 114558,
+  "aks-IndicCorp": 1009448,
+  "ie": 538301,
+  "heldout_drop_ie": 8171,
+  "dup_ie": 149395,
+  "ie_words": 189824,
+  "dup_ei": 185794,
+  "ei": 65791,
+  "heldout_drop_ei": 665,
+  "ei_words": 137774
+ },
+ "unique_natives": 1242898,
+ "unique_romans": 1513122,
+ "overlap_valid_test_natives": 0,
+ "overlap_valid_test_romans": 0,
+ "heldout_natives_total": 11576,
+ "heldout_romans_total": 24322,
+ "margins": {
+  "ie": 0.6,
+  "ei": 0.35,
+  "maxv_ie": 4,
+  "maxv_ei": 2
+ }
+}

models/bn/xlit.q4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f099f0b9524f446debf1d3f0ae1dc7ed1e88625c33f6dbb7b45e0348bf76376
+size 589040

models/bn/xlit.q4.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c87e62a4249a63c333942c1c8736eb2c89f9ee3148ef8fc422784ba6ce21e43
+size 540107

models/hi/dict/lex.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73bd5436b77db4ee433c564521f19a2b78cb6a45c429f274d43bef909708347f
+size 11675371

models/hi/dict/lex.bin.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b4dbff703f395e97b4503646fde7782bab0f3d79b9731ef703df6b961e38f55
+size 6472998