Release GoLLeM 149M after 20B continuation tokens with full GLINT evaluation

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+loss-curve.png filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,30 @@
+GoLLeM-149M-20B
+
+Weights and architecture derive from SlayerLab/gollem-v5-ckpts
+revision 963440ce6ab4ada7e95da4c1faa28ebb00c082d4, published under Apache-2.0.
+modeling_gollem.py extracts the inference classes unchanged from the pinned
+GoLLeM r6 training implementation. The continuation, export and release scripts
+were prepared for this 149M run.
+
+The likelihood routines in glint_metrics.py are adapted from
+Glint-Research/Glint-1.3/benchmark.py, revision
+c3f99e246aa2c64382f9668dc864533617482d0a, whose repository declares MIT.
+Copyright remains with the original contributors. MIT license text:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,133 @@
+---
+license: apache-2.0
+language:
+- en
+library_name: pytorch
+pipeline_tag: text-generation
+base_model: SlayerLab/gollem-v5-ckpts
+tags:
+- gollem
+- tiny-lm
+- muon
+- value-residual
+- continued-pretraining
+- glint-tiny-ml-leaderboard
+---
+
+# GoLLeM 149M — 20B-token continuation
+
+A **148,910,738-parameter English causal language model**, grown from the GoLLeM-v5
+122.8M final checkpoint and continued for **20,000,014,336 additional tokens**.
+This is a base completion model, not an instruction-tuned assistant.
+
+## Evaluation
+
+Full final-checkpoint evaluation, with no best-checkpoint selection:
+
+| Metric | Result | Coverage |
+|---|---:|---|
+| BLiMP accuracy | **79.47%** | 67,000 pairs / 67 tasks |
+| ARC-Easy accuracy | **54.97%** | 2,376 test examples |
+| WikiText-2 byte perplexity | **2.212573** | Full test text |
+| WikiText-2 token perplexity | 21.452430 | Same evaluation |
+| GLINT overall score, fixed board snapshot | **77.11235** | Calculated |
+| GLINT efficiency score, fixed board snapshot | **77.13585** | Calculated |
+
+The likelihood protocol follows `Glint-Research/Glint-1.3/benchmark.py` at
+`c3f99e246aa2c64382f9668dc864533617482d0a`: first-256-token truncation for BLiMP
+and ARC, summed unnormalized log probabilities, zero-shot ARC scoring, and
+non-overlapping 256-token WikiText contexts. Inference uses BF16 autocast with
+FP32 log probabilities. Byte perplexity is converted from token perplexity using
+the measured token/UTF-8-byte ratio. Dataset revisions are pinned in
+[evaluation.json](evaluation.json). Loader order and complete task counts are checked.
+
+Against the [GLINT board](https://huggingface.co/spaces/Glint-Research/Tiny-ML-Leaderboard)
+at revision `2c5ea9682babfe2c410ebbbf78113cc98f08a2f9`, this would place **4th in
+overall score** and **8th in size-adjusted efficiency** among the existing entries
+plus this model. This is a calculated, self-reported placement, not an accepted
+leaderboard entry. Competitor scores were not independently re-evaluated here;
+some competitors use lm-eval-harness and different context/scoring conventions.
+The ranking and normalized score can change as the board changes. This release
+does not establish a leaderboard win.
+
+## Use
+
+This release uses a small custom PyTorch architecture. Use the included loader;
+it is not packaged for `transformers.AutoModelForCausalLM`.
+
+```bash
+pip install huggingface-hub
+hf download SlayerLab/gollem-149m-20b --local-dir gollem-149m-20b
+cd gollem-149m-20b
+pip install -r requirements.txt
+python generate.py --prompt "The scientific method is" --max-new-tokens 64
+```
+
+A matching
+CUDA-enabled PyTorch installation is needed for GPU inference/evaluation.
+
+```python
+from load_model import load_model
+model, tokenizer = load_model('.', device='cuda')
+# model(input_ids) returns (logits, optional_loss)
+```
+
+Weights remain FP32 in `model.safetensors`; tied embeddings are restored by
+`safetensors.torch.load_model`. Export verification checks every model tensor
+and compares logits against the original checkpoint at lengths 16, 256 and 1024.
+The generation CLI uses BF16 autocast on CUDA, FP32 on CPU, and greedy decoding
+by default. It recomputes context rather than using a KV cache.
+
+## Architecture and training
+
+- 19 transformer layers; hidden size 768; 12 attention heads; FFN size 2160.
+- Vocabulary 12,288; tied input/output embeddings; context length 1024.
+- RMSNorm, RoPE (theta 100,000), QK normalization, SwiGLU and value residuals.
+- Source: `SlayerLab/gollem-v5-ckpts`, revision
+  `963440ce6ab4ada7e95da4c1faa28ebb00c082d4`, file
+  `final_128m_16x768/ckpt_760k.pt` (24,903,680,000 source training tokens).
+- FFNs were widened with zero new down-projection columns; three residual blocks
+  were appended with zero output projections. Initial FP32 logits exactly matched
+  the donor on the tested contexts. Optimizer state was reset for continuation.
+- Continuation: 20,000,014,336 tokens, 39,147 actual optimizer updates, seed 1337.
+  Inherited token lineage totals 44,903,694,336; newly added parameters only saw
+  the 20B-token continuation.
+- Cleaned ARC-MIX pool: 9,391,706,576 tokens, reconstructed from the published
+  SlayerLab corpus/removal list. SHA256:
+  `ecfd0a4040a0ece6f07f728f092a1b373b6d253b4adfbe259c5107c1fc16681d`.
+  Repeated shuffled passes; training corpus is not redistributed here. This work
+  did not independently audit all training/evaluation overlap.
+- Muon for hidden matrices and AdamW for auxiliary parameters; cosine learning
+  rate 2e-4 to 2e-5, 250 reference-unit warmup; Muon LR ratio 33.3333.
+- Eight H100 PCIe GPUs. Effective batch started at 256 sequences, then became
+  512 after 524,288,000 continuation tokens. Learning rate remained indexed by
+  token position. Grouped Muon, BF16 gradient communication and later CUDA graphs
+  improved throughput. Numerical ordering changed; this is not a bitwise replay
+  of the original trainer.
+- Optimized sustained full-segment throughput: 1.045M tokens/sec including
+  startup, validation and saves; later ordinary windows reached about 1.14M.
+  These are measurements on this host, not expected performance on every H100 node.
+
+`training_manifest.json` records configuration and hashes. Its `reference_step`
+uses 262,144 tokens/unit; it is not the actual optimizer-update count after the
+batch change. `training_metrics.jsonl` records the training history.
+
+## Reproduce evaluation
+
+```bash
+python evaluate_glint.py --model-dir . --out reproduced-results.json
+```
+
+This evaluates all three datasets on CUDA using pinned dataset revisions. It can
+take tens of minutes. `board_snapshot.json` pins the normalization constants.
+See `release_verification.json` for export and official-function parity checks.
+
+## Limitations and license
+
+English base model intended for research. It can generate incorrect, repetitive,
+biased or inappropriate text and is not optimized for instruction following.
+Small changes in precision, context length, tokenizer handling or benchmark
+harness can change scores. Overall score and size-adjusted efficiency are distinct.
+
+Weights and GoLLeM implementation: Apache-2.0, following the source model's
+published license. See `LICENSE` and `NOTICE` for source and evaluation attribution.

board_snapshot.json

@@ -0,0 +1,1263 @@
+{
+  "paramLogMin": 2.9822712330395684,
+  "paramLogMax": 8.176091259055681,
+  "wikiMinLog": 0.6205764877251099,
+  "wikiMaxLog": 6.214608098422191,
+  "ranking": [
+    {
+      "name": "JugnuLM-110M-R2+",
+      "org": "altslate",
+      "params": "110M",
+      "blimp": 82.52,
+      "arc": 55.13,
+      "wiki": 1.8735,
+      "tokens": "25B",
+      "releaseDate": "2026-09-19",
+      "links": {
+        "card": "https://huggingface.co/altslate/JugnuLM-110M-R2plus"
+      },
+      "score": 79.1735740039279,
+      "efficiency": 80.20023332743875
+    },
+    {
+      "name": "GPT-X2-125M",
+      "org": "axiomiclabs",
+      "params": "125M",
+      "blimp": 81.28,
+      "arc": 57.07,
+      "aci": 54.53,
+      "wiki": 1.86,
+      "tokens": "75B",
+      "releaseDate": "2026-04-22",
+      "links": {
+        "card": "https://huggingface.co/AxiomicLabs/GPT-X2-125M"
+      },
+      "score": 79.45,
+      "efficiency": 80.05561878992458
+    },
+    {
+      "name": "Haidass-143M-v1",
+      "org": "DALab",
+      "params": "143M",
+      "blimp": 79.05,
+      "arc": 60.23,
+      "wiki": 1.8892,
+      "tokens": "100B",
+      "releaseDate": "2026-08-11",
+      "links": {
+        "card": "https://huggingface.co/DALabCommunity/Haidass-143M-v1"
+      },
+      "score": 79.66718100767177,
+      "efficiency": 79.8263615325101
+    },
+    {
+      "name": "JugnuLM-53M",
+      "org": "altslate",
+      "params": "53M",
+      "blimp": 78.14,
+      "arc": 51.43,
+      "wiki": 2.04,
+      "tokens": "12B",
+      "releaseDate": "2026-09-05",
+      "links": {
+        "card": "https://huggingface.co/altslate/JugnuLM-53M"
+      },
+      "score": 75.97290550501255,
+      "efficiency": 79.27738349200743
+    },
+    {
+      "name": "Glint-2",
+      "org": "glintresearch",
+      "params": "1.06M",
+      "blimp": 66.36,
+      "arc": 36.8,
+      "aci": 48.02,
+      "wiki": 3.09,
+      "tokens": "~300B",
+      "releaseDate": "2026-07-19",
+      "links": {
+        "card": "https://huggingface.co/Glint-Research/Glint-2"
+      },
+      "score": 64.6953799614223,
+      "efficiency": 78.09071110206555
+    },
+    {
+      "name": "Supra-50M-Instruct",
+      "org": "supralabs",
+      "params": "51.8M",
+      "blimp": 76.3,
+      "arc": 52.2,
+      "aci": 53.28,
+      "wiki": 2.56,
+      "tokens": "20B",
+      "releaseDate": "2026-05-21",
+      "links": {
+        "card": "https://huggingface.co/SupraLabs/Supra-50M-Instruct",
+        "base": "https://huggingface.co/SupraLabs/Supra-50M-Base"
+      },
+      "score": 74.26326441586103,
+      "efficiency": 77.5644874221056
+    },
+    {
+      "name": "Glint-1.3 (merged)",
+      "org": "glintresearch",
+      "params": "982K",
+      "blimp": 68.7,
+      "arc": 32.5,
+      "aci": 52.73,
+      "wiki": 3.08,
+      "tokens": "100B",
+      "releaseDate": "2026-05-13",
+      "links": {
+        "card": "https://huggingface.co/Glint-Research/Glint-1.3"
+      },
+      "score": 64.06136182059981,
+      "efficiency": 77.5301302449298
+    },
+    {
+      "name": "Ivme-Conversate-v3-Base",
+      "org": "ivmelabs",
+      "params": "24.79M",
+      "blimp": 78.49,
+      "arc": 39.02,
+      "wiki": 2.1362,
+      "tokens": "~15B",
+      "releaseDate": "2026-09-13",
+      "links": {
+        "card": "https://huggingface.co/IvmeLabs/Ivme-Conversate-v3-Base"
+      },
+      "score": 71.67833464673554,
+      "efficiency": 77.07312862611622
+    },
+    {
+      "name": "GoLLeM-v5 128M",
+      "org": "Fabryka AI",
+      "params": "122.8M",
+      "blimp": 79.09,
+      "arc": 53.24,
+      "wiki": 2.2538,
+      "tokens": "24.9B",
+      "releaseDate": "2026-09-28",
+      "links": {
+        "card": "https://huggingface.co/SlayerLab/gollem-v5-ckpts"
+      },
+      "score": 76.29901139538448,
+      "efficiency": 76.93725470597208
+    },
+    {
+      "name": "GPT-S-5M",
+      "org": "axiomiclabs",
+      "params": "5.16M",
+      "blimp": 72.27,
+      "arc": 35.69,
+      "aci": 53.07,
+      "wiki": 2.57,
+      "tokens": "25B",
+      "releaseDate": "2026-05-19",
+      "links": {
+        "card": "https://huggingface.co/AxiomicLabs/GPT-S-5M"
+      },
+      "score": 67.3933667970715,
+      "efficiency": 76.88794431148578
+    },
+    {
+      "name": "Supra-50M-Base",
+      "org": "supralabs",
+      "params": "51.8M",
+      "blimp": 76.3,
+      "arc": 46,
+      "aci": 53.53,
+      "wiki": 2.04,
+      "tokens": "20B",
+      "releaseDate": "2026-05-27",
+      "links": {
+        "card": "https://huggingface.co/SupraLabs/Supra-50M-Base"
+      },
+      "score": 73.5495721716792,
+      "efficiency": 76.81906943472619
+    },
+    {
+      "name": "MicroSupra-1k",
+      "org": "supralabs",
+      "params": "1K",
+      "blimp": 58.61,
+      "arc": 26.39,
+      "aci": 0.57,
+      "wiki": 11.27,
+      "tokens": "—",
+      "releaseDate": "2026-05-13",
+      "links": {
+        "card": "https://huggingface.co/SupraLabs/MicroSupra-1k"
+      },
+      "score": 50.93160731709393,
+      "efficiency": 76.31048511062035
+    },
+    {
+      "name": "Hydrion-v1-Base",
+      "org": "opengcm",
+      "params": "114.1M",
+      "blimp": 80.08,
+      "arc": 47.26,
+      "wiki": 2.04,
+      "tokens": "~2.5B",
+      "links": {},
+      "score": 75.22957217167921,
+      "efficiency": 76.0899885430591
+    },
+    {
+      "name": "GoLLeM-v5 64M",
+      "org": "Fabryka AI",
+      "params": "62.9M",
+      "blimp": 76.16,
+      "arc": 48.19,
+      "wiki": 2.3474,
+      "tokens": "24.9B",
+      "releaseDate": "2026-09-27",
+      "links": {
+        "card": "https://huggingface.co/SlayerLab/gollem-v5-ckpts"
+      },
+      "score": 73.39654671767353,
+      "efficiency": 76.06345060452743
+    },
+    {
+      "name": "Ivme-Conversate-v2-Base",
+      "org": "ivmelabs",
+      "params": "23.85M",
+      "blimp": 75.09,
+      "arc": 39.98,
+      "aci": 53.72,
+      "wiki": 2.225,
+      "tokens": "~12.85B",
+      "releaseDate": "2026-07-07",
+      "links": {
+        "card": "https://huggingface.co/IvmeLabs/Ivme-Conversate-v2-Base",
+        "demo": "https://huggingface.co/spaces/IvmeLabs/Ivme-Conversate-Demo"
+      },
+      "score": 70.62231190929478,
+      "efficiency": 76.05176278508509
+    },
+    {
+      "name": "GoLLeM-v5 64M (v1 recipe)",
+      "org": "Fabryka AI",
+      "params": "62.9M",
+      "blimp": 75.83,
+      "arc": 47.94,
+      "wiki": 2.3718,
+      "tokens": "13.1B",
+      "releaseDate": "2026-09-24",
+      "links": {
+        "card": "https://huggingface.co/SlayerLab/gollem-v5-ckpts"
+      },
+      "score": 73.14159516582735,
+      "efficiency": 75.79923524784323
+    },
+    {
+      "name": "Pollock 1.5 Mini LM 128M",
+      "org": "Fabryka AI",
+      "params": "127.43M",
+      "blimp": 78.497,
+      "arc": 47.7273,
+      "wiki": 1.9435,
+      "tokens": "21.50B",
+      "releaseDate": "2026-09-20",
+      "links": {
+        "card": "https://huggingface.co/SlayerLab/pollock-mini-lm-125m"
+      },
+      "score": 75.14642835516423,
+      "efficiency": 75.65875244066669
+    },
+    {
+      "name": "Hummingbird-V2",
+      "org": "juinron",
+      "params": "9.59M",
+      "blimp": 72.9,
+      "arc": 38.85,
+      "wiki": 2.9551,
+      "tokens": "10B",
+      "releaseDate": "2026-09-23",
+      "links": {
+        "card": "https://huggingface.co/juinron/Hummingbird-V2"
+      },
+      "score": 67.82470273244657,
+      "efficiency": 75.62254586041912
+    },
+    {
+      "name": "Michel-Nano-v2",
+      "org": "finnianx",
+      "params": "9.94M",
+      "blimp": 72.52,
+      "arc": 35.9,
+      "aci": 54.25,
+      "wiki": 2.46,
+      "tokens": "6.5B",
+      "releaseDate": "2026-06-13",
+      "links": {
+        "card": "https://huggingface.co/finnianx/michel-nano-v2"
+      },
+      "score": 67.80736215979108,
+      "efficiency": 75.50158990691395
+    },
+    {
+      "name": "min-spark",
+      "org": "minimalabs",
+      "params": "5.76M",
+      "blimp": 69.19,
+      "arc": 37.08,
+      "wiki": 2.7747,
+      "tokens": "10B",
+      "releaseDate": "2026-08-06",
+      "links": {
+        "card": "https://huggingface.co/MinimaLabs/min-spark"
+      },
+      "score": 66.37337572758646,
+      "efficiency": 75.41900255718414
+    },
+    {
+      "name": "GoLLeM-v5 32M",
+      "org": "Fabryka AI",
+      "params": "31.6M",
+      "blimp": 73.48,
+      "arc": 44.44,
+      "wiki": 2.5386,
+      "tokens": "16B",
+      "releaseDate": "2026-09-23",
+      "links": {
+        "card": "https://huggingface.co/SlayerLab/gollem-v5-ckpts"
+      },
+      "score": 70.78661838489879,
+      "efficiency": 75.39597759176215
+    },
+    {
+      "name": "TextModel-v1",
+      "org": "benchlabs",
+      "params": "122.7M",
+      "blimp": 80.31,
+      "arc": 49.71,
+      "aci": 55.044,
+      "wiki": 2.628,
+      "tokens": "1.64B",
+      "releaseDate": "2026-07-29",
+      "links": {
+        "card": "https://huggingface.co/TobiasLogic/TextModel-v1"
+      },
+      "score": 74.61371791371184,
+      "efficiency": 75.2404050492078
+    },
+    {
+      "name": "Gros-Michel-90m-Base-v2",
+      "org": "finnianx",
+      "params": "95M",
+      "blimp": 80.2,
+      "arc": 43.18,
+      "aci": 53.9,
+      "wiki": 2.08,
+      "tokens": "9B",
+      "releaseDate": "2026-07-06",
+      "links": {
+        "card": "https://huggingface.co/finnianx/Gros-Michel-90m-Base-v2"
+      },
+      "score": 73.79386500847113,
+      "efficiency": 75.20307015909175
+    },
+    {
+      "name": "ObsidianSmall-Base",
+      "org": "dreamw",
+      "params": "8.7M",
+      "blimp": 71.891,
+      "arc": 33.291,
+      "wiki": 2.467,
+      "tokens": "4B",
+      "releaseDate": "2026-07-31",
+      "links": {
+        "card": "https://huggingface.co/Dream-W/ObsidianSmall-Base"
+      },
+      "score": 66.71109716428224,
+      "efficiency": 74.65256171819846
+    },
+    {
+      "name": "GoLLeM-v5 16M",
+      "org": "Fabryka AI",
+      "params": "17.4M",
+      "blimp": 70.08,
+      "arc": 40.91,
+      "wiki": 2.6746,
+      "tokens": "16B",
+      "releaseDate": "2026-09-23",
+      "links": {
+        "card": "https://huggingface.co/SlayerLab/gollem-v5-ckpts"
+      },
+      "score": 68.16564952787132,
+      "efficiency": 74.30485232133755
+    },
+    {
+      "name": "Gros-Michel-90m-Base",
+      "org": "finnianx",
+      "params": "91.1M",
+      "blimp": 78.35,
+      "arc": 41.5,
+      "aci": 54.51,
+      "wiki": 2.07,
+      "tokens": "6.5B",
+      "releaseDate": "2026-06-28",
+      "links": {
+        "card": "https://huggingface.co/finnianx/Gros-Michel-90m-Base"
+      },
+      "score": 72.64591517652724,
+      "efficiency": 74.16051667816828
+    },
+    {
+      "name": "Supra-Mini-v6",
+      "org": "supralabs",
+      "params": "1.41M",
+      "blimp": 61.86,
+      "arc": 30.26,
+      "aci": 51.04,
+      "wiki": 3,
+      "tokens": "—",
+      "releaseDate": "2026-05-30",
+      "links": {
+        "card": "https://huggingface.co/SupraLabs/Supra-Mini-v6-1M"
+      },
+      "score": 61.19151293252804,
+      "efficiency": 73.13141194113591
+    },
+    {
+      "name": "CreekwardGoat-500K",
+      "org": "wonderfulmonkey",
+      "params": "500k",
+      "blimp": 60.61,
+      "arc": 30.35,
+      "aci": 49.58,
+      "wiki": 4.1228,
+      "tokens": "300M",
+      "releaseDate": "2026-07-16",
+      "links": {
+        "card": "https://huggingface.co/wonderfulmonkey/CreekwardGoat-500K"
+      },
+      "score": 58.91044477014074,
+      "efficiency": 72.95870925891221
+    },
+    {
+      "name": "Michel-Micro",
+      "org": "finnianx",
+      "params": "28.4M",
+      "blimp": 69.75,
+      "arc": 38.59,
+      "aci": 53.5,
+      "wiki": 2.3,
+      "tokens": "2.6B",
+      "releaseDate": "2026-06-09",
+      "links": {
+        "card": "https://huggingface.co/finnianx/michel-micro"
+      },
+      "score": 68.18143346822255,
+      "efficiency": 72.92550367504299
+    },
+    {
+      "name": "KeyLM-75M-Instruct",
+      "org": "minimalabs",
+      "params": "75M",
+      "blimp": 76.02,
+      "arc": 39.1,
+      "aci": 52.26,
+      "wiki": 2.17,
+      "tokens": "~18B",
+      "releaseDate": "2026-05-29",
+      "links": {
+        "card": "https://huggingface.co/MinimaLabs/KeyLM-75M-Instruct",
+        "base": "https://huggingface.co/MinimaLabs/KeyLM-75M"
+      },
+      "score": 70.78812412850577,
+      "efficiency": 72.83953798119279
+    },
+    {
+      "name": "Archaea-74M",
+      "org": "GODELEV",
+      "params": "74M",
+      "blimp": 74.91,
+      "arc": 39.06,
+      "aci": 52.58,
+      "wiki": 2.2,
+      "tokens": "~1.2B",
+      "releaseDate": "2026-06-01",
+      "links": {
+        "card": "https://huggingface.co/GODELEV/Archaea-74M"
+      },
+      "score": 70.32297626040025,
+      "efficiency": 72.40037555332115
+    },
+    {
+      "name": "Michel-Tiny",
+      "org": "finnianx",
+      "params": "55.7M",
+      "blimp": 74.08,
+      "arc": 37.33,
+      "aci": 52.9,
+      "wiki": 2.29,
+      "tokens": "1.3B",
+      "releaseDate": "2026-06-05",
+      "links": {
+        "card": "https://huggingface.co/finnianx/michel-tiny"
+      },
+      "score": 69.23073081506284,
+      "efficiency": 72.09813447714069
+    },
+    {
+      "name": "KeyLM-75M",
+      "org": "minimalabs",
+      "params": "75M",
+      "blimp": 76.1,
+      "arc": 35.65,
+      "aci": 52.27,
+      "wiki": 2.08,
+      "tokens": "~18B",
+      "releaseDate": "2026-05-29",
+      "links": {
+        "card": "https://huggingface.co/MinimaLabs/KeyLM-75M"
+      },
+      "score": 69.91719834180446,
+      "efficiency": 71.94337308488805
+    },
+    {
+      "name": "Supra-1.5-Instruct-exp",
+      "org": "supralabs",
+      "params": "51.8M",
+      "blimp": 67.4,
+      "arc": 45.9,
+      "aci": 53.17,
+      "wiki": 2.7,
+      "tokens": "23B",
+      "releaseDate": "2026-06-12",
+      "links": {
+        "card": "https://huggingface.co/SupraLabs/Supra-1.5-50M-Instruct-exp",
+        "demo": "https://huggingface.co/spaces/SupraLabs/Supra1.5-50M-Instruct-Demo"
+      },
+      "score": 68.87932797416606,
+      "efficiency": 71.94121899055959
+    },
+    {
+      "name": "Glint-1",
+      "org": "glintresearch",
+      "params": "1M",
+      "blimp": 61.2,
+      "arc": 32,
+      "wiki": 4.45,
+      "tokens": "100B",
+      "releaseDate": "2026-05-02",
+      "links": {
+        "card": "https://huggingface.co/Glint-Research/Glint-1"
+      },
+      "score": 59.20203385107235,
+      "efficiency": 71.60417983768008
+    },
+    {
+      "name": "Supra-Mini-v5",
+      "org": "supralabs",
+      "params": "7.87M",
+      "blimp": 63.5,
+      "arc": 34.4,
+      "aci": 52.56,
+      "wiki": 2.73,
+      "tokens": "—",
+      "releaseDate": "2026-05-16",
+      "links": {
+        "card": "https://huggingface.co/SupraLabs/Supra-Mini-v5-8M"
+      },
+      "score": 63.68015163197632,
+      "efficiency": 71.52774878766438
+    },
+    {
+      "name": "Michel-Nano",
+      "org": "finnianx",
+      "params": "5.96M",
+      "blimp": 65.23,
+      "arc": 33.38,
+      "aci": 54.25,
+      "wiki": 3.25,
+      "tokens": "1.1B",
+      "releaseDate": "2026-06-10",
+      "links": {
+        "card": "https://huggingface.co/finnianx/michel-nano"
+      },
+      "score": 62.87789324852722,
+      "efficiency": 71.35741139542888
+    },
+    {
+      "name": "Supra-Mini-v4",
+      "org": "supralabs",
+      "params": "2.62M",
+      "blimp": 60.7,
+      "arc": 31.5,
+      "aci": 50.42,
+      "wiki": 3.17,
+      "tokens": "—",
+      "releaseDate": "2026-05-14",
+      "links": {
+        "card": "https://huggingface.co/SupraLabs/Supra-Mini-v4-2M"
+      },
+      "score": 60.88973848518265,
+      "efficiency": 71.1934620365565
+    },
+    {
+      "name": "Byrne-86M-Base",
+      "org": "quazim0t0",
+      "params": "86M",
+      "blimp": 73.56,
+      "arc": 39.31,
+      "aci": 52.42,
+      "wiki": 2.3753,
+      "tokens": "—",
+      "releaseDate": "2026-06-18",
+      "links": {
+        "card": "https://huggingface.co/Quazim0t0/Byrne-86M-Base",
+        "base": "https://huggingface.co/Quazim0t0/Byrne-86M"
+      },
+      "score": 69.4994751776389,
+      "efficiency": 71.11587440455948
+    },
+    {
+      "name": "Supra-50M-Reasoning",
+      "org": "supralabs",
+      "params": "51.8M",
+      "blimp": 64.14,
+      "arc": 45.16,
+      "aci": 53.16,
+      "wiki": 2.6,
+      "tokens": "20B",
+      "releaseDate": "2026-06-04",
+      "links": {
+        "card": "https://huggingface.co/SupraLabs/Supra-50M-Reasoning",
+        "demo": "https://huggingface.co/spaces/SupraLabs/Supra-50M-Reasoning-Demo"
+      },
+      "score": 67.77087912849917,
+      "efficiency": 70.78349629651903
+    },
+    {
+      "name": "Escarda-86M-Base",
+      "org": "quazim0t0",
+      "params": "85.7M",
+      "blimp": 71.44,
+      "arc": 38.01,
+      "aci": 52.51,
+      "wiki": 2.2228,
+      "tokens": "~20B",
+      "releaseDate": "2026-05-22",
+      "links": {
+        "card": "https://huggingface.co/Quazim0t0/Escarda-86M-Base",
+        "discussion": "https://huggingface.co/spaces/Glint-Research/Tiny-ML-Leaderboard/discussions/21"
+      },
+      "score": 68.75487327030368,
+      "efficiency": 70.36399981030752
+    },
+    {
+      "name": "Dumb 1.2",
+      "org": "56m",
+      "params": "34.6M",
+      "blimp": 70.51,
+      "arc": 34.97,
+      "wiki": 2.7195,
+      "tokens": "1.1B",
+      "links": {},
+      "score": 66.22978068417677,
+      "efficiency": 70.29127820867504
+    },
+    {
+      "name": "PotentSulfurLM 500K",
+      "org": "mihaipopa",
+      "params": "587K",
+      "blimp": 59.01,
+      "arc": 27.06,
+      "aci": 51.4,
+      "wiki": 4.52,
+      "tokens": "~200M",
+      "releaseDate": "2026-05-27",
+      "links": {
+        "card": "https://huggingface.co/MihaiPopa-1/PotentSulfurLM-500K-Base"
+      },
+      "score": 56.7323639102486,
+      "efficiency": 69.88073138943594
+    },
+    {
+      "name": "Glint-0.4",
+      "org": "glintresearch",
+      "params": "1M",
+      "blimp": 58.5,
+      "arc": 31,
+      "wiki": 5.01,
+      "tokens": "10B",
+      "releaseDate": "2026-04-19",
+      "links": {
+        "card": "https://huggingface.co/Glint-Research/Glint-0.4"
+      },
+      "score": 57.26240121419818,
+      "efficiency": 69.25821644562583
+    },
+    {
+      "name": "kirk-tung",
+      "org": "rtc",
+      "params": "53.1M",
+      "blimp": 72.81,
+      "arc": 30.3,
+      "aci": 52.77,
+      "wiki": 2.38,
+      "tokens": "1.1B",
+      "releaseDate": "2026-06-09",
+      "links": {
+        "card": "https://huggingface.co/rtc2022/kirk-tung"
+      },
+      "score": 66.23436296685166,
+      "efficiency": 69.11003843213756
+    },
+    {
+      "name": "Supra-Mini-v3",
+      "org": "supralabs",
+      "params": "468K",
+      "blimp": 55.3,
+      "arc": 27.3,
+      "aci": 48.96,
+      "wiki": 4.49,
+      "tokens": "—",
+      "releaseDate": "2026-05-14",
+      "links": {
+        "card": "https://huggingface.co/SupraLabs/Supra-Mini-v3-0.5M"
+      },
+      "score": 55.61537817827218,
+      "efficiency": 69.03166322587406
+    },
+    {
+      "name": "CinnabarLM 1.5M",
+      "org": "mihaipopa",
+      "params": "1.71M",
+      "blimp": 60.51,
+      "arc": 26.68,
+      "aci": 51.28,
+      "wiki": 4.23,
+      "tokens": "~50M",
+      "releaseDate": "2026-05-19",
+      "links": {
+        "card": "https://huggingface.co/MihaiPopa-1/CinnabarLM-1.5M-Base"
+      },
+      "score": 57.50082074544151,
+      "efficiency": 68.25683128055626
+    },
+    {
+      "name": "CinnabarLM 1.4M",
+      "org": "mihaipopa",
+      "params": "1.51M",
+      "blimp": 60.7,
+      "arc": 24.58,
+      "aci": 51.57,
+      "wiki": 4.09,
+      "tokens": "~30M",
+      "releaseDate": "2026-05-19",
+      "links": {
+        "card": "https://huggingface.co/MihaiPopa-1/CinnabarLM-1.4M-Base"
+      },
+      "score": 57.06470724773393,
+      "efficiency": 68.03589444426096
+    },
+    {
+      "name": "CinnabarLM 4M",
+      "org": "mihaipopa",
+      "params": "4.23M",
+      "blimp": 62.87,
+      "arc": 27.36,
+      "aci": 50.99,
+      "wiki": 3.77,
+      "tokens": "~80M",
+      "releaseDate": "2026-05-05",
+      "links": {
+        "card": "https://huggingface.co/MihaiPopa-1/CinnabarLM-4M-Base"
+      },
+      "score": 59.20016492992388,
+      "efficiency": 68.0323450909838
+    },
+    {
+      "name": "Dumb-1.2-RC1",
+      "org": "56m",
+      "params": "34.6M",
+      "blimp": 64.87,
+      "arc": 34.13,
+      "aci": 54.16,
+      "wiki": 2.838,
+      "tokens": "210M",
+      "releaseDate": "2026-06-28",
+      "links": {
+        "card": "https://huggingface.co/56m/Dumb-1.2-RC1"
+      },
+      "score": 63.81563160872634,
+      "efficiency": 67.72908303685495
+    },
+    {
+      "name": "Byrne-86M",
+      "org": "quazim0t0",
+      "params": "86M",
+      "blimp": 70.33,
+      "arc": 34.68,
+      "aci": 52.16,
+      "wiki": 2.6839,
+      "tokens": "—",
+      "releaseDate": "2026-06-18",
+      "links": {
+        "card": "https://huggingface.co/Quazim0t0/Byrne-86M",
+        "base": "https://huggingface.co/Quazim0t0/Byrne-86M-Base"
+      },
+      "score": 66.15163269721914,
+      "efficiency": 67.69016874627584
+    },
+    {
+      "name": "Byrne-100M-Ultra-MC-sft",
+      "org": "quazim0t0",
+      "params": "113.9M",
+      "blimp": 78,
+      "arc": 26.5,
+      "wiki": 2.383,
+      "tokens": "~2B",
+      "releaseDate": "2026-08-19",
+      "links": {
+        "card": "https://huggingface.co/Quazim0t0/Byrne-100M-Ultra-MC"
+      },
+      "score": 66.69019002372889,
+      "efficiency": 67.45783133477885
+    },
+    {
+      "name": "SRLM-1M",
+      "org": "martico2432",
+      "params": "904k",
+      "blimp": 53.1,
+      "arc": 28.66,
+      "wiki": 4.455,
+      "tokens": "~16M",
+      "releaseDate": "2026-07-04",
+      "links": {
+        "card": "https://huggingface.co/Martico2432/srlm-1m"
+      },
+      "score": 55.382009072148584,
+      "efficiency": 67.2175930561982
+    },
+    {
+      "name": "Byrne-100M-Ultra-MC-dpo",
+      "org": "quazim0t0",
+      "params": "113.9M",
+      "blimp": 77.9,
+      "arc": 25.5,
+      "wiki": 2.385,
+      "tokens": "~2B",
+      "releaseDate": "2026-08-19",
+      "links": {
+        "card": "https://huggingface.co/Quazim0t0/Byrne-100M-Ultra-MC"
+      },
+      "score": 66.31852442080252,
+      "efficiency": 67.08188765331347
+    },
+    {
+      "name": "Dumb-1.2-Preview-0625",
+      "org": "56m",
+      "params": "34.6M",
+      "blimp": 64.05,
+      "arc": 32.79,
+      "aci": 54.44,
+      "wiki": 2.875,
+      "tokens": "~115-165M",
+      "releaseDate": "2026-06-25",
+      "links": {
+        "card": "https://huggingface.co/56m/Dumb-1.2-Preview-0625"
+      },
+      "score": 63.01844758825059,
+      "efficiency": 66.88301223323896
+    },
+    {
+      "name": "Byrne-100M-Ultra-MC-base",
+      "org": "quazim0t0",
+      "params": "113.9M",
+      "blimp": 81.1,
+      "arc": 20.5,
+      "wiki": 2.308,
+      "tokens": "~2B",
+      "releaseDate": "2026-08-19",
+      "links": {
+        "card": "https://huggingface.co/Quazim0t0/Byrne-100M-Ultra-MC"
+      },
+      "score": 65.91407674024059,
+      "efficiency": 66.67278455420127
+    },
+    {
+      "name": "Echo88-150M-Instruct",
+      "org": "exnivo",
+      "params": "150M",
+      "blimp": 72.67,
+      "arc": 31.4,
+      "aci": 50.83,
+      "wiki": 2.45,
+      "tokens": "~1.47B",
+      "releaseDate": "2026-05-05",
+      "links": {
+        "card": "https://huggingface.co/exnivo/Echo88-150M-Instruct"
+      },
+      "score": 66.38163401278912,
+      "efficiency": 66.38163401278912
+    },
+    {
+      "name": "Supra-Mini-v2",
+      "org": "supralabs",
+      "params": "168K",
+      "blimp": 53.5,
+      "arc": 26.8,
+      "aci": 48.95,
+      "wiki": 7.79,
+      "tokens": "—",
+      "releaseDate": "2026-05-12",
+      "links": {
+        "card": "https://huggingface.co/SupraLabs/Supra-Mini-v2-0.1M"
+      },
+      "score": 51.565520922818735,
+      "efficiency": 66.21356504337999
+    },
+    {
+      "name": "Ivme-Conversate-v1-Base",
+      "org": "ivmelabs",
+      "params": "22.03M",
+      "blimp": 61.4,
+      "arc": 30.85,
+      "aci": 53.66,
+      "wiki": 3.14,
+      "tokens": "~1.57B",
+      "releaseDate": "2026-06-05",
+      "links": {
+        "card": "https://huggingface.co/IvmeLabs/Ivme-Conversate-22M-Base"
+      },
+      "score": 60.96306546788355,
+      "efficiency": 65.8522330672023
+    },
+    {
+      "name": "Glimmer 1",
+      "org": "glintresearch",
+      "params": "11.9K",
+      "blimp": 52.43,
+      "arc": 25.46,
+      "aci": 50,
+      "wiki": 14.73,
+      "tokens": "500K",
+      "releaseDate": "2026-06-16",
+      "links": {
+        "card": "https://huggingface.co/Glint-Research/Glimmer-1-Base"
+      },
+      "score": 46.966205163181236,
+      "efficiency": 65.50622039282746
+    },
+    {
+      "name": "Aurora-80K",
+      "org": "auroraairesearch",
+      "params": "80K",
+      "blimp": 52.31,
+      "arc": 26.05,
+      "wiki": 9.78,
+      "tokens": "80M",
+      "releaseDate": "2026-08-20",
+      "links": {
+        "card": "https://huggingface.co/AuroraAI-Research/Aurora-80K"
+      },
+      "score": 49.563250998988714,
+      "efficiency": 65.17994379492052
+    },
+    {
+      "name": "MicroLM2-1M",
+      "org": "cromia",
+      "params": "1.71M",
+      "blimp": 54.2,
+      "arc": 27.4,
+      "aci": 49.54,
+      "wiki": 4.82,
+      "tokens": "~4.5B",
+      "releaseDate": "2026-05-22",
+      "links": {
+        "card": "https://huggingface.co/CromIA/MicroLM2-1M"
+      },
+      "score": 54.85944428751184,
+      "efficiency": 65.12136321418728
+    },
+    {
+      "name": "TinyMoE-100m-2x8-retrained",
+      "org": "flamef0x",
+      "params": "99.8M",
+      "blimp": 66.01,
+      "arc": 33.88,
+      "aci": 51.69,
+      "wiki": 2.879,
+      "tokens": "—",
+      "releaseDate": "2026-07-05",
+      "links": {
+        "card": "https://huggingface.co/FlameF0X/TinyMoE-100m-2x8-retrained"
+      },
+      "score": 64.02682960753184,
+      "efficiency": 65.11757145717472
+    },
+    {
+      "name": "DistillSupra-0.2M",
+      "org": "supralabs",
+      "params": "0.2M",
+      "blimp": 51.84,
+      "arc": 26.77,
+      "aci": 49.26,
+      "wiki": 8.0696,
+      "tokens": "1.5M",
+      "releaseDate": "2026-05-15",
+      "links": {
+        "card": "https://huggingface.co/SupraLabs/DistillSupra-0.2M"
+      },
+      "score": 50.792064507542136,
+      "efficiency": 64.8501466534245
+    },
+    {
+      "name": "Cosmos-T2-Accelerate-Beta2",
+      "org": "wop",
+      "params": "9.96M",
+      "blimp": 69,
+      "arc": 28,
+      "aci": 50.21,
+      "wiki": 6.72,
+      "tokens": "~10M",
+      "releaseDate": "2026-06-03",
+      "links": {
+        "card": "https://huggingface.co/wop/Cosmos-T2-Accelerate-Beta2"
+      },
+      "score": 58.012606314477104,
+      "efficiency": 64.59052961312662
+    },
+    {
+      "name": "Ivme-Conversate-N-v1-Base",
+      "org": "ivmelabs",
+      "params": "9.55M",
+      "blimp": 59.24,
+      "arc": 26.81,
+      "wiki": 4.95,
+      "tokens": "~836M",
+      "releaseDate": "2026-08-08",
+      "links": {
+        "card": "https://huggingface.co/IvmeLabs/Ivme-Conversate-S-v1-Base"
+      },
+      "score": 56.18419403051056,
+      "efficiency": 62.653539676937676
+    },
+    {
+      "name": "Cosmos-T2-Accelerate-beta",
+      "org": "wop",
+      "params": "5.03M",
+      "blimp": 54.3,
+      "arc": 26.3,
+      "aci": 48.4,
+      "wiki": 6.26,
+      "tokens": "~22M",
+      "releaseDate": "2026-06-02",
+      "links": {
+        "card": "https://huggingface.co/wop/Cosmos-T2-Accelerate-beta"
+      },
+      "score": 52.96846121101516,
+      "efficiency": 60.48732290167559
+    },
+    {
+      "name": "StorySupra-10M",
+      "org": "supralabs",
+      "params": "12.6M",
+      "blimp": 61.47,
+      "arc": 28.45,
+      "aci": 49.86,
+      "wiki": 8.76,
+      "tokens": "—",
+      "releaseDate": "2026-05-15",
+      "links": {
+        "card": "https://huggingface.co/SupraLabs/StorySupra-10M"
+      },
+      "score": 54.0729003655732,
+      "efficiency": 59.672568692026175
+    },
+    {
+      "name": "Cosmos-T2-Accelerate-Preview",
+      "org": "wop",
+      "params": "9.96M",
+      "blimp": 56.95,
+      "arc": 26.8,
+      "aci": 50.71,
+      "wiki": 6.73,
+      "tokens": "462M",
+      "releaseDate": "2026-06-01",
+      "links": {
+        "card": "https://huggingface.co/wop/Cosmos-T2-Accelerate-Preview",
+        "demo": "https://huggingface.co/spaces/wop/Cosmos-T2-Chat"
+      },
+      "score": 53.58707907863565,
+      "efficiency": 59.663201466020375
+    },
+    {
+      "name": "Cosmos-T2A-low",
+      "org": "wop",
+      "params": "9.96M",
+      "blimp": 47.8,
+      "arc": 31,
+      "aci": 49.54,
+      "wiki": 5.31,
+      "tokens": "~46.7M",
+      "releaseDate": "2026-06-04",
+      "links": {
+        "card": "https://huggingface.co/wop/Cosmos-T2A-low"
+      },
+      "score": 53.349199024172016,
+      "efficiency": 59.398348709381324
+    },
+    {
+      "name": "Glint-0.3",
+      "org": "glintresearch",
+      "params": "1M",
+      "blimp": 47.3,
+      "arc": 25.5,
+      "wiki": 7.87,
+      "tokens": "~100M",
+      "releaseDate": "2026-04-04",
+      "links": {
+        "card": "https://huggingface.co/Glint-Research/Glint-0.3"
+      },
+      "score": 49.00463935440382,
+      "efficiency": 59.2705483402886
+    },
+    {
+      "name": "TinyMoE-100m-2x8",
+      "org": "flamef0x",
+      "params": "99.8M",
+      "blimp": 61.13,
+      "arc": 25.88,
+      "aci": 50.66,
+      "wiki": 3.878,
+      "tokens": "~625M",
+      "releaseDate": "2026-06-15",
+      "links": {
+        "card": "https://huggingface.co/FlameF0X/TinyMoE-100m-2x8"
+      },
+      "score": 57.95852986999959,
+      "efficiency": 58.945893986893935
+    },
+    {
+      "name": "Supra-Mini-0.1M",
+      "org": "supralabs",
+      "params": "117K",
+      "blimp": 51.77,
+      "arc": 26.39,
+      "aci": 52.23,
+      "wiki": 25.17,
+      "tokens": "500M",
+      "releaseDate": "2026-05-18",
+      "links": {
+        "card": "https://huggingface.co/SupraLabs/Supra-Mini-0.1M"
+      },
+      "score": 43.863715881997,
+      "efficiency": 56.987416612177725
+    },
+    {
+      "name": "Blink-1-Instruct",
+      "org": "glintresearch",
+      "params": "1.09K",
+      "blimp": 52.46,
+      "arc": 26.8,
+      "wiki": 70.44,
+      "tokens": "100B",
+      "releaseDate": "2026-06-21",
+      "links": {
+        "card": "https://huggingface.co/Glint-Research/Blink-1-Base",
+        "base": "https://huggingface.co/Glint-Research/Blink-1-Base"
+      },
+      "score": 38.09820128775882,
+      "efficiency": 56.94501186635348
+    },
+    {
+      "name": "Blink-1-Base",
+      "org": "glintresearch",
+      "params": "1.09K",
+      "blimp": 52.84,
+      "arc": 26.6,
+      "wiki": 71.35,
+      "tokens": "100B",
+      "releaseDate": "2026-06-21",
+      "links": {
+        "card": "https://huggingface.co/Glint-Research/Blink-1-Base"
+      },
+      "score": 38.0817146487351,
+      "efficiency": 56.920369446944996
+    },
+    {
+      "name": "peacebell-v1-148M",
+      "org": "wayneworkman",
+      "params": "148.55M",
+      "blimp": 56.26,
+      "arc": 27.36,
+      "wiki": 6.8841,
+      "tokens": "~35B",
+      "releaseDate": "2026-09-19",
+      "links": {
+        "card": "https://huggingface.co/wayneworkman2012/peacebell-v1-148M"
+      },
+      "score": 53.40884446359834,
+      "efficiency": 53.43053473259743
+    },
+    {
+      "name": "Cosmos-T2-80M-Test",
+      "org": "wop",
+      "params": "87.60M",
+      "blimp": 57.61,
+      "arc": 25,
+      "aci": 49.5,
+      "wiki": 11.24,
+      "tokens": "~18M",
+      "releaseDate": "2026-05-31",
+      "links": {
+        "card": "https://huggingface.co/wop/Cosmos-T2-80M-Test"
+      },
+      "score": 50.15082355189342,
+      "efficiency": 51.278566526214455
+    },
+    {
+      "name": "Cosmos-T-80M",
+      "org": "wop",
+      "params": "79.7M",
+      "blimp": 50.47,
+      "arc": 27.82,
+      "aci": 49.77,
+      "wiki": 12.42,
+      "tokens": "~21M",
+      "releaseDate": "2026-05-30",
+      "links": {
+        "card": "https://huggingface.co/wop/Cosmos-T-80M"
+      },
+      "score": 48.11596794513753,
+      "efficiency": 49.38807879600606
+    },
+    {
+      "name": "Glint-0.2",
+      "org": "glintresearch",
+      "params": "1M",
+      "blimp": 49.8,
+      "arc": 27,
+      "wiki": 636.4,
+      "tokens": "~100M",
+      "releaseDate": "2026-03-22",
+      "links": {
+        "card": "https://huggingface.co/Glint-Research/Glint-0.2"
+      },
+      "score": 25.599999999999998,
+      "efficiency": 30.962905910561155
+    },
+    {
+      "name": "Glint-0.1",
+      "org": "glintresearch",
+      "params": "1M",
+      "blimp": 46.7,
+      "arc": 21,
+      "wiki": 4106963.13,
+      "tokens": "~100M",
+      "releaseDate": "2026-03-09",
+      "links": {
+        "card": "https://huggingface.co/Glint-Research/Glint-0.1"
+      },
+      "score": 22.566666666666666,
+      "efficiency": 27.294124090429563
+    }
+  ]
+}

config.json

@@ -0,0 +1,17 @@
+{
+  "vocab": 12288,
+  "n_layer": 19,
+  "n_embd": 768,
+  "n_head": 12,
+  "block": 1024,
+  "ffn_mult": 2.8125,
+  "norm": "rmsnorm",
+  "norm_eps": 1e-06,
+  "pos": "rope",
+  "rope_theta": 100000.0,
+  "ffn": "swiglu",
+  "value_residual": true,
+  "qk_norm": true,
+  "logit_cap": 0.0,
+  "z_loss": 0.0
+}

evaluate_glint.py

@@ -0,0 +1,39 @@
+"""Full pinned GLINT-style evaluation of the released safetensors weights."""
+import argparse,json,math
+from pathlib import Path
+import torch
+import pyarrow.parquet as pq
+from huggingface_hub import hf_hub_download,list_repo_files
+from load_model import load_model
+import glint_metrics as metrics
+
+def main():
+    p=argparse.ArgumentParser();p.add_argument('--model-dir',default='.');p.add_argument('--out',default='glint-results.json');a=p.parse_args()
+    root=Path(a.model_dir);torch.set_num_threads(4)
+    model,tok=load_model(root,'cuda')
+    reference=json.loads((root/'evaluation.json').read_text())
+    revisions=reference['dataset_revisions']
+    files={r:list_repo_files(r,repo_type='dataset',revision=rev) for r,rev in revisions.items()}
+    def rows(repo,config,split):
+        fs=[f for f in files[repo] if f.endswith('.parquet') and f.split('/')[0]==config and (split in Path(f).name or '/'+split+'/' in '/'+f)]
+        result=[]
+        for f in sorted(fs):result.extend(pq.read_table(hf_hub_download(repo,f,repo_type='dataset',revision=revisions[repo])).to_pylist())
+        if not result:raise ValueError(f'No data: {repo}/{config}/{split}')
+        return result
+    metrics._rows=rows
+    @torch.inference_mode()
+    def logits(x):
+        with torch.autocast('cuda',dtype=torch.bfloat16):return model(x)[0].float()
+    text=' '.join(r['text'] for r in rows('Salesforce/wikitext','wikitext-2-raw-v1','test')).strip()
+    ppl=metrics.compute_perplexity(logits,tok,text,'cuda')
+    byte_ppl=math.exp(math.log(ppl)*len(tok.encode(text).ids)/len(text.encode('utf-8')))
+    result={'parameters':sum(p.numel() for p in model.parameters()),'dataset_revisions':revisions,'protocol':reference['protocol'],'wikitext2_token_ppl':ppl,'wikitext2_byte_ppl':byte_ppl}
+    result.update(metrics.evaluate_blimp(logits,tok,'cuda'));result.update(metrics.evaluate_arc_easy(logits,tok,'cuda'))
+    assert result['blimp_n']==67000 and result['arc_n']==2376
+    board=json.loads((root/'board_snapshot.json').read_text())
+    wiki=100*max(0,min(1,1-(math.log(min(byte_ppl,500))-board['wikiMinLog'])/(board['wikiMaxLog']-board['wikiMinLog'])))
+    result['overall_score_fixed_board_snapshot']=(result['blimp_acc']+result['arc_easy_acc']+wiki)/3
+    bonus=1+.5*max(0,min(1,(board['paramLogMax']-math.log10(result['parameters']))/(board['paramLogMax']-board['paramLogMin'])))
+    result['efficiency_fixed_board_snapshot']=result['overall_score_fixed_board_snapshot']*bonus
+    Path(a.out).write_text(json.dumps(result,indent=2));print(json.dumps(result,indent=2))
+if __name__=='__main__':main()

evaluation.json

@@ -0,0 +1,46 @@
+{
+  "step": 76294,
+  "parameters": 148910738,
+  "dataset_revisions": {
+    "Salesforce/wikitext": "b08601e04326c79dfdd32d625aee71d232d685c3",
+    "nyu-mll/blimp": "877fba0801ffb7cbd8c39c1ff314a46f053f6036",
+    "allenai/ai2_arc": "210d026faf9955653af8916fad021475a3f00453"
+  },
+  "continuation_tokens": 20000014336,
+  "provenance": {
+    "source_repo": "SlayerLab/gollem-v5-ckpts",
+    "source_revision": "963440ce6ab4ada7e95da4c1faa28ebb00c082d4",
+    "source_file": "final_128m_16x768/ckpt_760k.pt",
+    "source_sha256": "95d8b43fcd9c5fa06566da13dd03856b00300bad6f37fb4dc9442c18be9ac372",
+    "source_steps": 760000,
+    "source_training_tokens": 24903680000,
+    "parameters": 148910738,
+    "checks": [
+      {
+        "length": 16,
+        "max_absolute_logit_error": 0.0
+      },
+      {
+        "length": 128,
+        "max_absolute_logit_error": 0.0
+      },
+      {
+        "length": 256,
+        "max_absolute_logit_error": 0.0
+      }
+    ],
+    "optimizer": "reset for continued pretraining",
+    "method": "FFN expansion with zero new down columns; append three identity residual blocks"
+  },
+  "protocol": "GoLLeM r6 GLINT parity; bf16 forward, fp32 logprob; context 256",
+  "wikitext2_token_ppl": 21.452430290724198,
+  "bytes_per_token": 3.8604970807412586,
+  "wikitext2_byte_ppl": 2.2125733814416444,
+  "blimp_acc": 79.47,
+  "blimp_n": 67000,
+  "arc_easy_acc": 54.97,
+  "arc_n": 2376,
+  "efficiency_fixed_board_snapshot": 77.1358484476474,
+  "leader_reported_efficiency_snapshot": 80.20023332743875,
+  "comparison_caveat": "Competitor scores are self-reported, not independently re-evaluated under this protocol"
+}

generate.py

@@ -0,0 +1,28 @@
+"""Simple causal completion CLI (no KV cache)."""
+import argparse
+from contextlib import nullcontext
+import torch
+from load_model import load_model
+
+def main():
+    p=argparse.ArgumentParser()
+    p.add_argument('--model-dir',default='.')
+    p.add_argument('--prompt',default='The scientific method is')
+    p.add_argument('--max-new-tokens',type=int,default=64)
+    p.add_argument('--temperature',type=float,default=0.0)
+    p.add_argument('--device',default='cuda' if torch.cuda.is_available() else 'cpu')
+    args=p.parse_args();torch.set_num_threads(4)
+    model,tok=load_model(args.model_dir,args.device)
+    ids=tok.encode(args.prompt).ids
+    if not ids:raise ValueError('Prompt must encode to at least one token')
+    eos=tok.token_to_id('<|endoftext|>')
+    with torch.inference_mode():
+        for _ in range(args.max_new_tokens):
+            x=torch.tensor([ids[-model.block:]],device=args.device)
+            ctx=torch.autocast('cuda',dtype=torch.bfloat16) if args.device.startswith('cuda') else nullcontext()
+            with ctx:logits=model(x)[0][0,-1].float()
+            nxt=int(logits.argmax()) if args.temperature<=0 else int(torch.multinomial(torch.softmax(logits/args.temperature,dim=-1),1))
+            ids.append(nxt)
+            if nxt==eos:break
+    print(tok.decode(ids,skip_special_tokens=True))
+if __name__=='__main__':main()

glint_metrics.py

@@ -0,0 +1,118 @@
+"""GLINT-1.3 likelihood protocol adapted to a logits callable; see NOTICE."""
+import math
+import torch
+import torch.nn.functional as F
+import numpy as np
+
+def _rows(*args):
+    raise RuntimeError("Use evaluate_glint.py to install the pinned dataset loader")
+
+def tokenize_many(tokenizer, texts, max_length=256):
+    all_ids = []
+    for text in texts:
+        ids = tokenizer.encode(text).ids
+        ids = [i for i in ids if i < tokenizer.get_vocab_size()]
+        if len(ids) > max_length:
+            ids = ids[:max_length]
+        all_ids.append(ids)
+    return all_ids
+
+def batch_log_probs(logits_fn, tokenizer, texts, device, max_length=256, batch_size=128):
+    all_ids = tokenize_many(tokenizer, texts, max_length)
+    results = [-float("inf")] * len(all_ids)
+    with torch.inference_mode():
+        for start in range(0, len(all_ids), batch_size):
+            end = min(start + batch_size, len(all_ids))
+            batch = all_ids[start:end]
+            batch_indices = [j for j in range(start, end) if len(batch[j-start]) >= 2]
+            batch_seqs = [batch[j-start] for j in range(start, end) if len(batch[j-start]) >= 2]
+            if not batch_seqs:
+                continue
+            max_len = max(len(s) for s in batch_seqs)
+            B = len(batch_seqs)
+            padded_np = np.zeros((B, max_len - 1), dtype=np.int64)
+            targets_np = np.zeros((B, max_len - 1), dtype=np.int64)
+            mask_np = np.zeros((B, max_len - 1), dtype=bool)
+            for j, ids in enumerate(batch_seqs):
+                padded_np[j, :len(ids)-1] = ids[:-1]
+                targets_np[j, :len(ids)-1] = ids[1:]
+                mask_np[j, :len(ids)-1] = True
+            padded = torch.from_numpy(padded_np).to(device)
+            targets = torch.from_numpy(targets_np).to(device)
+            mask = torch.from_numpy(mask_np).to(device)
+            logits = logits_fn(padded)
+            log_probs = F.log_softmax(logits, dim=-1)
+            log_probs_flat = log_probs.view(-1, logits.size(-1))
+            targets_flat = targets.view(-1)
+            gathered = log_probs_flat[torch.arange(targets_flat.size(0), device=device), targets_flat]
+            gathered = gathered.view(B, -1)
+            gathered[~mask] = 0.0
+            sums = gathered.sum(dim=-1).tolist()
+            for bi, val in zip(batch_indices, sums):
+                results[bi] = val
+    return results
+
+def compute_perplexity(logits_fn, tokenizer, text, device, max_length=256):
+    ids = tokenizer.encode(text).ids
+    ids = [i for i in ids if i < tokenizer.get_vocab_size()]
+    if len(ids) < 2:
+        return float("inf")
+    nll = 0.0; n_tokens = 0
+    for i in range(0, len(ids) - 1, max_length):
+        chunk = ids[i:i + max_length + 1]
+        if len(chunk) < 2:
+            continue
+        inputs = torch.tensor([chunk[:-1]], device=device)
+        targets = torch.tensor([chunk[1:]], device=device)
+        with torch.no_grad():
+            logits = logits_fn(inputs)
+        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), reduction="sum")
+        nll += loss.item(); n_tokens += targets.numel()
+    return math.exp(nll / n_tokens) if n_tokens > 0 else float("inf")
+
+BLIMP_CONFIGS = [
+    "adjunct_island","anaphor_gender_agreement","anaphor_number_agreement","animate_subject_passive",
+    "animate_subject_trans","causative","complex_NP_island","coordinate_structure_constraint_complex_left_branch",
+    "coordinate_structure_constraint_object_extraction","determiner_noun_agreement_1","determiner_noun_agreement_2",
+    "determiner_noun_agreement_irregular_1","determiner_noun_agreement_irregular_2","determiner_noun_agreement_with_adj_2",
+    "determiner_noun_agreement_with_adj_irregular_1","determiner_noun_agreement_with_adj_irregular_2",
+    "determiner_noun_agreement_with_adjective_1","distractor_agreement_relational_noun",
+    "distractor_agreement_relative_clause","drop_argument","ellipsis_n_bar_1","ellipsis_n_bar_2",
+    "existential_there_object_raising","existential_there_quantifiers_1","existential_there_quantifiers_2",
+    "existential_there_subject_raising","expletive_it_object_raising","inchoative","intransitive",
+    "irregular_past_participle_adjectives","irregular_past_participle_verbs","irregular_plural_subject_verb_agreement_1",
+    "irregular_plural_subject_verb_agreement_2","left_branch_island_echo_question","left_branch_island_simple_question",
+    "matrix_question_npi_licensor_present","npi_present_1","npi_present_2","only_npi_licensor_present","only_npi_scope",
+    "passive_1","passive_2","principle_A_c_command","principle_A_case_1","principle_A_case_2","principle_A_domain_1",
+    "principle_A_domain_2","principle_A_domain_3","principle_A_reconstruction","regular_plural_subject_verb_agreement_1",
+    "regular_plural_subject_verb_agreement_2","sentential_negation_npi_licensor_present","sentential_negation_npi_scope",
+    "sentential_subject_island","superlative_quantifiers_1","superlative_quantifiers_2","tough_vs_raising_1",
+    "tough_vs_raising_2","transitive","wh_island","wh_questions_object_gap","wh_questions_subject_gap",
+    "wh_questions_subject_gap_long_distance","wh_vs_that_no_gap","wh_vs_that_no_gap_long_distance",
+    "wh_vs_that_with_gap","wh_vs_that_with_gap_long_distance",
+]
+
+def evaluate_blimp(logits_fn, tokenizer, device):
+    import os
+    ds = []
+    for c in BLIMP_CONFIGS:
+        ds.extend(_rows("nyu-mll/blimp", c, "train"))
+    assert len(ds) == 67000, f"BLiMP: {len(ds)} par, oczekiwano 67000 (67 fenomenow x 1000)"
+    good = batch_log_probs(logits_fn, tokenizer, [e["sentence_good"] for e in ds], device)
+    bad  = batch_log_probs(logits_fn, tokenizer, [e["sentence_bad"]  for e in ds], device)
+    correct = sum(1 for g, b in zip(good, bad) if g > b)
+    return {"blimp_acc": round(correct/len(ds)*100, 2), "blimp_n": len(ds)}
+
+def evaluate_arc_easy(logits_fn, tokenizer, device):
+    ds = _rows("allenai/ai2_arc", "ARC-Easy", "test")
+    correct = 0; total = 0
+    for ex in ds:
+        q = ex["question"]; ch = ex["choices"]
+        full = [q + " " + t for t in ch["text"]]
+        lps = batch_log_probs(logits_fn, tokenizer, full, device, batch_size=4)
+        lpq = batch_log_probs(logits_fn, tokenizer, [q], device)[0]
+        best = max(range(len(lps)), key=lambda j: lps[j] - lpq)
+        if ch["label"][best] == ex["answerKey"]:
+            correct += 1
+        total += 1
+    return {"arc_easy_acc": round(correct/total*100, 2), "arc_n": total}

load_model.py

@@ -0,0 +1,15 @@
+"""Load the local safetensors release without Transformers remote code."""
+import json
+from pathlib import Path
+from types import SimpleNamespace
+import torch
+from safetensors.torch import load_model as load_safetensors
+from tokenizers import Tokenizer
+from modeling_gollem import GPT
+
+def load_model(directory='.',device='cpu'):
+    directory=Path(directory)
+    cfg=SimpleNamespace(**json.loads((directory/'config.json').read_text()))
+    model=GPT(cfg.vocab,cfg.n_layer,cfg.n_embd,cfg.n_head,cfg.block,cfg)
+    load_safetensors(model,str(directory/'model.safetensors'),strict=True)
+    return model.eval().to(device),Tokenizer.from_file(str(directory/'tokenizer.json'))

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05e2a496967bdd36da3e1bdd4fb8b10277b92a319b1f5f1e61062454eecc7b2e
+size 595664120

modeling_gollem.py

@@ -0,0 +1,140 @@
+"""GoLLeM inference architecture, extracted unchanged from the pinned r6 trainer.
+Source: SlayerLab/gollem-v5-ckpts; Apache-2.0. See README and NOTICE.
+"""
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+class RMSNorm(nn.Module):
+    """Qwen3-style RMSNorm (fp32-compute dla stabilnosci). 1D weight -> AdamW w split-Muon."""
+    def __init__(self, d, eps=1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(d))
+        self.eps = eps
+
+    def forward(self, x):
+        return x * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps).to(x.dtype) * self.weight
+
+
+def make_norm(d, cfg):
+    return RMSNorm(d, cfg.norm_eps) if cfg.norm == "rmsnorm" else nn.LayerNorm(d)
+
+
+def apply_rope(x, base=100000.0):
+    """Parameter-free RoPE na [B,H,T,D] (interleaved-conv, port z qwen_model.py). Train==eval
+    MUSZA uzywac tej samej konwencji (self-contained eval -> spojne)."""
+    _, _, T, dim = x.shape
+    pos = torch.arange(T, device=x.device, dtype=torch.float32)
+    freq = 1.0 / (base ** (torch.arange(0, dim, 2, device=x.device, dtype=torch.float32) / dim))
+    ang = torch.outer(pos, freq)
+    cos, sin = ang.cos().to(x.dtype)[None, None], ang.sin().to(x.dtype)[None, None]
+    even, odd = x[..., ::2], x[..., 1::2]
+    return torch.stack((even * cos - odd * sin, even * sin + odd * cos), dim=-1).flatten(-2)
+
+
+class SwiGLU(nn.Module):
+    """Qwen3 gated-MLP: down(silu(gate(x))*up(x)). 3x 2D bez-bias -> wszystkie do Muon."""
+    def __init__(self, d, hidden):
+        super().__init__()
+        self.gate = nn.Linear(d, hidden, bias=False)
+        self.up = nn.Linear(d, hidden, bias=False)
+        self.down = nn.Linear(hidden, d, bias=False)
+
+    def forward(self, x):
+        return self.down(F.silu(self.gate(x)) * self.up(x))
+
+
+class Block(nn.Module):
+    def __init__(self, d, nh, block, cfg, is_first=False):
+        super().__init__()
+        self.ln1 = make_norm(d, cfg)
+        self.ln2 = make_norm(d, cfg)
+        self.qkv = nn.Linear(d, 3 * d)
+        self.proj = nn.Linear(d, d)
+        if cfg.ffn == "swiglu":
+            self.mlp = SwiGLU(d, int(round(cfg.ffn_mult * d)))
+        else:
+            self.mlp = nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(), nn.Linear(4 * d, d))
+        self.nh = nh
+        self.d = d
+        self.cfg = cfg
+        self.is_first = is_first
+        if cfg.value_residual and not is_first:
+            self.vr_lambda = nn.Parameter(torch.zeros(1))
+        if cfg.qk_norm:
+            hd = d // nh
+            self.q_norm = RMSNorm(hd, cfg.norm_eps)
+            self.k_norm = RMSNorm(hd, cfg.norm_eps)
+
+    def forward(self, x, v0=None):
+        B, T, D = x.size()
+        h = self.ln1(x)
+        q, k, v = self.qkv(h).split(self.d, dim=2)
+        hd = D // self.nh
+        q = q.view(B, T, self.nh, hd).transpose(1, 2)
+        k = k.view(B, T, self.nh, hd).transpose(1, 2)
+        v = v.view(B, T, self.nh, hd).transpose(1, 2)
+        if self.cfg.qk_norm:
+            q = self.q_norm(q)
+            k = self.k_norm(k)
+        if self.cfg.pos == "rope":
+            q = apply_rope(q, self.cfg.rope_theta)
+            k = apply_rope(k, self.cfg.rope_theta)
+        if self.cfg.value_residual:
+            if self.is_first:
+                v0 = v
+            else:
+                v = v + self.vr_lambda * v0
+        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
+        y = y.transpose(1, 2).contiguous().view(B, T, D)
+        x = x + self.proj(y)
+        x = x + self.mlp(self.ln2(x))
+        return x, v0
+
+
+class GPT(nn.Module):
+    def __init__(self, vocab, n_layer, n_embd, n_head, block, cfg):
+        super().__init__()
+        self.cfg = cfg
+        self.tok = nn.Embedding(vocab, n_embd)
+        self.use_rope = cfg.pos == "rope"
+        if not self.use_rope:
+            self.pos = nn.Embedding(block, n_embd)
+        self.blocks = nn.ModuleList([Block(n_embd, n_head, block, cfg, is_first=(i == 0)) for i in range(n_layer)])
+        self.lnf = make_norm(n_embd, cfg)
+        self.head = nn.Linear(n_embd, vocab, bias=False)
+        self.head.weight = self.tok.weight  # tie
+        self.block = block
+        self.apply(self._init)
+
+    def _init(self, m):
+        if isinstance(m, nn.Linear):
+            nn.init.normal_(m.weight, 0.0, 0.02)
+            if m.bias is not None:
+                nn.init.zeros_(m.bias)
+        elif isinstance(m, nn.Embedding):
+            nn.init.normal_(m.weight, 0.0, 0.02)
+
+    def forward(self, idx, targets=None):
+        B, T = idx.size()
+        x = self.tok(idx)
+        if not self.use_rope:
+            pos = torch.arange(T, device=idx.device)
+            x = x + self.pos(pos)[None]
+        v0 = None
+        for b in self.blocks:
+            x, v0 = b(x, v0)
+        logits = self.head(self.lnf(x))
+        cap = getattr(self.cfg, "logit_cap", 0.0)
+        if cap and cap > 0:
+            logits = cap * torch.tanh(logits / cap)
+        loss = None
+        if targets is not None:
+            flat = logits.view(-1, logits.size(-1))
+            loss = F.cross_entropy(flat, targets.view(-1))
+            zc = getattr(self.cfg, "z_loss", 0.0)
+            if zc and zc > 0:
+                lse = torch.logsumexp(flat, dim=-1)
+                loss = loss + zc * (lse * lse).mean()
+        return logits, loss
+

release_verification.json

@@ -0,0 +1,7 @@
+{
+  "weights_and_logits": "exact match to final training checkpoint at contexts 16, 256, 1024",
+  "glint_likelihood_and_perplexity_functions": "exact match on golden text fixtures including padding, truncation and empty input",
+  "official_harness_revision": "c3f99e246aa2c64382f9668dc864533617482d0a",
+  "token_12285": "<|endoftext|>",
+  "status": "passed"
+}

requirements.txt

@@ -0,0 +1,6 @@
+torch==2.11.0
+tokenizers==0.22.2
+safetensors==0.6.2
+huggingface-hub==0.36.0
+numpy>=1.26,<3
+pyarrow>=20,<25

training_manifest.json

@@ -0,0 +1,72 @@
+{
+  "checkpoint_sha256": "c4e8b5fb5730242d790d469806a8735be2424b9351dcdc25f643980ba3028236",
+  "weights_sha256": "05e2a496967bdd36da3e1bdd4fb8b10277b92a319b1f5f1e61062454eecc7b2e",
+  "parameters": 148910738,
+  "reference_step": 76294,
+  "optimizer_updates": 39147,
+  "continuation_tokens": 20000014336,
+  "provenance": {
+    "source_repo": "SlayerLab/gollem-v5-ckpts",
+    "source_revision": "963440ce6ab4ada7e95da4c1faa28ebb00c082d4",
+    "source_file": "final_128m_16x768/ckpt_760k.pt",
+    "source_sha256": "95d8b43fcd9c5fa06566da13dd03856b00300bad6f37fb4dc9442c18be9ac372",
+    "source_steps": 760000,
+    "source_training_tokens": 24903680000,
+    "parameters": 148910738,
+    "checks": [
+      {
+        "length": 16,
+        "max_absolute_logit_error": 0.0
+      },
+      {
+        "length": 128,
+        "max_absolute_logit_error": 0.0
+      },
+      {
+        "length": 256,
+        "max_absolute_logit_error": 0.0
+      }
+    ],
+    "optimizer": "reset for continued pretraining",
+    "method": "FFN expansion with zero new down columns; append three identity residual blocks"
+  },
+  "safetensors_roundtrip": "all model tensors exactly equal; tied weights restored",
+  "training_config": {
+    "model": {
+      "vocab": 12288,
+      "n_layer": 19,
+      "n_embd": 768,
+      "n_head": 12,
+      "block": 1024,
+      "ffn_mult": 2.8125,
+      "norm": "rmsnorm",
+      "norm_eps": 1e-06,
+      "pos": "rope",
+      "rope_theta": 100000.0,
+      "ffn": "swiglu",
+      "value_residual": true,
+      "qk_norm": true,
+      "logit_cap": 0.0,
+      "z_loss": 0.0
+    },
+    "train": {
+      "seed": 1337,
+      "micro_batch": 32,
+      "global_batch": 256,
+      "steps": 76294,
+      "warmup": 250,
+      "lr": 0.0002,
+      "min_lr": 2e-05,
+      "muon_lr": 0.006666666666666667,
+      "checkpoint_every": 1000,
+      "validate_every": 1000,
+      "log_every": 20,
+      "compile": true
+    },
+    "data": {
+      "train": "/data/final/train.bin",
+      "validation": "/data/final/val.bin",
+      "sha256": "ecfd0a4040a0ece6f07f728f092a1b373b6d253b4adfbe259c5107c1fc16681d"
+    }
+  }
+}