Release Bazza 0.5.0 decision head and standalone inference code
#1
by adrianjohnstonaus - opened
- LICENSE +202 -0
- NOTICE +8 -0
- README.md +172 -0
- VERSION +1 -0
- config.json +43 -0
- data.py +281 -0
- features/__init__.py +1 -0
- features/bank.py +102 -0
- features/store.py +139 -0
- model.safetensors +3 -0
- model/NOTICE.md +37 -0
- model/__init__.py +15 -0
- model/backbone.py +92 -0
- model/gemma.py +373 -0
- model/gemma_backbone.py +220 -0
- model/model.py +92 -0
- model/readers.py +127 -0
- model/readout.py +216 -0
- model/reasoner.py +398 -0
- requirements.txt +8 -0
- systemone.py +319 -0
- utils.py +71 -0
LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
NOTICE
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Bazza
|
| 2 |
+
Copyright 2026 Bazza contributors.
|
| 3 |
+
|
| 4 |
+
Licensed under the Apache License, Version 2.0. See LICENSE.
|
| 5 |
+
|
| 6 |
+
Third-party source attributions and license notices are retained in
|
| 7 |
+
model/NOTICE.md, including the Hugging Face Transformers attribution
|
| 8 |
+
and the bitsandbytes MIT notice.
|
README.md
CHANGED
|
@@ -1,3 +1,175 @@
|
|
| 1 |
---
|
| 2 |
license: apache-2.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
license: apache-2.0
|
| 3 |
+
base_model: google/gemma-4-12B-it
|
| 4 |
+
base_model_relation: adapter
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
tags:
|
| 8 |
+
- pytorch
|
| 9 |
+
- system-1
|
| 10 |
+
- decision-model
|
| 11 |
+
- custom-head
|
| 12 |
+
- gemma4
|
| 13 |
---
|
| 14 |
+
|
| 15 |
+
# Bazza 0.5.0
|
| 16 |
+
|
| 17 |
+
Bazza is a System 1 decision model implemented in PyTorch. It scores supplied
|
| 18 |
+
candidate answers using a frozen Gemma 4 12B IT backbone and a separate trained
|
| 19 |
+
decision head: a bidirectional GRU, recurrent attention, and a gated pretrained
|
| 20 |
+
readout.
|
| 21 |
+
|
| 22 |
+
This repository contains the **head weights and inference code**, not the Gemma
|
| 23 |
+
backbone. The head is a custom decision module, not a PEFT/LoRA adapter. It does
|
| 24 |
+
not load through Transformers AutoModel or text-generation pipelines.
|
| 25 |
+
|
| 26 |
+
## Loading
|
| 27 |
+
|
| 28 |
+
Use Python 3.10 or newer and a suitable PyTorch/CUDA installation. Clone this
|
| 29 |
+
repository with Git LFS installed, or download it using `snapshot_download`.
|
| 30 |
+
|
| 31 |
+
```bash
|
| 32 |
+
git clone https://huggingface.co/0xdeadbe3f/Bazza
|
| 33 |
+
cd Bazza
|
| 34 |
+
pip install -r requirements.txt
|
| 35 |
+
```
|
| 36 |
+
|
| 37 |
+
The loader downloads the head and the exact backbone revision into the Hugging
|
| 38 |
+
Face cache. The initial backbone download contains the original BF16 weights
|
| 39 |
+
and is substantially larger than the 115 MiB head; linear weights are quantized
|
| 40 |
+
to bitsandbytes NF4 during CUDA loading. Embeddings and normalization remain BF16.
|
| 41 |
+
The evaluated configuration used an RTX 3090 with 24 GiB VRAM.
|
| 42 |
+
|
| 43 |
+
```python
|
| 44 |
+
from systemone import SystemOne
|
| 45 |
+
|
| 46 |
+
model = SystemOne.from_pretrained("0xdeadbe3f/Bazza", revision="0.5.0")
|
| 47 |
+
|
| 48 |
+
result = model(
|
| 49 |
+
state={
|
| 50 |
+
"ticket": "Order #1182 arrived broken. The customer requests a refund.",
|
| 51 |
+
"order_age_days": 40,
|
| 52 |
+
"refund_policy_days": 30,
|
| 53 |
+
},
|
| 54 |
+
questions={
|
| 55 |
+
"refund": {
|
| 56 |
+
"type": "noul",
|
| 57 |
+
"instructions": "Is the refund allowed under the policy?",
|
| 58 |
+
"criteria": {
|
| 59 |
+
"true": "The refund is allowed by the stated policy.",
|
| 60 |
+
"false": "The refund is not allowed by the stated policy.",
|
| 61 |
+
},
|
| 62 |
+
},
|
| 63 |
+
"action": {
|
| 64 |
+
"type": "choice",
|
| 65 |
+
"instructions": "Select the next action using the stated policy.",
|
| 66 |
+
"criteria": {
|
| 67 |
+
"refund_order": "Issue a refund because the policy permits it.",
|
| 68 |
+
"explain_policy": "Explain that the order is outside the refund window.",
|
| 69 |
+
"ask_order_age": "Ask the order age because it is unknown.",
|
| 70 |
+
},
|
| 71 |
+
},
|
| 72 |
+
},
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
print(result["answers"]["refund"]["value"]) # boolean decision
|
| 76 |
+
print(result["answers"]["refund"]["noul"]) # probability assigned to true
|
| 77 |
+
print(result["answers"]["action"]["choice"]) # selected criterion key
|
| 78 |
+
print(result["answers"]["action"]["probabilities"]) # scores for all candidates
|
| 79 |
+
```
|
| 80 |
+
|
| 81 |
+
Pass `backbone_path="/path/to/Gemma4-12B-it"` to reuse existing local weights.
|
| 82 |
+
For offline use, call `SystemOne.from_pretrained(".", backbone_path=..., local_files_only=True)`.
|
| 83 |
+
The bundled Python implementation is imported normally; no remote Python code
|
| 84 |
+
is downloaded or executed by `from_pretrained`.
|
| 85 |
+
|
| 86 |
+
## Output types
|
| 87 |
+
|
| 88 |
+
| Type | Criteria | Result |
|
| 89 |
+
| --- | --- | --- |
|
| 90 |
+
| `choice` | Named candidate descriptions | Selected key and candidate probabilities |
|
| 91 |
+
| `noul` | Descriptions keyed by `"true"` and `"false"` | Probability of true and a boolean value |
|
| 92 |
+
| `score` | Descriptions keyed by numeric levels | Probability-weighted mean of those levels |
|
| 93 |
+
|
| 94 |
+
Each question uses 2–26 candidates and is scored separately. Returned
|
| 95 |
+
probabilities are softmax scores; they have not been calibrated. The current API
|
| 96 |
+
accepts text decisions with JSON-compatible state. Overlong inputs are rejected
|
| 97 |
+
rather than truncated.
|
| 98 |
+
|
| 99 |
+
## Head-only adaptation
|
| 100 |
+
|
| 101 |
+
Training updates the decision head while leaving all pretrained backbone weights
|
| 102 |
+
unchanged. The head is stored separately and can train on reusable token features.
|
| 103 |
+
|
| 104 |
+
| Adaptation cost | Current model |
|
| 105 |
+
| --- | ---: |
|
| 106 |
+
| Trainable parameters | 30,117,964 (about 0.25% of 12B) |
|
| 107 |
+
| Backbone parameters updated | 0 |
|
| 108 |
+
| Head weight tensors, FP32 | 115.3 MiB |
|
| 109 |
+
| One pass over 96,000 cached examples, RTX 3090 | 8.93 minutes |
|
| 110 |
+
|
| 111 |
+
The training time includes validation, checkpointing, and logging. It excludes
|
| 112 |
+
feature extraction, model setup, and subsequent testing. Feature extraction is
|
| 113 |
+
an additional cost that can be reused across head-training experiments. The
|
| 114 |
+
weight size excludes optimizer state and checkpoint metadata.
|
| 115 |
+
|
| 116 |
+
## Benchmarks
|
| 117 |
+
|
| 118 |
+
Accuracy (%) on the same 13,499 public-benchmark requests for every model.
|
| 119 |
+
MMLU-Pro uses a 2,000-example test subset. Bazza gain is relative to stock Gemma;
|
| 120 |
+
positive values indicate an improvement in percentage points.
|
| 121 |
+
|
| 122 |
+
| Benchmark | Examples | Stock Gemma 12B (NF4) | Bazza 12B (NF4) | Bazza gain (pp) | [Winnow 12B (Q8)](https://huggingface.co/EldanRing/Winnow-12B/tree/b6ac22b0d51b69b18200acacb3fbdd98073fffe8) | [PPLX-Decider v1.1 27B (NF4)](https://huggingface.co/perplexity-ai/pplx-decider-v1.1-27b/tree/3b45dead91dfa6d95aad6b95764a606fab2bf7a6) |
|
| 123 |
+
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
|
| 124 |
+
| When2Call (test) | 3,652 | 63.34 | 79.41 | +16.07 | 83.19 | 74.70 |
|
| 125 |
+
| ARC-Challenge (test) | 1,172 | 90.27 | 92.41 | +2.13 | 94.88 | 97.18 |
|
| 126 |
+
| ANLI round 3 (test) | 1,200 | 58.25 | 61.42 | +3.17 | 62.17 | 70.17 |
|
| 127 |
+
| WinoGrande (validation) | 1,267 | 71.82 | 71.43 | -0.39 | 73.88 | 92.34 |
|
| 128 |
+
| MMLU-Pro (test subset) | 2,000 | 44.00 | 48.30 | +4.30 | 54.30 | 68.15 |
|
| 129 |
+
| GSM8K, 4 choices | 1,319 | 50.87 | 62.32 | +11.45 | 61.18 | 81.12 |
|
| 130 |
+
| GSM8K, 10 choices | 1,319 | 24.34 | 43.97 | +19.64 | 35.86 | 71.04 |
|
| 131 |
+
| CRUXEval output choice | 570 | 56.67 | 55.44 | -1.23 | 69.65 | 85.26 |
|
| 132 |
+
| RAGTruth classification | 1,000 | 76.90 | 82.90 | +6.00 | 82.60 | 88.40 |
|
| 133 |
+
| **Eight-dataset macro** | — | **62.36** | **68.06** | **+5.70** | **71.15** | **81.54** |
|
| 134 |
+
|
| 135 |
+
GSM8K and CRUXEval use adapted multiple-choice formats; RAGTruth uses binary
|
| 136 |
+
response-level classification accuracy. The macro weights eight datasets equally,
|
| 137 |
+
combining the two GSM8K views first; they share the same 1,319 source problems.
|
| 138 |
+
|
| 139 |
+
Bazza uses the frozen [Gemma 4 12B IT backbone](https://huggingface.co/google/gemma-4-12B-it/tree/707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7)
|
| 140 |
+
and the head checkpoint trained on Groundbeef v0.5 at step 2,661 (80,019 training examples),
|
| 141 |
+
selected on validation. Stock Gemma uses its original answer-letter weights and
|
| 142 |
+
softcapping on the same cached features and prompts as Bazza. The head adds
|
| 143 |
+
**16.07 percentage points on When2Call** and **4.30 points on MMLU-Pro**; WinoGrande
|
| 144 |
+
drops by 0.39 points.
|
| 145 |
+
|
| 146 |
+
All runs use an RTX 3090 with no calibration fitted on the test panel. Winnow and
|
| 147 |
+
PPLX-Decider use their own prompts and readouts. These are decision-format accuracy
|
| 148 |
+
measurements, not official Decision Index scores or generative benchmark results.
|
| 149 |
+
|
| 150 |
+
## Training data and selection
|
| 151 |
+
|
| 152 |
+
Groundbeef is our internally curated training dataset: a mixture of existing
|
| 153 |
+
source datasets with our preprocessing and data splits. This release uses
|
| 154 |
+
Groundbeef v0.5. The dataset is not included and may be released separately.
|
| 155 |
+
The published head was selected on validation at step 2,661 (80,019 examples)
|
| 156 |
+
from a 96,000-example training run. Backbone weights were frozen throughout.
|
| 157 |
+
|
| 158 |
+
## Files and provenance
|
| 159 |
+
|
| 160 |
+
- `model.safetensors`: inference head parameters and required readout buffers.
|
| 161 |
+
- `config.json`: architecture, pinned backbone identity, tensor checksum,
|
| 162 |
+
tokenizer identity, and selected-checkpoint provenance.
|
| 163 |
+
- `systemone.py`, `model/`, `data.py`, `utils.py`, and `features/`: PyTorch source.
|
| 164 |
+
- `LICENSE`, `NOTICE`, and `model/NOTICE.md`: Apache 2.0 and third-party notices.
|
| 165 |
+
|
| 166 |
+
Optimizer state, random state, local paths, data caches, and training records
|
| 167 |
+
are excluded. Configuration format version 1 is specific to Bazza.
|
| 168 |
+
|
| 169 |
+
The source and training implementation are at
|
| 170 |
+
[deadbe3f/Bazza](https://github.com/deadbe3f/Bazza).
|
| 171 |
+
|
| 172 |
+
## License
|
| 173 |
+
|
| 174 |
+
Apache License 2.0. See [LICENSE](LICENSE), [NOTICE](NOTICE), and
|
| 175 |
+
[third-party attributions](model/NOTICE.md).
|
VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
0.5.0
|
config.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_type": "bazza",
|
| 3 |
+
"format_version": 1,
|
| 4 |
+
"bazza_version": "0.5.0",
|
| 5 |
+
"config": {
|
| 6 |
+
"backbone_checkpoint": "google/gemma-4-12B-it",
|
| 7 |
+
"backbone_revision": "707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7",
|
| 8 |
+
"quantization": "bnb-nf4",
|
| 9 |
+
"encoder_width": 3840,
|
| 10 |
+
"hidden_width": 1024,
|
| 11 |
+
"attention_heads": 4,
|
| 12 |
+
"low_steps": 2,
|
| 13 |
+
"high_cycles": 2,
|
| 14 |
+
"gate_width": 32,
|
| 15 |
+
"adapter_rank": 64,
|
| 16 |
+
"precision": "bf16",
|
| 17 |
+
"softcap": 30.0
|
| 18 |
+
},
|
| 19 |
+
"readout": {
|
| 20 |
+
"softcap": 30.0
|
| 21 |
+
},
|
| 22 |
+
"identity": {
|
| 23 |
+
"readout_files": {
|
| 24 |
+
"config.json": "478c46e8d2c52d5c2d85bf67e3b3e8c90e7c9d91086cee27e3c267907e936bd9",
|
| 25 |
+
"tokenizer.json": "cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f"
|
| 26 |
+
},
|
| 27 |
+
"source_sha256": {
|
| 28 |
+
"model/gemma.py": "c6d1d195da4340065745dd83e5da16aa1cd8558c42c11371dc18460bbcd86bb1"
|
| 29 |
+
},
|
| 30 |
+
"backbone_weight_sha256": "5a84cb313260ac447237b890387116dfa8682e49a6b44bc585ae8353abbff18d"
|
| 31 |
+
},
|
| 32 |
+
"weights": {
|
| 33 |
+
"filename": "model.safetensors",
|
| 34 |
+
"sha256": "aa18635a7398211d6eda876534f6cf3244a048a20f823d2190d2e4b14d836989"
|
| 35 |
+
},
|
| 36 |
+
"provenance": {
|
| 37 |
+
"training_dataset": "Groundbeef v0.5",
|
| 38 |
+
"checkpoint_sha256": "893723966557a4ea2484a0993d2e072e7e2feeb19f6fbb4771d3e022a95ef018",
|
| 39 |
+
"step": 2661,
|
| 40 |
+
"training_examples_seen": 80019,
|
| 41 |
+
"selection": "validation"
|
| 42 |
+
}
|
| 43 |
+
}
|
data.py
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Named batches, fixed Gemma serialization and deterministic batching."""
|
| 2 |
+
|
| 3 |
+
import hashlib
|
| 4 |
+
import json
|
| 5 |
+
import math
|
| 6 |
+
import random
|
| 7 |
+
import unicodedata
|
| 8 |
+
from dataclasses import dataclass
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@dataclass
|
| 15 |
+
class DecisionBatch:
|
| 16 |
+
rows: list
|
| 17 |
+
attention_mask: torch.Tensor
|
| 18 |
+
option_positions: torch.Tensor
|
| 19 |
+
option_mask: torch.Tensor
|
| 20 |
+
targets: torch.Tensor | None = None
|
| 21 |
+
hidden: torch.Tensor | None = None
|
| 22 |
+
input_ids: torch.Tensor | None = None
|
| 23 |
+
split: str = "train"
|
| 24 |
+
feature_space: str | None = None
|
| 25 |
+
feature_source: str | None = None
|
| 26 |
+
|
| 27 |
+
def inputs(self):
|
| 28 |
+
return dict(
|
| 29 |
+
hidden=self.hidden,
|
| 30 |
+
attention_mask=self.attention_mask,
|
| 31 |
+
option_positions=self.option_positions,
|
| 32 |
+
option_mask=self.option_mask,
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
def to(self, device):
|
| 36 |
+
return DecisionBatch(
|
| 37 |
+
self.rows,
|
| 38 |
+
**{
|
| 39 |
+
key: value.to(device) if isinstance(value, torch.Tensor) else value
|
| 40 |
+
for key, value in vars(self).items()
|
| 41 |
+
if key != "rows"
|
| 42 |
+
},
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def collate_records(rows, hidden=None, *, target_key="target"):
|
| 47 |
+
"""Pad only the head input. No truncation, option permutation or relabelling."""
|
| 48 |
+
if not rows:
|
| 49 |
+
raise ValueError("Cannot collate an empty batch")
|
| 50 |
+
length = max(len(row["ids"]) for row in rows)
|
| 51 |
+
choices = max(len(row["positions"]) for row in rows)
|
| 52 |
+
ids = torch.zeros(len(rows), length, dtype=torch.long)
|
| 53 |
+
mask = torch.zeros_like(ids)
|
| 54 |
+
positions = torch.zeros(len(rows), choices, dtype=torch.long)
|
| 55 |
+
option_mask = torch.zeros_like(positions, dtype=torch.bool)
|
| 56 |
+
targets = torch.zeros(len(rows), choices)
|
| 57 |
+
memory = None
|
| 58 |
+
if hidden is not None:
|
| 59 |
+
memory = hidden[0].new_zeros(len(rows), length, hidden[0].shape[-1])
|
| 60 |
+
has_targets = [target_key in row for row in rows]
|
| 61 |
+
if any(has_targets) and not all(has_targets):
|
| 62 |
+
raise ValueError(
|
| 63 |
+
"Mixed missing/present targets; choose an explicit target policy"
|
| 64 |
+
)
|
| 65 |
+
for i, row in enumerate(rows):
|
| 66 |
+
n, k = len(row["ids"]), len(row["positions"])
|
| 67 |
+
if n < 1 or not 2 <= k <= 26 or any(p < 0 or p >= n for p in row["positions"]):
|
| 68 |
+
raise ValueError("Invalid token/option positions: " + row["id"])
|
| 69 |
+
ids[i, :n] = torch.tensor(row["ids"])
|
| 70 |
+
mask[i, :n] = 1
|
| 71 |
+
positions[i, :k] = torch.tensor(row["positions"])
|
| 72 |
+
option_mask[i, :k] = True
|
| 73 |
+
if all(has_targets):
|
| 74 |
+
value = torch.tensor(row[target_key], dtype=torch.float32)
|
| 75 |
+
# Validate the supplied probabilities before FP32 conversion moves a
|
| 76 |
+
# rounded sum (e.g. 0.9999) across the acceptance boundary.
|
| 77 |
+
if (
|
| 78 |
+
value.shape != (k,)
|
| 79 |
+
or not torch.isfinite(value).all()
|
| 80 |
+
or (value < 0).any()
|
| 81 |
+
or abs(math.fsum(row[target_key]) - 1) > 1e-4
|
| 82 |
+
):
|
| 83 |
+
raise ValueError("Invalid target distribution: " + row["id"])
|
| 84 |
+
targets[i, :k] = value
|
| 85 |
+
if memory is not None:
|
| 86 |
+
if hidden[i].shape != (n, memory.shape[-1]):
|
| 87 |
+
raise ValueError("Cached hidden shape differs: " + row["id"])
|
| 88 |
+
memory[i, :n].copy_(hidden[i])
|
| 89 |
+
return DecisionBatch(
|
| 90 |
+
rows,
|
| 91 |
+
mask,
|
| 92 |
+
positions,
|
| 93 |
+
option_mask,
|
| 94 |
+
targets if all(has_targets) else None,
|
| 95 |
+
memory,
|
| 96 |
+
ids,
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def load_selection(path):
|
| 101 |
+
selection = json.loads(Path(path).read_text())
|
| 102 |
+
splits = selection["splits"]
|
| 103 |
+
for name, rows in splits.items():
|
| 104 |
+
if len({row["id"] for row in rows}) != len(rows):
|
| 105 |
+
raise ValueError("Duplicate example IDs in " + name)
|
| 106 |
+
train_ids = {r["id"] for r in splits.get("train", [])}
|
| 107 |
+
train_groups = {r["group_id"] for r in splits.get("train", []) if r.get("group_id")}
|
| 108 |
+
for name, rows in splits.items():
|
| 109 |
+
if name == "train":
|
| 110 |
+
continue
|
| 111 |
+
if train_ids & {r["id"] for r in rows} or train_groups & {
|
| 112 |
+
r["group_id"] for r in rows if r.get("group_id")
|
| 113 |
+
}:
|
| 114 |
+
raise ValueError("Training/held-out membership overlap in " + name)
|
| 115 |
+
return selection
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def label(value):
|
| 119 |
+
return " ".join(unicodedata.normalize("NFKC", str(value)).split())
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def fields(row):
|
| 123 |
+
meta = row.get("metadata", {})
|
| 124 |
+
source = meta.get("dataset_source", row.get("dataset_source", row["source"]))
|
| 125 |
+
task = meta.get(
|
| 126 |
+
"task_type", row.get("task_type", row["source"] + "/" + row["kind"])
|
| 127 |
+
)
|
| 128 |
+
schema = meta.get("balance_schema", row.get("balance_schema"))
|
| 129 |
+
if schema is None:
|
| 130 |
+
schema = hashlib.sha256(
|
| 131 |
+
json.dumps(
|
| 132 |
+
[
|
| 133 |
+
row.get("question", ""),
|
| 134 |
+
sorted(label(o) for o in row.get("options", [])),
|
| 135 |
+
],
|
| 136 |
+
sort_keys=True,
|
| 137 |
+
).encode()
|
| 138 |
+
).hexdigest()
|
| 139 |
+
return {
|
| 140 |
+
"dataset_source": source,
|
| 141 |
+
"task_type": task,
|
| 142 |
+
"task_key": source + "/" + task,
|
| 143 |
+
"balance_schema": schema,
|
| 144 |
+
"domain": row.get("domain", row["source"]),
|
| 145 |
+
"options": [str(o) for o in row.get("options", [])],
|
| 146 |
+
"loss_weight": float(meta.get("loss_weight", row.get("loss_weight", 1.0))),
|
| 147 |
+
"pair_id": meta.get("upstream_paired_index", row.get("pair_id")),
|
| 148 |
+
}
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def schema_key(row):
|
| 152 |
+
data = fields(row)
|
| 153 |
+
return data["task_key"] + "/" + data["balance_schema"]
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def encode_input(row, tokenizer, max_length, order=None):
|
| 157 |
+
options = row["options"]
|
| 158 |
+
if order is None:
|
| 159 |
+
order = list(range(len(options)))
|
| 160 |
+
if not 2 <= len(order) <= 26:
|
| 161 |
+
return None
|
| 162 |
+
state = row["state"]
|
| 163 |
+
if not isinstance(state, str):
|
| 164 |
+
state = json.dumps(state, ensure_ascii=False, sort_keys=True)
|
| 165 |
+
prompt = "<bos><|turn>user\nChoose the best answer. Reply with its letter only.\n"
|
| 166 |
+
prompt += "Context:\n" + state + "\nQuestion:\n" + row["question"] + "\nOptions:\n"
|
| 167 |
+
ends = []
|
| 168 |
+
for index, original in enumerate(order):
|
| 169 |
+
prompt += chr(65 + index) + ". " + str(options[original])
|
| 170 |
+
ends.append(len(prompt))
|
| 171 |
+
prompt += "\n"
|
| 172 |
+
prompt += "<turn|>\n<|turn>model\n<|channel>thought\n<channel|>"
|
| 173 |
+
encoded = tokenizer.encode(prompt, add_special_tokens=False)
|
| 174 |
+
if len(encoded.ids) > max_length:
|
| 175 |
+
return None # No silently truncated states, questions or choices.
|
| 176 |
+
positions = []
|
| 177 |
+
for end in ends:
|
| 178 |
+
eligible = [i for i, (a, b) in enumerate(encoded.offsets) if 0 < b <= end]
|
| 179 |
+
if not eligible:
|
| 180 |
+
raise ValueError("Cannot locate option's final token")
|
| 181 |
+
positions.append(eligible[-1])
|
| 182 |
+
return {
|
| 183 |
+
"id": row["id"],
|
| 184 |
+
"group_id": row["group_id"],
|
| 185 |
+
"source": row["source"],
|
| 186 |
+
"kind": row["kind"],
|
| 187 |
+
"ids": encoded.ids,
|
| 188 |
+
"positions": positions,
|
| 189 |
+
"order": order,
|
| 190 |
+
**{**fields(row), "options": [str(options[i]) for i in order]},
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def encode(row, tokenizer, max_length, order=None):
|
| 195 |
+
result = encode_input(row, tokenizer, max_length, order)
|
| 196 |
+
if result is None:
|
| 197 |
+
return None
|
| 198 |
+
target = [float(row["target"][i]) for i in result["order"]]
|
| 199 |
+
if (
|
| 200 |
+
len(target) != len(result["order"])
|
| 201 |
+
or min(target) < 0
|
| 202 |
+
or abs(sum(target) - 1) > 1e-4
|
| 203 |
+
):
|
| 204 |
+
raise ValueError("Invalid target distribution")
|
| 205 |
+
result["target"] = target
|
| 206 |
+
return result
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def length_batches(records, max_rows, token_budget):
|
| 210 |
+
"""Bucket by length, bounding padded tokens without dropping long rows."""
|
| 211 |
+
batch = []
|
| 212 |
+
for row in sorted(records, key=lambda r: (len(r["ids"]), r["id"])):
|
| 213 |
+
if batch and (
|
| 214 |
+
len(batch) >= max_rows or (len(batch) + 1) * len(row["ids"]) > token_budget
|
| 215 |
+
):
|
| 216 |
+
yield batch
|
| 217 |
+
batch = []
|
| 218 |
+
batch.append(row)
|
| 219 |
+
if batch:
|
| 220 |
+
yield batch
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def training_batches(rows, batch_size, token_budget, seed):
|
| 224 |
+
"""Shuffle examples, bucket locally by length, then shuffle the batches."""
|
| 225 |
+
rng = random.Random(seed)
|
| 226 |
+
rows = list(rows)
|
| 227 |
+
rng.shuffle(rows)
|
| 228 |
+
batches = []
|
| 229 |
+
for start in range(0, len(rows), batch_size * 16):
|
| 230 |
+
batches.extend(
|
| 231 |
+
length_batches(
|
| 232 |
+
rows[start : start + batch_size * 16], batch_size, token_budget
|
| 233 |
+
)
|
| 234 |
+
)
|
| 235 |
+
rng.shuffle(batches)
|
| 236 |
+
return batches
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
class BatchLoader:
|
| 240 |
+
"""Normal CPU collation/prefetch; it never runs a backbone or touches CUDA in its worker."""
|
| 241 |
+
|
| 242 |
+
def __init__(self, collate_fn, device="cuda", prefetch_batches=8):
|
| 243 |
+
self.collate_fn, self.device, self.depth = collate_fn, device, prefetch_batches
|
| 244 |
+
if prefetch_batches < 1:
|
| 245 |
+
raise ValueError("prefetch_batches must be positive")
|
| 246 |
+
|
| 247 |
+
def __call__(self, rows, split):
|
| 248 |
+
return self.collate_fn(rows, split).to(self.device)
|
| 249 |
+
|
| 250 |
+
def iter_batches(self, batches, start, split):
|
| 251 |
+
from collections import deque
|
| 252 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 253 |
+
from itertools import islice
|
| 254 |
+
|
| 255 |
+
indices = iter(range(start, len(batches)))
|
| 256 |
+
pending = deque()
|
| 257 |
+
with ThreadPoolExecutor(
|
| 258 |
+
max_workers=1, thread_name_prefix="batch-loader"
|
| 259 |
+
) as pool:
|
| 260 |
+
try:
|
| 261 |
+
for index in islice(indices, self.depth):
|
| 262 |
+
pending.append(
|
| 263 |
+
(index, pool.submit(self.collate_fn, batches[index], split))
|
| 264 |
+
)
|
| 265 |
+
while pending:
|
| 266 |
+
index, future = pending.popleft()
|
| 267 |
+
batch = future.result()
|
| 268 |
+
next_index = next(indices, None)
|
| 269 |
+
if next_index is not None:
|
| 270 |
+
pending.append(
|
| 271 |
+
(
|
| 272 |
+
next_index,
|
| 273 |
+
pool.submit(
|
| 274 |
+
self.collate_fn, batches[next_index], split
|
| 275 |
+
),
|
| 276 |
+
)
|
| 277 |
+
)
|
| 278 |
+
yield index, batch.to(self.device)
|
| 279 |
+
finally:
|
| 280 |
+
for _, future in pending:
|
| 281 |
+
future.cancel()
|
features/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Explicit frozen-feature inputs; no automatic training-data export."""
|
features/bank.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Read only committed raw feature shards, including added layers."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from collections import OrderedDict
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def normalize_final(hidden, artifact):
|
| 11 |
+
"""Gemma RMSNorm: stored weight is the effective multiplier, already converted."""
|
| 12 |
+
x = hidden.float()
|
| 13 |
+
x = x * (x.square().mean(-1, keepdim=True) + artifact["eps"]).pow(-0.5)
|
| 14 |
+
return (x * artifact["weight"].to(x.device).float()).to(hidden.dtype)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def read_journal_readonly(path):
|
| 18 |
+
# Readers never repair or mutate an active export's journal.
|
| 19 |
+
with path.open("rb") as stream:
|
| 20 |
+
return [json.loads(line) for line in stream if line.endswith(b"\n")]
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def commits(directory):
|
| 24 |
+
read = read_journal_readonly
|
| 25 |
+
merged = {}
|
| 26 |
+
for name in ("shards.jsonl", "layer-additions.jsonl"):
|
| 27 |
+
p = directory / name
|
| 28 |
+
if not p.exists():
|
| 29 |
+
continue
|
| 30 |
+
for entry in read(p):
|
| 31 |
+
n = entry["shard"]
|
| 32 |
+
if n not in merged:
|
| 33 |
+
merged[n] = {**entry, "files": dict(entry["files"])}
|
| 34 |
+
continue
|
| 35 |
+
previous = merged[n]
|
| 36 |
+
if (
|
| 37 |
+
previous["records_sha256"] != entry["records_sha256"]
|
| 38 |
+
or previous["rows"] != entry["rows"]
|
| 39 |
+
):
|
| 40 |
+
raise ValueError("Layer journals disagree about inputs/offsets")
|
| 41 |
+
for layer, item in entry["files"].items():
|
| 42 |
+
if layer in previous["files"] and previous["files"][layer] != item:
|
| 43 |
+
raise ValueError("Conflicting committed layer file")
|
| 44 |
+
previous["files"][layer] = item
|
| 45 |
+
return merged
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class LayerBank:
|
| 49 |
+
"""Read only the requested layers and examples; mmap shards, bounded handles."""
|
| 50 |
+
|
| 51 |
+
def __init__(self, split_dir, layers, max_open_shards=8):
|
| 52 |
+
self.directory = Path(split_dir)
|
| 53 |
+
self.layers = list(layers)
|
| 54 |
+
self.max_open = max_open_shards
|
| 55 |
+
self.cache = OrderedDict()
|
| 56 |
+
self.records = {
|
| 57 |
+
r["id"]: r
|
| 58 |
+
for r in json.loads((self.directory / "records.json").read_text())
|
| 59 |
+
}
|
| 60 |
+
self.locations = {}
|
| 61 |
+
for entry in read_journal_readonly(self.directory / "shards.jsonl"):
|
| 62 |
+
for row in entry["rows"]:
|
| 63 |
+
self.locations[row["id"]] = (entry, row)
|
| 64 |
+
|
| 65 |
+
def load(self, example_id):
|
| 66 |
+
entry, row = self.locations[example_id]
|
| 67 |
+
result = {}
|
| 68 |
+
for layer in self.layers:
|
| 69 |
+
key = entry["files"][layer]["file"]
|
| 70 |
+
if key not in self.cache:
|
| 71 |
+
self.cache[key] = torch.load(
|
| 72 |
+
self.directory / key,
|
| 73 |
+
map_location="cpu",
|
| 74 |
+
weights_only=True,
|
| 75 |
+
mmap=True,
|
| 76 |
+
)
|
| 77 |
+
while len(self.cache) > self.max_open:
|
| 78 |
+
self.cache.popitem(last=False)
|
| 79 |
+
self.cache.move_to_end(key)
|
| 80 |
+
result[layer] = self.cache[key]["hidden"][
|
| 81 |
+
row["offset"] : row["offset"] + row["tokens"]
|
| 82 |
+
]
|
| 83 |
+
return result
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class AddedLayerBank(LayerBank):
|
| 87 |
+
"""Mmap only requested layers; works with original and added-layer commits."""
|
| 88 |
+
|
| 89 |
+
def __init__(self, split_dir, layers, max_open_shards=8):
|
| 90 |
+
self.directory = Path(split_dir)
|
| 91 |
+
self.layers = list(layers)
|
| 92 |
+
self.max_open = max_open_shards
|
| 93 |
+
self.cache = OrderedDict()
|
| 94 |
+
self.records = {
|
| 95 |
+
r["id"]: r
|
| 96 |
+
for r in json.loads((self.directory / "records.json").read_text())
|
| 97 |
+
}
|
| 98 |
+
self.locations = {}
|
| 99 |
+
for entry in commits(self.directory).values():
|
| 100 |
+
if set(self.layers).issubset(entry["files"]):
|
| 101 |
+
for row in entry["rows"]:
|
| 102 |
+
self.locations[row["id"]] = (entry, row)
|
features/store.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CPU IO for immutable per-example or raw-layer feature banks."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
from data import collate_records
|
| 9 |
+
from utils import file_hash
|
| 10 |
+
|
| 11 |
+
from .bank import AddedLayerBank
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class FeatureStore:
|
| 15 |
+
def __init__(self, config, selection, device="cuda"):
|
| 16 |
+
self.config, self.selection, self.device = config, selection, device
|
| 17 |
+
self.kind = config["kind"]
|
| 18 |
+
self.resident = {}
|
| 19 |
+
self.checked = set()
|
| 20 |
+
self.banks = {}
|
| 21 |
+
self.normalization = None
|
| 22 |
+
if self.kind in ("layer", "routed"):
|
| 23 |
+
if not config.get("layer"):
|
| 24 |
+
raise ValueError("Raw cache requires an explicit final layer")
|
| 25 |
+
roots = config.get("banks", {"default": config.get("root")})
|
| 26 |
+
self.roots = {k: Path(v) for k, v in roots.items()}
|
| 27 |
+
manifests = [
|
| 28 |
+
json.loads((root / "manifest.json").read_text())
|
| 29 |
+
for root in self.roots.values()
|
| 30 |
+
]
|
| 31 |
+
for manifest in manifests:
|
| 32 |
+
if manifest.get("final_layer", config["layer"]) != config["layer"]:
|
| 33 |
+
raise ValueError(
|
| 34 |
+
"Final normalization requires the bank's final layer"
|
| 35 |
+
)
|
| 36 |
+
for name, value in manifest["backbone"].items():
|
| 37 |
+
if selection["identity"].get(name) != value:
|
| 38 |
+
raise ValueError(
|
| 39 |
+
"Feature backbone differs from selection: " + name
|
| 40 |
+
)
|
| 41 |
+
if manifest.get("serialization_sha256") != selection["identity"].get(
|
| 42 |
+
"serialization_sha256"
|
| 43 |
+
):
|
| 44 |
+
raise ValueError("Feature serialization differs from selection")
|
| 45 |
+
norms = [
|
| 46 |
+
torch.load(
|
| 47 |
+
root / "final-rmsnorm.pt", map_location="cpu", weights_only=True
|
| 48 |
+
)
|
| 49 |
+
for root in self.roots.values()
|
| 50 |
+
]
|
| 51 |
+
self.normalization = norms[0]
|
| 52 |
+
for norm in norms[1:]:
|
| 53 |
+
if norm["eps"] != norms[0]["eps"] or not torch.equal(
|
| 54 |
+
norm["weight"], norms[0]["weight"]
|
| 55 |
+
):
|
| 56 |
+
raise ValueError("Feature components have different normalization")
|
| 57 |
+
elif self.kind == "pt":
|
| 58 |
+
self.root = Path(config["root"])
|
| 59 |
+
self.index = (
|
| 60 |
+
json.loads((self.root / "index.json").read_text())
|
| 61 |
+
if (self.root / "index.json").exists()
|
| 62 |
+
else {}
|
| 63 |
+
)
|
| 64 |
+
else:
|
| 65 |
+
raise ValueError("Unknown feature mode: " + self.kind)
|
| 66 |
+
|
| 67 |
+
def identity(self):
|
| 68 |
+
"""Hash manifests/indices, not hundreds of GB of tensors on every run."""
|
| 69 |
+
paths = []
|
| 70 |
+
if self.kind in ("layer", "routed"):
|
| 71 |
+
for root in self.roots.values():
|
| 72 |
+
paths.extend([root / "manifest.json", root / "final-rmsnorm.pt"])
|
| 73 |
+
for child in root.iterdir():
|
| 74 |
+
if child.is_dir() and (child / "records.json").exists():
|
| 75 |
+
paths.extend([child / "records.json", child / "index.json"])
|
| 76 |
+
paths.extend(
|
| 77 |
+
p
|
| 78 |
+
for p in [
|
| 79 |
+
child / "shards.jsonl",
|
| 80 |
+
child / "layer-additions.jsonl",
|
| 81 |
+
]
|
| 82 |
+
if p.exists()
|
| 83 |
+
)
|
| 84 |
+
elif self.kind == "pt":
|
| 85 |
+
paths.extend(
|
| 86 |
+
p
|
| 87 |
+
for p in [self.root / "index.json", self.root / "records.json"]
|
| 88 |
+
if p.exists()
|
| 89 |
+
)
|
| 90 |
+
return {str(p): file_hash(p) for p in sorted(set(paths))}
|
| 91 |
+
|
| 92 |
+
def _load(self, row, split):
|
| 93 |
+
if row["id"] in self.resident:
|
| 94 |
+
return self.resident[row["id"]]
|
| 95 |
+
if self.kind == "pt":
|
| 96 |
+
entry = self.index.get("files", {}).get(row["id"])
|
| 97 |
+
if entry and row["id"] not in self.checked:
|
| 98 |
+
if (
|
| 99 |
+
entry["file"] != row["cache"]
|
| 100 |
+
or file_hash(self.root / row["cache"]) != entry["sha256"]
|
| 101 |
+
):
|
| 102 |
+
raise ValueError("Cached feature checksum differs: " + row["id"])
|
| 103 |
+
self.checked.add(row["id"])
|
| 104 |
+
return torch.load(
|
| 105 |
+
self.root / row["cache"], map_location="cpu", weights_only=True
|
| 106 |
+
)["hidden"]
|
| 107 |
+
if self.kind == "routed":
|
| 108 |
+
route_split = self.config.get("split_map", {}).get(split, split)
|
| 109 |
+
component, bank_split = self.selection["routes"][route_split][row["id"]]
|
| 110 |
+
else:
|
| 111 |
+
component = "default"
|
| 112 |
+
bank_split = self.config.get("split_map", {}).get(split, split)
|
| 113 |
+
key = component, bank_split
|
| 114 |
+
if key not in self.banks:
|
| 115 |
+
self.banks[key] = AddedLayerBank(
|
| 116 |
+
self.roots[component] / bank_split,
|
| 117 |
+
[self.config["layer"]],
|
| 118 |
+
self.config.get("max_open_shards", 128),
|
| 119 |
+
)
|
| 120 |
+
bank = self.banks[key]
|
| 121 |
+
expected = bank.records[row["id"]]
|
| 122 |
+
for name in ("ids", "positions", "order", "target"):
|
| 123 |
+
actual = (
|
| 124 |
+
row.get("_cache_target", row.get("target"))
|
| 125 |
+
if name == "target"
|
| 126 |
+
else row.get(name)
|
| 127 |
+
)
|
| 128 |
+
if actual != expected.get(name):
|
| 129 |
+
raise ValueError(
|
| 130 |
+
"Cached input identity differs: " + row["id"] + "/" + name
|
| 131 |
+
)
|
| 132 |
+
return bank.load(row["id"])[self.config["layer"]]
|
| 133 |
+
|
| 134 |
+
def prepare(self, rows, split):
|
| 135 |
+
return collate_records(rows, [self._load(row, split) for row in rows])
|
| 136 |
+
|
| 137 |
+
def preload(self, rows, split):
|
| 138 |
+
for row in rows:
|
| 139 |
+
self.resident[row["id"]] = self._load(row, split).clone()
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:aa18635a7398211d6eda876534f6cf3244a048a20f823d2190d2e4b14d836989
|
| 3 |
+
size 120678224
|
model/NOTICE.md
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Source attribution
|
| 2 |
+
|
| 3 |
+
Model equations and checkpoint layout were independently expressed in a
|
| 4 |
+
small PyTorch implementation after studying:
|
| 5 |
+
|
| 6 |
+
- Hugging Face Transformers, `models/gemma4_unified`, commit
|
| 7 |
+
`02d8fb9784e8f14a1251e4c992cd82a5762417c6`.
|
| 8 |
+
Copyright 2026 the HuggingFace Team. Licensed under Apache License 2.0:
|
| 9 |
+
<https://www.apache.org/licenses/LICENSE-2.0>.
|
| 10 |
+
- bitsandbytes NF4 codebook, version 0.50.2. MIT License; attribution below.
|
| 11 |
+
|
| 12 |
+
No upstream reference package is vendored or imported by the core model.
|
| 13 |
+
The optional accelerated quantized linear path imports bitsandbytes.
|
| 14 |
+
|
| 15 |
+
## bitsandbytes notice
|
| 16 |
+
|
| 17 |
+
MIT License
|
| 18 |
+
|
| 19 |
+
Copyright (c) Facebook, Inc. and its affiliates.
|
| 20 |
+
|
| 21 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 22 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 23 |
+
in the Software without restriction, including without limitation the rights
|
| 24 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 25 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 26 |
+
furnished to do so, subject to the following conditions:
|
| 27 |
+
|
| 28 |
+
The above copyright notice and this permission notice shall be included in all
|
| 29 |
+
copies or substantial portions of the Software.
|
| 30 |
+
|
| 31 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 32 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 33 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 34 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 35 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 36 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 37 |
+
SOFTWARE.
|
model/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Public model API, independent of datasets, optimizers and experiment runners."""
|
| 2 |
+
|
| 3 |
+
from .backbone import BackboneOutput, FeatureCache
|
| 4 |
+
from .gemma_backbone import Gemma4, Gemma4FeatureCache
|
| 5 |
+
from .model import Bazza, BazzaDecisionHead, DecisionOutput
|
| 6 |
+
|
| 7 |
+
__all__ = [
|
| 8 |
+
"Bazza",
|
| 9 |
+
"BazzaDecisionHead",
|
| 10 |
+
"DecisionOutput",
|
| 11 |
+
"Gemma4",
|
| 12 |
+
"Gemma4FeatureCache",
|
| 13 |
+
"BackboneOutput",
|
| 14 |
+
"FeatureCache",
|
| 15 |
+
]
|
model/backbone.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Backbone output contract and model-independent feature-cache machinery."""
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
from torch import nn
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
@dataclass
|
| 10 |
+
class BackboneOutput:
|
| 11 |
+
hidden: torch.Tensor
|
| 12 |
+
layer_states: dict[int, torch.Tensor]
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class FeatureCacheMixin:
|
| 16 |
+
"""Prepare cached token states and bind them to their source bank."""
|
| 17 |
+
|
| 18 |
+
def prepare_batch(self, rows, split):
|
| 19 |
+
batch = self.store.prepare(rows, split)
|
| 20 |
+
batch.split = split
|
| 21 |
+
batch.feature_space = (
|
| 22 |
+
"raw-final" if self.store.normalization is not None else "normalized-final"
|
| 23 |
+
)
|
| 24 |
+
batch.feature_source = self.cache_identity
|
| 25 |
+
return batch
|
| 26 |
+
|
| 27 |
+
def identity(self):
|
| 28 |
+
return self.store.identity()
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class FeatureCache(FeatureCacheMixin, nn.Module):
|
| 32 |
+
"""Frozen token-state source with optional stored final RMS normalization."""
|
| 33 |
+
|
| 34 |
+
def __init__(
|
| 35 |
+
self,
|
| 36 |
+
directory=None,
|
| 37 |
+
*,
|
| 38 |
+
selection,
|
| 39 |
+
format="pt",
|
| 40 |
+
banks=None,
|
| 41 |
+
split_map=None,
|
| 42 |
+
layer=None,
|
| 43 |
+
max_open_shards=128,
|
| 44 |
+
):
|
| 45 |
+
super().__init__()
|
| 46 |
+
if format in ("layer", "routed") and layer is None:
|
| 47 |
+
raise ValueError("A layer-shard cache requires an explicit final layer")
|
| 48 |
+
self.layer = layer
|
| 49 |
+
from features.store import FeatureStore
|
| 50 |
+
|
| 51 |
+
settings = {
|
| 52 |
+
"kind": format,
|
| 53 |
+
"root": directory,
|
| 54 |
+
"layer": f"layer-{layer}" if layer is not None else None,
|
| 55 |
+
"max_open_shards": max_open_shards,
|
| 56 |
+
"split_map": split_map or {"full": "validation", "retention": "validation"},
|
| 57 |
+
}
|
| 58 |
+
if banks:
|
| 59 |
+
settings["banks"] = banks
|
| 60 |
+
self.store = FeatureStore(settings, selection, "cpu")
|
| 61 |
+
from utils import digest
|
| 62 |
+
|
| 63 |
+
self.cache_identity = digest(
|
| 64 |
+
{
|
| 65 |
+
"files": self.store.identity(),
|
| 66 |
+
"root": str(directory),
|
| 67 |
+
"banks": {name: str(path) for name, path in (banks or {}).items()},
|
| 68 |
+
}
|
| 69 |
+
)
|
| 70 |
+
self.register_buffer("_device", torch.empty(0), persistent=False)
|
| 71 |
+
|
| 72 |
+
def forward(self, batch):
|
| 73 |
+
from features.bank import normalize_final
|
| 74 |
+
|
| 75 |
+
if batch.hidden is None:
|
| 76 |
+
batch = self.prepare_batch(batch.rows, batch.split)
|
| 77 |
+
expected = (
|
| 78 |
+
"raw-final" if self.store.normalization is not None else "normalized-final"
|
| 79 |
+
)
|
| 80 |
+
if (
|
| 81 |
+
batch.feature_space != expected
|
| 82 |
+
or batch.feature_source != self.cache_identity
|
| 83 |
+
):
|
| 84 |
+
raise ValueError(
|
| 85 |
+
"Cached batch belongs to a different feature space or cache"
|
| 86 |
+
)
|
| 87 |
+
raw = batch.hidden.to(self._device.device)
|
| 88 |
+
if self.store.normalization is None:
|
| 89 |
+
return BackboneOutput(raw, {})
|
| 90 |
+
return BackboneOutput(
|
| 91 |
+
normalize_final(raw, self.store.normalization), {self.layer: raw}
|
| 92 |
+
)
|
model/gemma.py
ADDED
|
@@ -0,0 +1,373 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Readable Gemma 4 Unified text backbone, including a PyTorch NF4 path.
|
| 2 |
+
|
| 3 |
+
Equations and checkpoint names follow Transformers' Apache-2.0 reference at
|
| 4 |
+
02d8fb9784e8f14a1251e4c992cd82a5762417c6. Runtime does not import Transformers.
|
| 5 |
+
This first port implements full-sequence text prefill, without a KV cache.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import hashlib
|
| 11 |
+
import json
|
| 12 |
+
import math
|
| 13 |
+
from dataclasses import dataclass
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
from safetensors import safe_open
|
| 18 |
+
from torch import nn
|
| 19 |
+
from torch.nn import functional as F
|
| 20 |
+
|
| 21 |
+
CHECKPOINT = "google/gemma-4-12B-it"
|
| 22 |
+
REVISION = "707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7"
|
| 23 |
+
WEIGHT_SHA256 = "5a84cb313260ac447237b890387116dfa8682e49a6b44bc585ae8353abbff18d"
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@dataclass(frozen=True)
|
| 27 |
+
class GemmaConfig:
|
| 28 |
+
hidden_size: int = 3840
|
| 29 |
+
intermediate_size: int = 15360
|
| 30 |
+
num_hidden_layers: int = 48
|
| 31 |
+
num_attention_heads: int = 16
|
| 32 |
+
num_key_value_heads: int = 8
|
| 33 |
+
num_global_key_value_heads: int = 1
|
| 34 |
+
head_dim: int = 256
|
| 35 |
+
global_head_dim: int = 512
|
| 36 |
+
vocab_size: int = 262144
|
| 37 |
+
sliding_window: int = 1024
|
| 38 |
+
rms_norm_eps: float = 1e-6
|
| 39 |
+
final_logit_softcapping: float = 30.0
|
| 40 |
+
layer_types: tuple[str, ...] = tuple(
|
| 41 |
+
"full_attention" if i % 6 == 5 else "sliding_attention" for i in range(48)
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
@classmethod
|
| 45 |
+
def from_json(cls, path: Path) -> "GemmaConfig":
|
| 46 |
+
raw = json.loads(path.read_text())["text_config"]
|
| 47 |
+
if (
|
| 48 |
+
raw.get("num_kv_shared_layers", 0)
|
| 49 |
+
or raw.get("enable_moe_block", False)
|
| 50 |
+
or raw.get("use_double_wide_mlp", False)
|
| 51 |
+
or raw.get("attention_bias", False)
|
| 52 |
+
or not raw.get("attention_k_eq_v", False)
|
| 53 |
+
or raw.get("hidden_activation") != "gelu_pytorch_tanh"
|
| 54 |
+
):
|
| 55 |
+
raise ValueError(
|
| 56 |
+
"This port supports the dense Unified 12B text architecture"
|
| 57 |
+
)
|
| 58 |
+
expected_rope = {
|
| 59 |
+
"full_attention": {
|
| 60 |
+
"partial_rotary_factor": 0.25,
|
| 61 |
+
"rope_theta": 1000000.0,
|
| 62 |
+
"rope_type": "proportional",
|
| 63 |
+
},
|
| 64 |
+
"sliding_attention": {"rope_theta": 10000.0, "rope_type": "default"},
|
| 65 |
+
}
|
| 66 |
+
if raw["rope_parameters"] != expected_rope:
|
| 67 |
+
raise ValueError("Unsupported rotary-position configuration")
|
| 68 |
+
values = {key: raw[key] for key in cls.__dataclass_fields__}
|
| 69 |
+
values["layer_types"] = tuple(values["layer_types"])
|
| 70 |
+
return cls(**values)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class RMSNorm(nn.Module):
|
| 74 |
+
def __init__(self, width: int, eps: float, with_scale: bool = True):
|
| 75 |
+
super().__init__()
|
| 76 |
+
self.eps = eps
|
| 77 |
+
self.weight = nn.Parameter(torch.ones(width)) if with_scale else None
|
| 78 |
+
|
| 79 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 80 |
+
y = x.float()
|
| 81 |
+
y = y * (y.square().mean(-1, keepdim=True) + self.eps).pow(-0.5)
|
| 82 |
+
if self.weight is not None:
|
| 83 |
+
y = y * self.weight.float()
|
| 84 |
+
return y.to(x.dtype)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
# NormalFloat4 levels, as used by the Apache-2.0 bitsandbytes reference.
|
| 88 |
+
# Scales remain FP32 here: no double quantization or fused CUDA dependency.
|
| 89 |
+
NF4_LEVELS = (
|
| 90 |
+
-1.0,
|
| 91 |
+
-0.6961928,
|
| 92 |
+
-0.52507305,
|
| 93 |
+
-0.39491749,
|
| 94 |
+
-0.28444138,
|
| 95 |
+
-0.18477343,
|
| 96 |
+
-0.09105004,
|
| 97 |
+
0.0,
|
| 98 |
+
0.07958030,
|
| 99 |
+
0.16093020,
|
| 100 |
+
0.24611230,
|
| 101 |
+
0.33791524,
|
| 102 |
+
0.44070983,
|
| 103 |
+
0.56261700,
|
| 104 |
+
0.72295684,
|
| 105 |
+
1.0,
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
class NF4Linear(nn.Module):
|
| 110 |
+
"""Packed 4-bit weights; materialize one BF16 matrix for each linear call.
|
| 111 |
+
|
| 112 |
+
This intentionally transparent implementation measures fit and quality.
|
| 113 |
+
It is not a fused quantized matmul performance claim.
|
| 114 |
+
"""
|
| 115 |
+
|
| 116 |
+
def __init__(self, weight: torch.Tensor, block_size: int = 64):
|
| 117 |
+
super().__init__()
|
| 118 |
+
self.shape = tuple(weight.shape)
|
| 119 |
+
self.block_size = block_size
|
| 120 |
+
if weight.numel() % block_size or block_size % 2:
|
| 121 |
+
raise ValueError("NF4 weight size must be divisible by an even block size")
|
| 122 |
+
blocks = weight.float().reshape(-1, block_size)
|
| 123 |
+
scales = blocks.abs().amax(-1).clamp_min(1e-12)
|
| 124 |
+
levels = torch.tensor(NF4_LEVELS, device=weight.device)
|
| 125 |
+
thresholds = (levels[1:] + levels[:-1]) / 2
|
| 126 |
+
codes = torch.bucketize((blocks / scales[:, None]).contiguous(), thresholds).to(
|
| 127 |
+
torch.uint8
|
| 128 |
+
)
|
| 129 |
+
packed = codes[:, 0::2] | (codes[:, 1::2] << 4)
|
| 130 |
+
self.register_buffer("packed", packed)
|
| 131 |
+
self.register_buffer("scales", scales)
|
| 132 |
+
self.register_buffer("levels", levels)
|
| 133 |
+
|
| 134 |
+
def dequantize(self, dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:
|
| 135 |
+
codes = torch.stack((self.packed & 15, self.packed >> 4), dim=-1).reshape(
|
| 136 |
+
-1, self.block_size
|
| 137 |
+
)
|
| 138 |
+
values = self.levels[codes.long()] * self.scales[:, None]
|
| 139 |
+
return values.reshape(self.shape).to(dtype)
|
| 140 |
+
|
| 141 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 142 |
+
return F.linear(x, self.dequantize(x.dtype))
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
|
| 146 |
+
a, b = x.chunk(2, dim=-1)
|
| 147 |
+
return x * cos[:, :, None] + torch.cat((-b, a), dim=-1) * sin[:, :, None]
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
class Attention(nn.Module):
|
| 151 |
+
def __init__(self, c: GemmaConfig, full: bool):
|
| 152 |
+
super().__init__()
|
| 153 |
+
self.heads = c.num_attention_heads
|
| 154 |
+
self.kv_heads = c.num_global_key_value_heads if full else c.num_key_value_heads
|
| 155 |
+
self.head_dim = c.global_head_dim if full else c.head_dim
|
| 156 |
+
self.q_proj = nn.Linear(c.hidden_size, self.heads * self.head_dim, bias=False)
|
| 157 |
+
self.k_proj = nn.Linear(
|
| 158 |
+
c.hidden_size, self.kv_heads * self.head_dim, bias=False
|
| 159 |
+
)
|
| 160 |
+
self.v_proj = (
|
| 161 |
+
None
|
| 162 |
+
if full
|
| 163 |
+
else nn.Linear(c.hidden_size, self.kv_heads * self.head_dim, bias=False)
|
| 164 |
+
)
|
| 165 |
+
self.o_proj = nn.Linear(self.heads * self.head_dim, c.hidden_size, bias=False)
|
| 166 |
+
self.q_norm = RMSNorm(self.head_dim, c.rms_norm_eps)
|
| 167 |
+
self.k_norm = RMSNorm(self.head_dim, c.rms_norm_eps)
|
| 168 |
+
self.v_norm = RMSNorm(self.head_dim, c.rms_norm_eps, with_scale=False)
|
| 169 |
+
|
| 170 |
+
def forward(self, x, positions, mask):
|
| 171 |
+
batch, length, _ = x.shape
|
| 172 |
+
cos, sin = positions
|
| 173 |
+
q = self.q_proj(x).reshape(batch, length, self.heads, self.head_dim)
|
| 174 |
+
k = self.k_proj(x).reshape(batch, length, self.kv_heads, self.head_dim)
|
| 175 |
+
v = self.v_proj(x).reshape_as(k) if self.v_proj is not None else k
|
| 176 |
+
q = rotary(self.q_norm(q), cos, sin).transpose(1, 2)
|
| 177 |
+
k = rotary(self.k_norm(k), cos, sin).transpose(1, 2)
|
| 178 |
+
v = self.v_norm(v).transpose(1, 2)
|
| 179 |
+
# Match the reference's explicit GQA expansion when a mask is present.
|
| 180 |
+
# Masked implicit GQA selects a different math kernel in Torch 2.8.
|
| 181 |
+
groups = self.heads // self.kv_heads
|
| 182 |
+
# Preserve the broadcast stride of the one-head global K/V tensors.
|
| 183 |
+
k = (
|
| 184 |
+
k[:, :, None]
|
| 185 |
+
.expand(-1, -1, groups, -1, -1)
|
| 186 |
+
.reshape(batch, self.heads, length, self.head_dim)
|
| 187 |
+
)
|
| 188 |
+
v = (
|
| 189 |
+
v[:, :, None]
|
| 190 |
+
.expand(-1, -1, groups, -1, -1)
|
| 191 |
+
.reshape(batch, self.heads, length, self.head_dim)
|
| 192 |
+
)
|
| 193 |
+
# Gemma's Q/K normalization makes its attention scale 1.0.
|
| 194 |
+
y = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, scale=1.0)
|
| 195 |
+
return self.o_proj(y.transpose(1, 2).reshape(batch, length, -1))
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
class MLP(nn.Module):
|
| 199 |
+
def __init__(self, c: GemmaConfig):
|
| 200 |
+
super().__init__()
|
| 201 |
+
self.gate_proj = nn.Linear(c.hidden_size, c.intermediate_size, bias=False)
|
| 202 |
+
self.up_proj = nn.Linear(c.hidden_size, c.intermediate_size, bias=False)
|
| 203 |
+
self.down_proj = nn.Linear(c.intermediate_size, c.hidden_size, bias=False)
|
| 204 |
+
|
| 205 |
+
def forward(self, x):
|
| 206 |
+
return self.down_proj(
|
| 207 |
+
F.gelu(self.gate_proj(x), approximate="tanh") * self.up_proj(x)
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
class Layer(nn.Module):
|
| 212 |
+
def __init__(self, c: GemmaConfig, full: bool):
|
| 213 |
+
super().__init__()
|
| 214 |
+
self.self_attn = Attention(c, full)
|
| 215 |
+
self.mlp = MLP(c)
|
| 216 |
+
self.input_layernorm = RMSNorm(c.hidden_size, c.rms_norm_eps)
|
| 217 |
+
self.post_attention_layernorm = RMSNorm(c.hidden_size, c.rms_norm_eps)
|
| 218 |
+
self.pre_feedforward_layernorm = RMSNorm(c.hidden_size, c.rms_norm_eps)
|
| 219 |
+
self.post_feedforward_layernorm = RMSNorm(c.hidden_size, c.rms_norm_eps)
|
| 220 |
+
self.register_buffer("layer_scalar", torch.ones(1))
|
| 221 |
+
|
| 222 |
+
def forward(self, x, positions, mask):
|
| 223 |
+
x = x + self.post_attention_layernorm(
|
| 224 |
+
self.self_attn(self.input_layernorm(x), positions, mask)
|
| 225 |
+
)
|
| 226 |
+
x = x + self.post_feedforward_layernorm(
|
| 227 |
+
self.mlp(self.pre_feedforward_layernorm(x))
|
| 228 |
+
)
|
| 229 |
+
return x * self.layer_scalar
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
class GemmaTextModel(nn.Module):
|
| 233 |
+
def __init__(self, config: GemmaConfig):
|
| 234 |
+
super().__init__()
|
| 235 |
+
self.config = config
|
| 236 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
| 237 |
+
self.layers = nn.ModuleList(
|
| 238 |
+
Layer(config, kind == "full_attention") for kind in config.layer_types
|
| 239 |
+
)
|
| 240 |
+
self.norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
|
| 241 |
+
|
| 242 |
+
def masks_and_positions(self, input_ids, attention_mask):
|
| 243 |
+
c = self.config
|
| 244 |
+
length = input_ids.shape[1]
|
| 245 |
+
p = torch.arange(length, device=input_ids.device)
|
| 246 |
+
allowed = p[:, None] >= p[None, :]
|
| 247 |
+
masks = {
|
| 248 |
+
"full_attention": allowed[None, None]
|
| 249 |
+
& attention_mask[:, None, None, :].bool(),
|
| 250 |
+
"sliding_attention": (
|
| 251 |
+
allowed & ((p[:, None] - p[None, :]) < c.sliding_window)
|
| 252 |
+
)[None, None]
|
| 253 |
+
& attention_mask[:, None, None, :].bool(),
|
| 254 |
+
}
|
| 255 |
+
positions = {}
|
| 256 |
+
for kind, dim, base, proportion in (
|
| 257 |
+
("sliding_attention", c.head_dim, 10000.0, 1.0),
|
| 258 |
+
("full_attention", c.global_head_dim, 1000000.0, 0.25),
|
| 259 |
+
):
|
| 260 |
+
count = int(dim * proportion // 2)
|
| 261 |
+
inv = 1.0 / (
|
| 262 |
+
base ** (torch.arange(0, 2 * count, 2, device=p.device).float() / dim)
|
| 263 |
+
)
|
| 264 |
+
inv = F.pad(inv, (0, dim // 2 - count))
|
| 265 |
+
angles = (inv[None, :, None] @ p[None, None, :].float()).transpose(1, 2)
|
| 266 |
+
angles = torch.cat((angles, angles), dim=-1)
|
| 267 |
+
positions[kind] = (
|
| 268 |
+
angles.cos().to(self.embed_tokens.weight.dtype),
|
| 269 |
+
angles.sin().to(self.embed_tokens.weight.dtype),
|
| 270 |
+
)
|
| 271 |
+
return masks, positions
|
| 272 |
+
|
| 273 |
+
def forward(self, input_ids, attention_mask=None):
|
| 274 |
+
if attention_mask is None:
|
| 275 |
+
attention_mask = torch.ones_like(input_ids)
|
| 276 |
+
if input_ids.ndim != 2 or attention_mask.shape != input_ids.shape:
|
| 277 |
+
raise ValueError("IDs and attention mask must have shape [batch, tokens]")
|
| 278 |
+
masks, positions = self.masks_and_positions(input_ids, attention_mask)
|
| 279 |
+
x = self.embed_tokens(input_ids)
|
| 280 |
+
# Match the BF16 rounding of the published embedding multiplier.
|
| 281 |
+
x = x * torch.tensor(
|
| 282 |
+
math.sqrt(self.config.hidden_size), dtype=x.dtype, device=x.device
|
| 283 |
+
)
|
| 284 |
+
for layer, kind in zip(self.layers, self.config.layer_types):
|
| 285 |
+
x = layer(x, positions[kind], masks[kind])
|
| 286 |
+
return self.norm(x)
|
| 287 |
+
|
| 288 |
+
def logits(self, hidden: torch.Tensor, token_ids: torch.Tensor | None = None):
|
| 289 |
+
weight = (
|
| 290 |
+
self.embed_tokens.weight
|
| 291 |
+
if token_ids is None
|
| 292 |
+
else self.embed_tokens.weight[token_ids]
|
| 293 |
+
)
|
| 294 |
+
logits = F.linear(hidden, weight).float()
|
| 295 |
+
cap = self.config.final_logit_softcapping
|
| 296 |
+
return (logits / cap).tanh() * cap
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
def verify_weights(directory: Path) -> str:
|
| 300 |
+
digest = hashlib.sha256()
|
| 301 |
+
with (directory / "model.safetensors").open("rb") as source:
|
| 302 |
+
for block in iter(lambda: source.read(16 * 1024 * 1024), b""):
|
| 303 |
+
digest.update(block)
|
| 304 |
+
actual = digest.hexdigest()
|
| 305 |
+
if actual != WEIGHT_SHA256:
|
| 306 |
+
raise ValueError(f"Gemma weight SHA-256 mismatch: {actual}")
|
| 307 |
+
return actual
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def load_model(
|
| 311 |
+
directory: Path, *, device="cuda", quantization="nf4", verify=True
|
| 312 |
+
) -> GemmaTextModel:
|
| 313 |
+
"""Stream official tensors and quantize only linear matrices.
|
| 314 |
+
|
| 315 |
+
Embeddings and normalization stay BF16. At no point are all dense model
|
| 316 |
+
weights resident on GPU. Quantization is frozen; head training is separate.
|
| 317 |
+
"""
|
| 318 |
+
if quantization not in ("nf4", "bnb-nf4", "bf16"):
|
| 319 |
+
raise ValueError("quantization must be nf4, bnb-nf4 or bf16")
|
| 320 |
+
if verify:
|
| 321 |
+
verify_weights(directory)
|
| 322 |
+
config = GemmaConfig.from_json(directory / "config.json")
|
| 323 |
+
with torch.device("meta"):
|
| 324 |
+
model = GemmaTextModel(config)
|
| 325 |
+
prefix = "model.language_model."
|
| 326 |
+
with safe_open(
|
| 327 |
+
directory / "model.safetensors", framework="pt", device="cpu"
|
| 328 |
+
) as source:
|
| 329 |
+
keys = set(source.keys())
|
| 330 |
+
wanted = {prefix + key for key in model.state_dict()}
|
| 331 |
+
if not wanted <= keys:
|
| 332 |
+
raise ValueError(f"Missing text weights: {sorted(wanted - keys)[:8]}")
|
| 333 |
+
for name, module in list(model.named_modules()):
|
| 334 |
+
if isinstance(module, nn.Linear) and quantization != "bf16":
|
| 335 |
+
weight = source.get_tensor(prefix + name + ".weight")
|
| 336 |
+
parent_name, attr = name.rsplit(".", 1)
|
| 337 |
+
if quantization == "nf4":
|
| 338 |
+
replacement = NF4Linear(weight.to(device))
|
| 339 |
+
else:
|
| 340 |
+
# Optional acceleration; model equations remain above.
|
| 341 |
+
from bitsandbytes.nn import Linear4bit, Params4bit
|
| 342 |
+
|
| 343 |
+
replacement = Linear4bit(
|
| 344 |
+
module.in_features,
|
| 345 |
+
module.out_features,
|
| 346 |
+
bias=False,
|
| 347 |
+
compute_dtype=torch.bfloat16,
|
| 348 |
+
compress_statistics=False,
|
| 349 |
+
quant_type="nf4",
|
| 350 |
+
)
|
| 351 |
+
replacement.weight = Params4bit(
|
| 352 |
+
weight,
|
| 353 |
+
requires_grad=False,
|
| 354 |
+
compress_statistics=False,
|
| 355 |
+
quant_type="nf4",
|
| 356 |
+
)
|
| 357 |
+
replacement = replacement.to(device)
|
| 358 |
+
setattr(model.get_submodule(parent_name), attr, replacement)
|
| 359 |
+
del weight
|
| 360 |
+
else:
|
| 361 |
+
for attr, tensor in list(module.named_parameters(recurse=False)):
|
| 362 |
+
value = source.get_tensor(prefix + name + "." + attr).to(
|
| 363 |
+
device=device, dtype=torch.bfloat16
|
| 364 |
+
)
|
| 365 |
+
setattr(module, attr, nn.Parameter(value, requires_grad=False))
|
| 366 |
+
for attr, tensor in list(module.named_buffers(recurse=False)):
|
| 367 |
+
value = source.get_tensor(prefix + name + "." + attr).to(
|
| 368 |
+
device=device, dtype=torch.bfloat16
|
| 369 |
+
)
|
| 370 |
+
setattr(module, attr, value)
|
| 371 |
+
model.requires_grad_(False).eval()
|
| 372 |
+
torch.cuda.empty_cache() if str(device).startswith("cuda") else None
|
| 373 |
+
return model
|
model/gemma_backbone.py
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Gemma-specific loading, readout extraction, and backbone adapters."""
|
| 2 |
+
|
| 3 |
+
import ast
|
| 4 |
+
import hashlib
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from torch import nn
|
| 9 |
+
|
| 10 |
+
from .backbone import BackboneOutput, FeatureCache
|
| 11 |
+
|
| 12 |
+
# Original source identity recorded by the existing frozen feature banks.
|
| 13 |
+
_REFERENCE_SOURCE_SHA256 = (
|
| 14 |
+
"e6eef82665a55340ca62e248d7aa3502d196e45b5689ab2e754166c8b2cef2d0"
|
| 15 |
+
)
|
| 16 |
+
_REFERENCE_AST_SHA256 = (
|
| 17 |
+
"006a9dd7edc3f07f61edd34b1b65a7df0878b34bf83ebfd7a05e851baae6ab93"
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _source_ast_hash(source):
|
| 22 |
+
tree = ast.parse(source)
|
| 23 |
+
for node in ast.walk(tree):
|
| 24 |
+
if isinstance(
|
| 25 |
+
node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)
|
| 26 |
+
):
|
| 27 |
+
if (
|
| 28 |
+
node.body
|
| 29 |
+
and isinstance(node.body[0], ast.Expr)
|
| 30 |
+
and isinstance(node.body[0].value, ast.Constant)
|
| 31 |
+
and isinstance(node.body[0].value.value, str)
|
| 32 |
+
):
|
| 33 |
+
node.body.pop(0)
|
| 34 |
+
imports = [
|
| 35 |
+
node for node in tree.body if isinstance(node, (ast.Import, ast.ImportFrom))
|
| 36 |
+
]
|
| 37 |
+
tree.body = sorted(imports, key=ast.dump) + [
|
| 38 |
+
node for node in tree.body if not isinstance(node, (ast.Import, ast.ImportFrom))
|
| 39 |
+
]
|
| 40 |
+
return hashlib.sha256(ast.dump(tree, include_attributes=False).encode()).hexdigest()
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def matches_gemma_source(expected_hash, source_path=None):
|
| 44 |
+
"""Accept the recorded source or a formatting-only version of its code.
|
| 45 |
+
|
| 46 |
+
Raw file hashes still identify complete runs. The legacy feature-bank hash
|
| 47 |
+
is accepted only when the executable syntax matches the frozen reference.
|
| 48 |
+
"""
|
| 49 |
+
from utils import file_hash
|
| 50 |
+
|
| 51 |
+
source_path = (
|
| 52 |
+
Path(source_path) if source_path else Path(__file__).with_name("gemma.py")
|
| 53 |
+
)
|
| 54 |
+
if file_hash(source_path) == expected_hash:
|
| 55 |
+
return True
|
| 56 |
+
return (
|
| 57 |
+
expected_hash == _REFERENCE_SOURCE_SHA256
|
| 58 |
+
and _source_ast_hash(source_path.read_text()) == _REFERENCE_AST_SHA256
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def load_gemma():
|
| 63 |
+
# Lazy import keeps cached training independent of live model construction.
|
| 64 |
+
from . import gemma
|
| 65 |
+
|
| 66 |
+
return gemma
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def load_letters(base, identity):
|
| 70 |
+
import json
|
| 71 |
+
|
| 72 |
+
import torch
|
| 73 |
+
from safetensors import safe_open
|
| 74 |
+
from tokenizers import Tokenizer
|
| 75 |
+
|
| 76 |
+
from utils import file_hash
|
| 77 |
+
|
| 78 |
+
config = json.loads((base / "config.json").read_text())
|
| 79 |
+
if file_hash(base / "tokenizer.json") != identity["tokenizer_sha256"]:
|
| 80 |
+
raise ValueError("Tokenizer differs from feature export")
|
| 81 |
+
tok = Tokenizer.from_file(str(base / "tokenizer.json"))
|
| 82 |
+
ids = [
|
| 83 |
+
tok.encode(c, add_special_tokens=False).ids
|
| 84 |
+
for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
| 85 |
+
]
|
| 86 |
+
if any(len(x) != 1 for x in ids):
|
| 87 |
+
raise ValueError("Expected single-token answer letters")
|
| 88 |
+
key = "model.language_model.embed_tokens.weight"
|
| 89 |
+
with safe_open(base / "model.safetensors", framework="pt", device="cpu") as source:
|
| 90 |
+
view = source.get_slice(key)
|
| 91 |
+
weight = torch.cat([view[x[0] : x[0] + 1] for x in ids]).contiguous()
|
| 92 |
+
if weight.shape != (26, identity["hidden_width"]) or weight.dtype != torch.bfloat16:
|
| 93 |
+
raise ValueError("Readout shape/dtype differs")
|
| 94 |
+
text = config["text_config"]
|
| 95 |
+
if not text.get("tie_word_embeddings"):
|
| 96 |
+
raise ValueError("This extraction expects the audited Gemma tied LM head")
|
| 97 |
+
provenance = {
|
| 98 |
+
"tensor_key": key,
|
| 99 |
+
"letter_token_ids": [x[0] for x in ids],
|
| 100 |
+
"selected_rows_sha256": hashlib.sha256(
|
| 101 |
+
weight.view(torch.uint8).numpy().tobytes()
|
| 102 |
+
).hexdigest(),
|
| 103 |
+
"weight_sha256_from_existing_feature_identity": identity["weight_sha256"],
|
| 104 |
+
"full_weight_hash_rescan": False,
|
| 105 |
+
"softcap": text["final_logit_softcapping"],
|
| 106 |
+
"readout_position": "last valid token of existing cached Bazza prompt",
|
| 107 |
+
"prompt_changed": False,
|
| 108 |
+
}
|
| 109 |
+
return weight, provenance
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
class Gemma4FeatureCache(FeatureCache):
|
| 113 |
+
"""Gemma final-layer cache using the stored effective RMSNorm weights."""
|
| 114 |
+
|
| 115 |
+
def __init__(
|
| 116 |
+
self,
|
| 117 |
+
directory=None,
|
| 118 |
+
*,
|
| 119 |
+
selection,
|
| 120 |
+
format="layer",
|
| 121 |
+
banks=None,
|
| 122 |
+
split_map=None,
|
| 123 |
+
layers=(48,),
|
| 124 |
+
max_open_shards=128,
|
| 125 |
+
):
|
| 126 |
+
if tuple(layers) != (48,):
|
| 127 |
+
raise ValueError(
|
| 128 |
+
"Whole-backbone replacement currently requires layers=(48,); prefix/tap fusion is separate work"
|
| 129 |
+
)
|
| 130 |
+
super().__init__(
|
| 131 |
+
directory,
|
| 132 |
+
selection=selection,
|
| 133 |
+
format=format,
|
| 134 |
+
banks=banks,
|
| 135 |
+
split_map=split_map,
|
| 136 |
+
layer=48,
|
| 137 |
+
max_open_shards=max_open_shards,
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
class Gemma4(nn.Module):
|
| 142 |
+
"""Live, frozen Gemma. Unpadded forwards retain the audited batch policy."""
|
| 143 |
+
|
| 144 |
+
def __init__(
|
| 145 |
+
self, weights=None, *, quantization="bnb-nf4", selection=None, encoder=None
|
| 146 |
+
):
|
| 147 |
+
super().__init__()
|
| 148 |
+
self.weights = Path(weights) if weights is not None else None
|
| 149 |
+
self.quantization = quantization
|
| 150 |
+
self.selection = selection or {}
|
| 151 |
+
self.encoder = (
|
| 152 |
+
encoder.requires_grad_(False).eval() if encoder is not None else None
|
| 153 |
+
)
|
| 154 |
+
self.register_buffer("_device", torch.empty(0), persistent=False)
|
| 155 |
+
|
| 156 |
+
def identity(self):
|
| 157 |
+
from utils import file_hash
|
| 158 |
+
|
| 159 |
+
if self.encoder is not None and self.weights is None:
|
| 160 |
+
return {"injected_encoder": type(self.encoder).__qualname__}
|
| 161 |
+
return {
|
| 162 |
+
str(path): file_hash(path)
|
| 163 |
+
for path in (
|
| 164 |
+
Path(__file__).with_name("gemma.py"),
|
| 165 |
+
self.weights / "tokenizer.json",
|
| 166 |
+
self.weights / "config.json",
|
| 167 |
+
)
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
def prepare_batch(self, rows, split):
|
| 171 |
+
from data import collate_records
|
| 172 |
+
|
| 173 |
+
batch = collate_records(rows)
|
| 174 |
+
batch.split = split
|
| 175 |
+
return batch
|
| 176 |
+
|
| 177 |
+
def _load(self):
|
| 178 |
+
if self.encoder is not None:
|
| 179 |
+
return
|
| 180 |
+
implementation = load_gemma()
|
| 181 |
+
expected = self.selection.get("identity", {})
|
| 182 |
+
for name, value in (
|
| 183 |
+
("checkpoint", implementation.CHECKPOINT),
|
| 184 |
+
("revision", implementation.REVISION),
|
| 185 |
+
("weight_sha256", implementation.WEIGHT_SHA256),
|
| 186 |
+
("quantization", self.quantization),
|
| 187 |
+
):
|
| 188 |
+
if expected.get(name) != value:
|
| 189 |
+
raise ValueError("Live backbone differs from selection: " + name)
|
| 190 |
+
if expected.get(
|
| 191 |
+
"implementation_sha256"
|
| 192 |
+
) is not None and not matches_gemma_source(
|
| 193 |
+
expected["implementation_sha256"], Path(implementation.__file__)
|
| 194 |
+
):
|
| 195 |
+
raise ValueError(
|
| 196 |
+
"Live backbone differs from selection: implementation_sha256"
|
| 197 |
+
)
|
| 198 |
+
self.encoder = (
|
| 199 |
+
implementation.load_model(
|
| 200 |
+
self.weights, device=self._device.device, quantization=self.quantization
|
| 201 |
+
)
|
| 202 |
+
.requires_grad_(False)
|
| 203 |
+
.eval()
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
def forward(self, batch):
|
| 207 |
+
if batch.hidden is not None:
|
| 208 |
+
raise ValueError("Live Gemma expects tokens, not cached hidden states")
|
| 209 |
+
batch = batch.to(self._device.device)
|
| 210 |
+
self._load()
|
| 211 |
+
chunks = []
|
| 212 |
+
for i, row in enumerate(batch.rows):
|
| 213 |
+
n = len(row["ids"])
|
| 214 |
+
hidden = self.encoder(
|
| 215 |
+
batch.input_ids[i : i + 1, :n], batch.attention_mask[i : i + 1, :n]
|
| 216 |
+
)
|
| 217 |
+
chunks.append(
|
| 218 |
+
torch.nn.functional.pad(hidden, (0, 0, 0, batch.input_ids.shape[1] - n))
|
| 219 |
+
)
|
| 220 |
+
return BackboneOutput(torch.cat(chunks), {})
|
model/model.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Bazza composition; state-dict keys of the existing mixed head are unchanged."""
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
from torch import nn
|
| 7 |
+
|
| 8 |
+
from .readers import make_head
|
| 9 |
+
from .readout import LetterReadout, MixedReadout
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class BazzaDecisionHead(MixedReadout):
|
| 13 |
+
def __init__(
|
| 14 |
+
self,
|
| 15 |
+
letter_weight,
|
| 16 |
+
softcap,
|
| 17 |
+
encoder_width=3840,
|
| 18 |
+
width=1024,
|
| 19 |
+
attention_heads=4,
|
| 20 |
+
low_steps=2,
|
| 21 |
+
high_cycles=2,
|
| 22 |
+
gate_width=32,
|
| 23 |
+
rank=64,
|
| 24 |
+
):
|
| 25 |
+
decision = make_head(
|
| 26 |
+
encoder_width, width, attention_heads, low_steps, high_cycles, "bigru"
|
| 27 |
+
)
|
| 28 |
+
super().__init__(
|
| 29 |
+
decision,
|
| 30 |
+
LetterReadout(letter_weight, softcap),
|
| 31 |
+
encoder_width,
|
| 32 |
+
width,
|
| 33 |
+
gate_width,
|
| 34 |
+
rank,
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
def forward(self, hidden, attention_mask, option_positions, option_mask):
|
| 38 |
+
return super().forward(hidden, attention_mask, option_positions, option_mask)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
@dataclass
|
| 42 |
+
class DecisionOutput:
|
| 43 |
+
logits: torch.Tensor
|
| 44 |
+
option_mask: torch.Tensor
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class Bazza(nn.Module):
|
| 48 |
+
"""One frozen backbone plus a trainable head; the original LM head is retained.
|
| 49 |
+
|
| 50 |
+
A backbone adapter must return normalized final token states. Generation and
|
| 51 |
+
System 1/System 2 routing are intentionally separate future operations.
|
| 52 |
+
"""
|
| 53 |
+
|
| 54 |
+
def __init__(self, backbone, head, *, head_precision="bf16"):
|
| 55 |
+
super().__init__()
|
| 56 |
+
self.backbone = backbone.requires_grad_(False).eval()
|
| 57 |
+
self.head = head
|
| 58 |
+
self.head_precision = head_precision
|
| 59 |
+
|
| 60 |
+
def train(self, mode=True):
|
| 61 |
+
super().train(mode)
|
| 62 |
+
self.backbone.eval()
|
| 63 |
+
return self
|
| 64 |
+
|
| 65 |
+
@property
|
| 66 |
+
def checkpoint_module(self):
|
| 67 |
+
"""Reference the frozen backbone by identity; save only the trained head."""
|
| 68 |
+
return self.head
|
| 69 |
+
|
| 70 |
+
def forward(self, batch):
|
| 71 |
+
batch = batch.to(self.head.letter.weight.device)
|
| 72 |
+
# Match the exporter/live path: backbone and original RMSNorm run
|
| 73 |
+
# outside the head's autocast context.
|
| 74 |
+
with (
|
| 75 |
+
torch.no_grad(),
|
| 76 |
+
torch.autocast(batch.attention_mask.device.type, enabled=False),
|
| 77 |
+
):
|
| 78 |
+
features = self.backbone(batch)
|
| 79 |
+
if not torch.isfinite(features.hidden).all():
|
| 80 |
+
raise FloatingPointError("Non-finite frozen features")
|
| 81 |
+
hidden = (
|
| 82 |
+
features.hidden.float()
|
| 83 |
+
if self.head_precision == "fp32"
|
| 84 |
+
else features.hidden
|
| 85 |
+
)
|
| 86 |
+
logits = self.head(
|
| 87 |
+
hidden, batch.attention_mask, batch.option_positions, batch.option_mask
|
| 88 |
+
)
|
| 89 |
+
return DecisionOutput(logits, batch.option_mask)
|
| 90 |
+
|
| 91 |
+
def diagnostics(self):
|
| 92 |
+
return self.head.diagnostics()
|
model/readers.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Cached-feature readers around the unchanged Bazza decision core."""
|
| 2 |
+
|
| 3 |
+
from dataclasses import replace
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
from torch import nn
|
| 7 |
+
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
|
| 8 |
+
|
| 9 |
+
from .reasoner import ReasoningConfig, ReasoningHead
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class RecurrentDecisionHead(nn.Module):
|
| 13 |
+
def __init__(
|
| 14 |
+
self,
|
| 15 |
+
encoder_width: int = 3840,
|
| 16 |
+
width: int = 256,
|
| 17 |
+
attention_heads: int = 4,
|
| 18 |
+
low_steps: int = 2,
|
| 19 |
+
high_cycles: int = 2,
|
| 20 |
+
):
|
| 21 |
+
super().__init__()
|
| 22 |
+
self.reasoner = ReasoningHead(
|
| 23 |
+
ReasoningConfig(
|
| 24 |
+
encoder_width=encoder_width,
|
| 25 |
+
width=width,
|
| 26 |
+
attention_heads=attention_heads,
|
| 27 |
+
low_steps=low_steps,
|
| 28 |
+
high_cycles=high_cycles,
|
| 29 |
+
)
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
def forward(self, hidden, attention_mask, option_positions, option_mask):
|
| 33 |
+
# A causal BOS state cannot summarize a request. Prepend the final
|
| 34 |
+
# valid token as the existing head's global-state seed. Option seeds
|
| 35 |
+
# come AFTER each option description so they have read its contents.
|
| 36 |
+
last = attention_mask.long().sum(-1) - 1
|
| 37 |
+
summary = hidden[torch.arange(hidden.shape[0], device=hidden.device), last]
|
| 38 |
+
memory = torch.cat((summary[:, None], hidden), dim=1)
|
| 39 |
+
mask = torch.cat(
|
| 40 |
+
(torch.ones_like(attention_mask[:, :1]), attention_mask), dim=1
|
| 41 |
+
)
|
| 42 |
+
positions = option_positions + 1
|
| 43 |
+
result = self.reasoner(
|
| 44 |
+
memory, mask, positions, positions[:, :0], action_mask=option_mask
|
| 45 |
+
)
|
| 46 |
+
return result.action_logits
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class FeatureReaderHead(nn.Module):
|
| 50 |
+
def __init__(
|
| 51 |
+
self, encoder_width, width, attention_heads, low_steps, high_cycles, reader
|
| 52 |
+
):
|
| 53 |
+
super().__init__()
|
| 54 |
+
self.decision = RecurrentDecisionHead(
|
| 55 |
+
encoder_width, width, attention_heads, low_steps, high_cycles
|
| 56 |
+
)
|
| 57 |
+
# Move the existing projection ahead of the optional token reader. The
|
| 58 |
+
# candidate/global seed construction and recurrent core are unchanged.
|
| 59 |
+
self.project = self.decision.reasoner.project
|
| 60 |
+
self.decision.reasoner.project = nn.Identity()
|
| 61 |
+
self.decision.reasoner.config = replace(
|
| 62 |
+
self.decision.reasoner.config, encoder_width=width
|
| 63 |
+
)
|
| 64 |
+
self.input_norm = nn.LayerNorm(encoder_width)
|
| 65 |
+
self.reader = reader
|
| 66 |
+
if reader != "norm":
|
| 67 |
+
self.reader_norm = nn.LayerNorm(width)
|
| 68 |
+
self.gate = nn.Parameter(torch.tensor(0.1))
|
| 69 |
+
if reader == "mlp":
|
| 70 |
+
self.update = nn.Sequential(
|
| 71 |
+
nn.Linear(width, 2 * width), nn.GELU(), nn.Linear(2 * width, width)
|
| 72 |
+
)
|
| 73 |
+
elif reader == "bigru":
|
| 74 |
+
if width % 2:
|
| 75 |
+
raise ValueError("Bidirectional GRU needs an even decision width")
|
| 76 |
+
self.update = nn.GRU(
|
| 77 |
+
width, width // 2, batch_first=True, bidirectional=True
|
| 78 |
+
)
|
| 79 |
+
elif reader != "norm":
|
| 80 |
+
raise ValueError("Unknown feature reader: " + reader)
|
| 81 |
+
|
| 82 |
+
def forward(self, hidden, attention_mask, option_positions, option_mask):
|
| 83 |
+
memory = self.project(self.input_norm(hidden))
|
| 84 |
+
if self.reader == "mlp":
|
| 85 |
+
memory = memory + self.gate.tanh() * self.update(self.reader_norm(memory))
|
| 86 |
+
elif self.reader == "bigru":
|
| 87 |
+
lengths = attention_mask.long().sum(-1)
|
| 88 |
+
expected = (
|
| 89 |
+
torch.arange(hidden.shape[1], device=hidden.device)[None]
|
| 90 |
+
< lengths[:, None]
|
| 91 |
+
)
|
| 92 |
+
if torch.any(lengths < 1) or not torch.equal(
|
| 93 |
+
attention_mask.bool(), expected
|
| 94 |
+
):
|
| 95 |
+
raise ValueError("Token GRU expects nonempty, right-padded inputs")
|
| 96 |
+
# Explicit FP32 recurrent compute works with this image's cuDNN;
|
| 97 |
+
# projection/attention retain the baseline's BF16 autocast policy.
|
| 98 |
+
with torch.autocast(hidden.device.type, enabled=False):
|
| 99 |
+
packed = pack_padded_sequence(
|
| 100 |
+
self.reader_norm(memory.float()),
|
| 101 |
+
lengths.cpu(),
|
| 102 |
+
batch_first=True,
|
| 103 |
+
enforce_sorted=False,
|
| 104 |
+
)
|
| 105 |
+
output, _ = self.update(packed)
|
| 106 |
+
output, _ = pad_packed_sequence(
|
| 107 |
+
output, batch_first=True, total_length=hidden.shape[1]
|
| 108 |
+
)
|
| 109 |
+
memory = memory + self.gate.tanh() * output.to(memory.dtype)
|
| 110 |
+
return self.decision(memory, attention_mask, option_positions, option_mask)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def make_head(
|
| 114 |
+
encoder_width,
|
| 115 |
+
width=256,
|
| 116 |
+
attention_heads=4,
|
| 117 |
+
low_steps=2,
|
| 118 |
+
high_cycles=2,
|
| 119 |
+
head_type="recurrent",
|
| 120 |
+
):
|
| 121 |
+
if head_type == "recurrent":
|
| 122 |
+
return RecurrentDecisionHead(
|
| 123 |
+
encoder_width, width, attention_heads, low_steps, high_cycles
|
| 124 |
+
)
|
| 125 |
+
return FeatureReaderHead(
|
| 126 |
+
encoder_width, width, attention_heads, low_steps, high_cycles, head_type
|
| 127 |
+
)
|
model/readout.py
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Frozen letter prior, request gate and residual answer projection."""
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
from torch import nn
|
| 7 |
+
from torch.nn import functional as F
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class LetterReadout(nn.Module):
|
| 11 |
+
def __init__(self, weight, softcap):
|
| 12 |
+
super().__init__()
|
| 13 |
+
self.register_buffer("weight", weight.clone())
|
| 14 |
+
self.softcap = softcap
|
| 15 |
+
|
| 16 |
+
def forward(self, hidden, mask, positions, option_mask):
|
| 17 |
+
last = mask.long().sum(-1) - 1
|
| 18 |
+
final = hidden[torch.arange(len(hidden), device=hidden.device), last]
|
| 19 |
+
logits = F.linear(final, self.weight[: option_mask.shape[1]]).float()
|
| 20 |
+
if self.softcap:
|
| 21 |
+
logits = self.softcap * (logits / self.softcap).tanh()
|
| 22 |
+
return logits.masked_fill(~option_mask, -1e4)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class HybridReadout(nn.Module):
|
| 26 |
+
def __init__(self, decision, letter, initial_prior=0.1):
|
| 27 |
+
super().__init__()
|
| 28 |
+
self.decision, self.letter = decision, letter
|
| 29 |
+
# Deterministic scalar: adding this branch must not change initialization
|
| 30 |
+
# of the shared decision core. AdamW uses the same auxiliary LR as before.
|
| 31 |
+
self.prior_gate = nn.Parameter(torch.tensor(math.atanh(initial_prior)))
|
| 32 |
+
|
| 33 |
+
def forward(self, hidden, mask, positions, option_mask):
|
| 34 |
+
decision = self.decision(hidden, mask, positions, option_mask)
|
| 35 |
+
prior = self.letter(hidden, mask, positions, option_mask)
|
| 36 |
+
return (decision + self.prior_gate.tanh() * prior).masked_fill(
|
| 37 |
+
~option_mask, -1e4
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def final_state(hidden, mask):
|
| 42 |
+
last = mask.long().sum(-1) - 1
|
| 43 |
+
return hidden[torch.arange(len(hidden), device=hidden.device), last]
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def confidence_features(logits, mask):
|
| 47 |
+
# Confidence is observable at inference. Detach to stop the scorer learning
|
| 48 |
+
# to manipulate the gate indirectly through confidence, rather than answers.
|
| 49 |
+
probs = logits.detach().float().masked_fill(~mask, -1e4).softmax(-1)
|
| 50 |
+
count = mask.sum(-1).float()
|
| 51 |
+
entropy = -(probs * probs.clamp_min(1e-8).log()).sum(-1) / count.log()
|
| 52 |
+
top = probs.topk(2, dim=-1).values
|
| 53 |
+
return torch.stack((entropy, top[:, 0] - top[:, 1]), -1)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class RequestGateReadout(nn.Module):
|
| 57 |
+
def __init__(self, decision, letter, encoder_width=3840, gate_width=32):
|
| 58 |
+
super().__init__()
|
| 59 |
+
self.decision, self.letter = decision, letter
|
| 60 |
+
self.request_norm = nn.LayerNorm(encoder_width)
|
| 61 |
+
self.request_project = nn.Linear(encoder_width, gate_width)
|
| 62 |
+
# Name the gate's output projection action_score so the existing optimizer
|
| 63 |
+
# puts it on the unchanged auxiliary AdamW LR, as for the original scorer.
|
| 64 |
+
self.action_score = nn.Linear(gate_width + 5, 1)
|
| 65 |
+
nn.init.zeros_(self.action_score.weight)
|
| 66 |
+
nn.init.constant_(self.action_score.bias, math.log(0.1 / 0.9))
|
| 67 |
+
self.last_gate = None
|
| 68 |
+
|
| 69 |
+
def forward(self, hidden, mask, positions, option_mask):
|
| 70 |
+
decision = self.decision(hidden, mask, positions, option_mask)
|
| 71 |
+
prior = self.letter(hidden, mask, positions, option_mask)
|
| 72 |
+
state = F.silu(
|
| 73 |
+
self.request_project(self.request_norm(final_state(hidden, mask)))
|
| 74 |
+
)
|
| 75 |
+
stats = torch.cat(
|
| 76 |
+
(
|
| 77 |
+
confidence_features(decision, option_mask),
|
| 78 |
+
confidence_features(prior, option_mask),
|
| 79 |
+
option_mask.sum(-1, keepdim=True).float().log(),
|
| 80 |
+
),
|
| 81 |
+
-1,
|
| 82 |
+
)
|
| 83 |
+
gate = self.action_score(torch.cat((state, stats), -1)).float().sigmoid()
|
| 84 |
+
self.last_gate = gate.detach()
|
| 85 |
+
return (decision + gate * prior).masked_fill(~option_mask, -1e4)
|
| 86 |
+
|
| 87 |
+
def diagnostics(self):
|
| 88 |
+
return {
|
| 89 |
+
"request_gate_mean": float(self.last_gate.mean()),
|
| 90 |
+
"request_gate_min": float(self.last_gate.min()),
|
| 91 |
+
"request_gate_max": float(self.last_gate.max()),
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
class AnswerAdapter(nn.Module):
|
| 96 |
+
def __init__(self, reasoner_width=1024, encoder_width=3840, rank=64):
|
| 97 |
+
super().__init__()
|
| 98 |
+
self.norm = nn.LayerNorm(reasoner_width)
|
| 99 |
+
self.down = nn.Linear(reasoner_width, rank, bias=False)
|
| 100 |
+
self.action_score = nn.Linear(rank, encoder_width, bias=False)
|
| 101 |
+
nn.init.zeros_(self.action_score.weight)
|
| 102 |
+
|
| 103 |
+
def forward(self, high):
|
| 104 |
+
return self.action_score(F.silu(self.down(self.norm(high))))
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
class ResidualAnswerReadout(nn.Module):
|
| 108 |
+
def __init__(
|
| 109 |
+
self, decision, letter, hybrid, reasoner_width=1024, encoder_width=3840, rank=64
|
| 110 |
+
):
|
| 111 |
+
super().__init__()
|
| 112 |
+
self.decision, self.letter, self.hybrid = decision, letter, hybrid
|
| 113 |
+
self.answer_adapter = AnswerAdapter(reasoner_width, encoder_width, rank)
|
| 114 |
+
if hybrid:
|
| 115 |
+
self.prior_gate = nn.Parameter(torch.tensor(math.atanh(0.1)))
|
| 116 |
+
self.last_relative_delta = None
|
| 117 |
+
|
| 118 |
+
def forward(self, hidden, mask, positions, option_mask):
|
| 119 |
+
# Capture the existing reasoner's output from its one normal forward.
|
| 120 |
+
# A scoped hook avoids duplicating/reimplementing the reader or reasoner.
|
| 121 |
+
high = []
|
| 122 |
+
handle = self.decision.decision.reasoner.register_forward_hook(
|
| 123 |
+
lambda module, inputs, output: high.append(output.high_state)
|
| 124 |
+
)
|
| 125 |
+
try:
|
| 126 |
+
decision = self.decision(hidden, mask, positions, option_mask)
|
| 127 |
+
finally:
|
| 128 |
+
handle.remove()
|
| 129 |
+
if len(high) != 1:
|
| 130 |
+
raise ValueError("Expected exactly one reasoner forward")
|
| 131 |
+
delta = self.answer_adapter(high.pop())
|
| 132 |
+
original = final_state(hidden, mask)
|
| 133 |
+
self.last_relative_delta = delta.detach().float().square().mean(
|
| 134 |
+
-1
|
| 135 |
+
).sqrt() / original.detach().float().square().mean(-1).sqrt().clamp_min(1e-8)
|
| 136 |
+
logits = F.linear(
|
| 137 |
+
original + delta, self.letter.weight[: option_mask.shape[1]]
|
| 138 |
+
).float()
|
| 139 |
+
cap = self.letter.softcap
|
| 140 |
+
if cap:
|
| 141 |
+
logits = cap * (logits / cap).tanh()
|
| 142 |
+
if self.hybrid:
|
| 143 |
+
logits = decision + self.prior_gate.tanh() * logits
|
| 144 |
+
return logits.masked_fill(~option_mask, -1e4)
|
| 145 |
+
|
| 146 |
+
def diagnostics(self):
|
| 147 |
+
values = {
|
| 148 |
+
"answer_delta_relative_rms_mean": float(self.last_relative_delta.mean())
|
| 149 |
+
}
|
| 150 |
+
if self.hybrid:
|
| 151 |
+
values["pretrained_prior_coefficient"] = float(
|
| 152 |
+
self.prior_gate.detach().tanh()
|
| 153 |
+
)
|
| 154 |
+
return values
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
class MixedReadout(RequestGateReadout):
|
| 158 |
+
def __init__(
|
| 159 |
+
self,
|
| 160 |
+
decision,
|
| 161 |
+
letter,
|
| 162 |
+
encoder_width=3840,
|
| 163 |
+
reasoner_width=1024,
|
| 164 |
+
gate_width=32,
|
| 165 |
+
rank=64,
|
| 166 |
+
):
|
| 167 |
+
component_rng = torch.get_rng_state()
|
| 168 |
+
super().__init__(decision, letter, encoder_width, gate_width)
|
| 169 |
+
# Match each component's separate trial initialization. Adding the gate
|
| 170 |
+
# must not move the adapter's CPU initialization along the RNG sequence.
|
| 171 |
+
with torch.random.fork_rng(devices=[]):
|
| 172 |
+
torch.set_rng_state(component_rng)
|
| 173 |
+
self.answer_adapter = AnswerAdapter(reasoner_width, encoder_width, rank)
|
| 174 |
+
self.last_relative_delta = None
|
| 175 |
+
|
| 176 |
+
def forward(self, hidden, mask, positions, option_mask):
|
| 177 |
+
high = []
|
| 178 |
+
handle = self.decision.decision.reasoner.register_forward_hook(
|
| 179 |
+
lambda module, inputs, output: high.append(output.high_state)
|
| 180 |
+
)
|
| 181 |
+
try:
|
| 182 |
+
decision = self.decision(hidden, mask, positions, option_mask)
|
| 183 |
+
finally:
|
| 184 |
+
handle.remove()
|
| 185 |
+
if len(high) != 1:
|
| 186 |
+
raise ValueError("Expected exactly one reasoner forward")
|
| 187 |
+
original = final_state(hidden, mask)
|
| 188 |
+
delta = self.answer_adapter(high.pop())
|
| 189 |
+
prior = F.linear(
|
| 190 |
+
original + delta, self.letter.weight[: option_mask.shape[1]]
|
| 191 |
+
).float()
|
| 192 |
+
cap = self.letter.softcap
|
| 193 |
+
if cap:
|
| 194 |
+
prior = cap * (prior / cap).tanh()
|
| 195 |
+
prior = prior.masked_fill(~option_mask, -1e4)
|
| 196 |
+
state = F.silu(self.request_project(self.request_norm(original)))
|
| 197 |
+
stats = torch.cat(
|
| 198 |
+
(
|
| 199 |
+
confidence_features(decision, option_mask),
|
| 200 |
+
confidence_features(prior, option_mask),
|
| 201 |
+
option_mask.sum(-1, keepdim=True).float().log(),
|
| 202 |
+
),
|
| 203 |
+
-1,
|
| 204 |
+
)
|
| 205 |
+
gate = self.action_score(torch.cat((state, stats), -1)).float().sigmoid()
|
| 206 |
+
self.last_gate = gate.detach()
|
| 207 |
+
self.last_relative_delta = delta.detach().float().square().mean(
|
| 208 |
+
-1
|
| 209 |
+
).sqrt() / original.detach().float().square().mean(-1).sqrt().clamp_min(1e-8)
|
| 210 |
+
return (decision + gate * prior).masked_fill(~option_mask, -1e4)
|
| 211 |
+
|
| 212 |
+
def diagnostics(self):
|
| 213 |
+
return {
|
| 214 |
+
**super().diagnostics(),
|
| 215 |
+
"answer_delta_relative_rms_mean": float(self.last_relative_delta.mean()),
|
| 216 |
+
}
|
model/reasoner.py
ADDED
|
@@ -0,0 +1,398 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Two-timescale action/slot reasoning with a small copy-capable value decoder.
|
| 2 |
+
|
| 3 |
+
This is an HRM-inspired candidate, not a reproduction of the puzzle model's
|
| 4 |
+
training algorithm. The encoder token states remain available to the decoder.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
from torch import nn
|
| 13 |
+
from torch.nn import functional as F
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@dataclass(frozen=True)
|
| 17 |
+
class ReasoningConfig:
|
| 18 |
+
encoder_width: int = 1024
|
| 19 |
+
width: int = 256
|
| 20 |
+
attention_heads: int = 4
|
| 21 |
+
low_steps: int = 2
|
| 22 |
+
high_cycles: int = 2
|
| 23 |
+
slot_gate_classes: int = 4 # omit, select, copy, generate
|
| 24 |
+
|
| 25 |
+
def __post_init__(self) -> None:
|
| 26 |
+
if self.width < 1 or self.width % self.attention_heads:
|
| 27 |
+
raise ValueError("reasoning width must be divisible by attention_heads")
|
| 28 |
+
if self.encoder_width < 1 or self.low_steps < 1 or self.high_cycles < 1:
|
| 29 |
+
raise ValueError("encoder width and recurrence counts must be positive")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@dataclass
|
| 33 |
+
class ReasoningOutput:
|
| 34 |
+
action_logits: torch.Tensor
|
| 35 |
+
slot_gate_logits: torch.Tensor
|
| 36 |
+
slot_states: torch.Tensor
|
| 37 |
+
memory: torch.Tensor
|
| 38 |
+
high_state: torch.Tensor
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class LowLevelUpdate(nn.Module):
|
| 42 |
+
"""Shared update applied repeatedly to the small action/slot workspace."""
|
| 43 |
+
|
| 44 |
+
def __init__(self, width: int, attention_heads: int) -> None:
|
| 45 |
+
super().__init__()
|
| 46 |
+
self.high_condition = nn.Linear(width, width)
|
| 47 |
+
self.self_norm = nn.LayerNorm(width)
|
| 48 |
+
self.self_attention = nn.MultiheadAttention(
|
| 49 |
+
width, attention_heads, batch_first=True
|
| 50 |
+
)
|
| 51 |
+
self.cross_norm = nn.LayerNorm(width)
|
| 52 |
+
self.cross_attention = nn.MultiheadAttention(
|
| 53 |
+
width, attention_heads, batch_first=True
|
| 54 |
+
)
|
| 55 |
+
self.ffn_norm = nn.LayerNorm(width)
|
| 56 |
+
self.ffn = nn.Sequential(
|
| 57 |
+
nn.Linear(width, 2 * width), nn.GELU(), nn.Linear(2 * width, width)
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
def forward(
|
| 61 |
+
self,
|
| 62 |
+
low: torch.Tensor,
|
| 63 |
+
high: torch.Tensor,
|
| 64 |
+
memory: torch.Tensor,
|
| 65 |
+
workspace_mask: torch.Tensor,
|
| 66 |
+
memory_mask: torch.Tensor,
|
| 67 |
+
) -> torch.Tensor:
|
| 68 |
+
conditioned = low + self.high_condition(high)[:, None, :]
|
| 69 |
+
query = self.self_norm(conditioned)
|
| 70 |
+
update, _ = self.self_attention(
|
| 71 |
+
query,
|
| 72 |
+
query,
|
| 73 |
+
query,
|
| 74 |
+
key_padding_mask=~workspace_mask,
|
| 75 |
+
need_weights=False,
|
| 76 |
+
)
|
| 77 |
+
low = low + update
|
| 78 |
+
update, _ = self.cross_attention(
|
| 79 |
+
self.cross_norm(low),
|
| 80 |
+
memory,
|
| 81 |
+
memory,
|
| 82 |
+
key_padding_mask=~memory_mask,
|
| 83 |
+
need_weights=False,
|
| 84 |
+
)
|
| 85 |
+
low = low + update
|
| 86 |
+
low = low + self.ffn(self.ffn_norm(low))
|
| 87 |
+
return low.masked_fill(~workspace_mask[:, :, None], 0)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
class ReasoningHead(nn.Module):
|
| 91 |
+
"""Slow global state and fast per-action/per-slot states over encoder memory."""
|
| 92 |
+
|
| 93 |
+
def __init__(self, config: ReasoningConfig) -> None:
|
| 94 |
+
super().__init__()
|
| 95 |
+
self.config = config
|
| 96 |
+
self.project = nn.Linear(config.encoder_width, config.width)
|
| 97 |
+
self.high_init = nn.Linear(config.width, config.width)
|
| 98 |
+
self.low_update = LowLevelUpdate(config.width, config.attention_heads)
|
| 99 |
+
self.high_update = nn.GRUCell(config.width, config.width)
|
| 100 |
+
self.action_norm = nn.LayerNorm(config.width)
|
| 101 |
+
self.action_score = nn.Linear(config.width, 1)
|
| 102 |
+
self.slot_norm = nn.LayerNorm(config.width)
|
| 103 |
+
self.slot_gate = nn.Linear(config.width, config.slot_gate_classes)
|
| 104 |
+
|
| 105 |
+
@staticmethod
|
| 106 |
+
def _gather(memory: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
| 107 |
+
if positions.ndim != 2 or positions.shape[0] != memory.shape[0]:
|
| 108 |
+
raise ValueError("marker positions must have shape [batch, markers]")
|
| 109 |
+
if torch.any(positions < 0) or torch.any(positions >= memory.shape[1]):
|
| 110 |
+
raise ValueError("marker positions are outside encoder memory")
|
| 111 |
+
index = positions[:, :, None].expand(-1, -1, memory.shape[-1])
|
| 112 |
+
return memory.gather(dim=1, index=index)
|
| 113 |
+
|
| 114 |
+
def forward(
|
| 115 |
+
self,
|
| 116 |
+
encoder_hidden: torch.Tensor,
|
| 117 |
+
attention_mask: torch.Tensor,
|
| 118 |
+
action_positions: torch.Tensor,
|
| 119 |
+
slot_positions: torch.Tensor,
|
| 120 |
+
action_mask: torch.Tensor | None = None,
|
| 121 |
+
slot_mask: torch.Tensor | None = None,
|
| 122 |
+
) -> ReasoningOutput:
|
| 123 |
+
if (
|
| 124 |
+
encoder_hidden.ndim != 3
|
| 125 |
+
or encoder_hidden.shape[-1] != self.config.encoder_width
|
| 126 |
+
):
|
| 127 |
+
raise ValueError("encoder_hidden has the wrong shape")
|
| 128 |
+
if attention_mask.shape != encoder_hidden.shape[:2]:
|
| 129 |
+
raise ValueError("attention_mask has the wrong shape")
|
| 130 |
+
if action_positions.shape[1] < 1:
|
| 131 |
+
raise ValueError("at least one action is required")
|
| 132 |
+
if action_mask is None:
|
| 133 |
+
action_mask = torch.ones_like(action_positions, dtype=torch.bool)
|
| 134 |
+
if slot_mask is None:
|
| 135 |
+
slot_mask = torch.ones_like(slot_positions, dtype=torch.bool)
|
| 136 |
+
if (
|
| 137 |
+
action_mask.shape != action_positions.shape
|
| 138 |
+
or slot_mask.shape != slot_positions.shape
|
| 139 |
+
):
|
| 140 |
+
raise ValueError("marker masks must match marker positions")
|
| 141 |
+
memory_mask = attention_mask.bool()
|
| 142 |
+
if not torch.all(memory_mask.any(dim=1)):
|
| 143 |
+
raise ValueError("each input needs at least one unmasked token")
|
| 144 |
+
workspace_mask = torch.cat((action_mask.bool(), slot_mask.bool()), dim=1)
|
| 145 |
+
if not torch.all(workspace_mask.any(dim=1)):
|
| 146 |
+
raise ValueError("each input needs an unmasked marker")
|
| 147 |
+
|
| 148 |
+
memory = self.project(encoder_hidden)
|
| 149 |
+
low = torch.cat(
|
| 150 |
+
(
|
| 151 |
+
self._gather(memory, action_positions),
|
| 152 |
+
self._gather(memory, slot_positions),
|
| 153 |
+
),
|
| 154 |
+
dim=1,
|
| 155 |
+
)
|
| 156 |
+
low = low.masked_fill(~workspace_mask[:, :, None], 0)
|
| 157 |
+
high = self.high_init(memory[:, 0])
|
| 158 |
+
for _ in range(self.config.high_cycles):
|
| 159 |
+
for _ in range(self.config.low_steps):
|
| 160 |
+
low = self.low_update(low, high, memory, workspace_mask, memory_mask)
|
| 161 |
+
pooled = (low * workspace_mask[:, :, None]).sum(dim=1) / workspace_mask.sum(
|
| 162 |
+
dim=1,
|
| 163 |
+
keepdim=True,
|
| 164 |
+
).clamp_min(1)
|
| 165 |
+
high = self.high_update(pooled, high)
|
| 166 |
+
|
| 167 |
+
n_actions = action_positions.shape[1]
|
| 168 |
+
action_states = low[:, :n_actions] + high[:, None, :]
|
| 169 |
+
slot_states = low[:, n_actions:] + high[:, None, :]
|
| 170 |
+
action_logits = (
|
| 171 |
+
self.action_score(self.action_norm(action_states)).squeeze(-1).float()
|
| 172 |
+
)
|
| 173 |
+
action_logits = action_logits.masked_fill(~action_mask.bool(), -1e4)
|
| 174 |
+
slot_gate_logits = self.slot_gate(self.slot_norm(slot_states)).float()
|
| 175 |
+
slot_gate_logits = slot_gate_logits.masked_fill(
|
| 176 |
+
~slot_mask.bool()[:, :, None], -1e4
|
| 177 |
+
)
|
| 178 |
+
return ReasoningOutput(
|
| 179 |
+
action_logits, slot_gate_logits, slot_states, memory, high
|
| 180 |
+
)
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
class SmallValueDecoder(nn.Module):
|
| 184 |
+
"""Short token decoder with a learned choice between generating and copying."""
|
| 185 |
+
|
| 186 |
+
def __init__(self, vocab_size: int, width: int = 256) -> None:
|
| 187 |
+
super().__init__()
|
| 188 |
+
if vocab_size < 2 or width < 1:
|
| 189 |
+
raise ValueError("vocab_size and width must be positive")
|
| 190 |
+
self.vocab_size = vocab_size
|
| 191 |
+
self.width = width
|
| 192 |
+
self.embedding = nn.Embedding(vocab_size, width)
|
| 193 |
+
self.memory_key = nn.Linear(width, width, bias=False)
|
| 194 |
+
self.query = nn.Linear(width, width, bias=False)
|
| 195 |
+
self.initial = nn.Linear(width, width)
|
| 196 |
+
self.cell = nn.GRUCell(2 * width, width)
|
| 197 |
+
self.output_norm = nn.LayerNorm(width)
|
| 198 |
+
self.output_bias = nn.Parameter(torch.zeros(vocab_size))
|
| 199 |
+
self.generate_gate = nn.Linear(2 * width, 1)
|
| 200 |
+
|
| 201 |
+
def _prepare(
|
| 202 |
+
self,
|
| 203 |
+
slot_states: torch.Tensor,
|
| 204 |
+
memory: torch.Tensor,
|
| 205 |
+
source_ids: torch.Tensor,
|
| 206 |
+
source_mask: torch.Tensor,
|
| 207 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 208 |
+
if slot_states.ndim != 3 or memory.ndim != 3:
|
| 209 |
+
raise ValueError(
|
| 210 |
+
"slot_states and memory must have [batch, length, width] shape"
|
| 211 |
+
)
|
| 212 |
+
batch, slots, width = slot_states.shape
|
| 213 |
+
if width != self.width or memory.shape[0] != batch or memory.shape[2] != width:
|
| 214 |
+
raise ValueError("decoder hidden widths or batch sizes do not match")
|
| 215 |
+
if (
|
| 216 |
+
source_ids.shape != memory.shape[:2]
|
| 217 |
+
or source_mask.shape != source_ids.shape
|
| 218 |
+
):
|
| 219 |
+
raise ValueError("source IDs and mask must match memory")
|
| 220 |
+
if slots < 1 or not torch.all(source_mask.bool().any(dim=1)):
|
| 221 |
+
raise ValueError("at least one slot and one source token are required")
|
| 222 |
+
if torch.any(source_ids < 0) or torch.any(source_ids >= self.vocab_size):
|
| 223 |
+
raise ValueError("source token ID is outside decoder vocabulary")
|
| 224 |
+
expanded_memory = (
|
| 225 |
+
memory[:, None]
|
| 226 |
+
.expand(-1, slots, -1, -1)
|
| 227 |
+
.reshape(
|
| 228 |
+
batch * slots,
|
| 229 |
+
memory.shape[1],
|
| 230 |
+
width,
|
| 231 |
+
)
|
| 232 |
+
)
|
| 233 |
+
expanded_ids = (
|
| 234 |
+
source_ids[:, None].expand(-1, slots, -1).reshape(batch * slots, -1)
|
| 235 |
+
)
|
| 236 |
+
expanded_mask = (
|
| 237 |
+
source_mask.bool()[:, None].expand(-1, slots, -1).reshape(batch * slots, -1)
|
| 238 |
+
)
|
| 239 |
+
hidden = torch.tanh(self.initial(slot_states.reshape(batch * slots, width)))
|
| 240 |
+
keys = self.memory_key(expanded_memory)
|
| 241 |
+
return hidden, expanded_memory, keys, expanded_ids, expanded_mask
|
| 242 |
+
|
| 243 |
+
def _step(
|
| 244 |
+
self,
|
| 245 |
+
token: torch.Tensor,
|
| 246 |
+
hidden: torch.Tensor,
|
| 247 |
+
memory: torch.Tensor,
|
| 248 |
+
keys: torch.Tensor,
|
| 249 |
+
source_ids: torch.Tensor,
|
| 250 |
+
source_mask: torch.Tensor,
|
| 251 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 252 |
+
scores = torch.bmm(keys, self.query(hidden)[:, :, None]).squeeze(-1)
|
| 253 |
+
scores = scores.float().div(self.width**0.5).masked_fill(~source_mask, -1e4)
|
| 254 |
+
attention = torch.softmax(scores, dim=-1)
|
| 255 |
+
context = torch.bmm(attention.to(memory.dtype)[:, None, :], memory).squeeze(1)
|
| 256 |
+
token_embedding = self.embedding(token).to(context.dtype)
|
| 257 |
+
hidden = self.cell(torch.cat((token_embedding, context), dim=-1), hidden)
|
| 258 |
+
logits = F.linear(
|
| 259 |
+
self.output_norm(hidden), self.embedding.weight, self.output_bias
|
| 260 |
+
)
|
| 261 |
+
generated = torch.softmax(logits.float(), dim=-1)
|
| 262 |
+
copied = torch.zeros_like(generated).scatter_add_(1, source_ids, attention)
|
| 263 |
+
gate = torch.sigmoid(
|
| 264 |
+
self.generate_gate(torch.cat((hidden, context), dim=-1))
|
| 265 |
+
).float()
|
| 266 |
+
probabilities = gate * generated + (1 - gate) * copied
|
| 267 |
+
return probabilities, hidden
|
| 268 |
+
|
| 269 |
+
def generate(
|
| 270 |
+
self,
|
| 271 |
+
slot_states: torch.Tensor,
|
| 272 |
+
memory: torch.Tensor,
|
| 273 |
+
source_ids: torch.Tensor,
|
| 274 |
+
source_mask: torch.Tensor,
|
| 275 |
+
*,
|
| 276 |
+
bos_token_id: int,
|
| 277 |
+
max_new_tokens: int,
|
| 278 |
+
eos_token_id: int | None = None,
|
| 279 |
+
) -> torch.Tensor:
|
| 280 |
+
"""Greedy fixed-length decoding; fixed length makes latency comparable."""
|
| 281 |
+
if not 0 <= bos_token_id < self.vocab_size or max_new_tokens < 1:
|
| 282 |
+
raise ValueError("invalid BOS token or decoding length")
|
| 283 |
+
batch, slots, _ = slot_states.shape
|
| 284 |
+
hidden, expanded_memory, keys, expanded_ids, expanded_mask = self._prepare(
|
| 285 |
+
slot_states,
|
| 286 |
+
memory,
|
| 287 |
+
source_ids,
|
| 288 |
+
source_mask,
|
| 289 |
+
)
|
| 290 |
+
token = torch.full(
|
| 291 |
+
(batch * slots,), bos_token_id, device=source_ids.device, dtype=torch.long
|
| 292 |
+
)
|
| 293 |
+
result = []
|
| 294 |
+
finished = torch.zeros(
|
| 295 |
+
batch * slots, device=source_ids.device, dtype=torch.bool
|
| 296 |
+
)
|
| 297 |
+
for _ in range(max_new_tokens):
|
| 298 |
+
probabilities, hidden = self._step(
|
| 299 |
+
token,
|
| 300 |
+
hidden,
|
| 301 |
+
expanded_memory,
|
| 302 |
+
keys,
|
| 303 |
+
expanded_ids,
|
| 304 |
+
expanded_mask,
|
| 305 |
+
)
|
| 306 |
+
token = probabilities.argmax(dim=-1)
|
| 307 |
+
result.append(token)
|
| 308 |
+
if eos_token_id is not None:
|
| 309 |
+
finished |= token == eos_token_id
|
| 310 |
+
if torch.all(finished):
|
| 311 |
+
break
|
| 312 |
+
return torch.stack(result, dim=-1).reshape(batch, slots, len(result))
|
| 313 |
+
|
| 314 |
+
def teacher_forced_nll(
|
| 315 |
+
self,
|
| 316 |
+
slot_states: torch.Tensor,
|
| 317 |
+
memory: torch.Tensor,
|
| 318 |
+
source_ids: torch.Tensor,
|
| 319 |
+
source_mask: torch.Tensor,
|
| 320 |
+
target_ids: torch.Tensor,
|
| 321 |
+
*,
|
| 322 |
+
bos_token_id: int,
|
| 323 |
+
pad_token_id: int | None = None,
|
| 324 |
+
) -> torch.Tensor:
|
| 325 |
+
"""Mean token NLL for later supervised training; no training is run here."""
|
| 326 |
+
batch, slots, steps = target_ids.shape
|
| 327 |
+
if slot_states.shape[:2] != (batch, slots) or steps < 1:
|
| 328 |
+
raise ValueError("target IDs do not match slot states")
|
| 329 |
+
hidden, expanded_memory, keys, expanded_ids, expanded_mask = self._prepare(
|
| 330 |
+
slot_states,
|
| 331 |
+
memory,
|
| 332 |
+
source_ids,
|
| 333 |
+
source_mask,
|
| 334 |
+
)
|
| 335 |
+
target = target_ids.reshape(batch * slots, steps)
|
| 336 |
+
token = torch.full(
|
| 337 |
+
(batch * slots,), bos_token_id, device=target.device, dtype=torch.long
|
| 338 |
+
)
|
| 339 |
+
losses = []
|
| 340 |
+
for step in range(steps):
|
| 341 |
+
probabilities, hidden = self._step(
|
| 342 |
+
token,
|
| 343 |
+
hidden,
|
| 344 |
+
expanded_memory,
|
| 345 |
+
keys,
|
| 346 |
+
expanded_ids,
|
| 347 |
+
expanded_mask,
|
| 348 |
+
)
|
| 349 |
+
gold_probability = probabilities.gather(
|
| 350 |
+
1, target[:, step : step + 1]
|
| 351 |
+
).squeeze(1)
|
| 352 |
+
losses.append(-gold_probability.clamp_min(1e-12).log())
|
| 353 |
+
token = target[:, step]
|
| 354 |
+
losses = torch.stack(losses, dim=-1)
|
| 355 |
+
if pad_token_id is None:
|
| 356 |
+
return losses.mean()
|
| 357 |
+
valid = target != pad_token_id
|
| 358 |
+
return (losses * valid).sum() / valid.sum().clamp_min(1)
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
class StructuredAgent(nn.Module):
|
| 362 |
+
"""Encoder, recurrent workspace, typed heads, and optional value decoder."""
|
| 363 |
+
|
| 364 |
+
def __init__(
|
| 365 |
+
self, encoder: nn.Module, config: ReasoningConfig, vocab_size: int
|
| 366 |
+
) -> None:
|
| 367 |
+
super().__init__()
|
| 368 |
+
self.encoder = encoder
|
| 369 |
+
self.reasoner = ReasoningHead(config)
|
| 370 |
+
self.decoder = SmallValueDecoder(vocab_size, config.width)
|
| 371 |
+
|
| 372 |
+
def forward(
|
| 373 |
+
self,
|
| 374 |
+
input_ids: torch.Tensor,
|
| 375 |
+
attention_mask: torch.Tensor,
|
| 376 |
+
action_positions: torch.Tensor,
|
| 377 |
+
slot_positions: torch.Tensor,
|
| 378 |
+
*,
|
| 379 |
+
generated_slot_indices: torch.Tensor | None = None,
|
| 380 |
+
bos_token_id: int = 0,
|
| 381 |
+
max_new_tokens: int = 16,
|
| 382 |
+
) -> tuple[ReasoningOutput, torch.Tensor | None]:
|
| 383 |
+
hidden = self.encoder(input_ids=input_ids, attention_mask=attention_mask)
|
| 384 |
+
result = self.reasoner(hidden, attention_mask, action_positions, slot_positions)
|
| 385 |
+
if generated_slot_indices is None:
|
| 386 |
+
return result, None
|
| 387 |
+
if generated_slot_indices.ndim != 1 or generated_slot_indices.numel() < 1:
|
| 388 |
+
raise ValueError("generated_slot_indices must be a nonempty vector")
|
| 389 |
+
slots = result.slot_states.index_select(1, generated_slot_indices)
|
| 390 |
+
values = self.decoder.generate(
|
| 391 |
+
slots,
|
| 392 |
+
result.memory,
|
| 393 |
+
input_ids,
|
| 394 |
+
attention_mask,
|
| 395 |
+
bos_token_id=bos_token_id,
|
| 396 |
+
max_new_tokens=max_new_tokens,
|
| 397 |
+
)
|
| 398 |
+
return result, values
|
requirements.txt
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Install the PyTorch build appropriate for your CPU/CUDA environment.
|
| 2 |
+
torch
|
| 3 |
+
numpy
|
| 4 |
+
safetensors
|
| 5 |
+
tokenizers
|
| 6 |
+
tqdm
|
| 7 |
+
bitsandbytes==0.50.2
|
| 8 |
+
huggingface_hub
|
systemone.py
ADDED
|
@@ -0,0 +1,319 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""System 1 inference using the state/questions/criteria request format."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import math
|
| 5 |
+
from collections.abc import Mapping
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
from data import encode_input
|
| 11 |
+
from model import Bazza, BazzaDecisionHead, Gemma4
|
| 12 |
+
from model.gemma_backbone import matches_gemma_source
|
| 13 |
+
from utils import autocast, file_hash
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _checkpoint_settings(saved):
|
| 17 |
+
"""Validate the checkpoint identity and required architecture settings."""
|
| 18 |
+
from model import gemma
|
| 19 |
+
|
| 20 |
+
config = saved.get("config", {})
|
| 21 |
+
state = saved.get("head")
|
| 22 |
+
if not isinstance(state, dict) or "letter.weight" not in state:
|
| 23 |
+
raise ValueError("Expected a Bazza decision-head checkpoint")
|
| 24 |
+
for name, expected in [
|
| 25 |
+
("backbone_checkpoint", gemma.CHECKPOINT),
|
| 26 |
+
("backbone_revision", gemma.REVISION),
|
| 27 |
+
]:
|
| 28 |
+
if config.get(name) != expected:
|
| 29 |
+
raise ValueError("Checkpoint backbone identity differs: " + name)
|
| 30 |
+
required = [
|
| 31 |
+
"hidden_width",
|
| 32 |
+
"attention_heads",
|
| 33 |
+
"low_steps",
|
| 34 |
+
"high_cycles",
|
| 35 |
+
"gate_width",
|
| 36 |
+
"adapter_rank",
|
| 37 |
+
"quantization",
|
| 38 |
+
]
|
| 39 |
+
missing = [name for name in required if name not in config]
|
| 40 |
+
if missing:
|
| 41 |
+
raise ValueError(
|
| 42 |
+
"Checkpoint lacks architecture settings: " + ", ".join(missing)
|
| 43 |
+
)
|
| 44 |
+
if config["quantization"] == "bnb-nf4":
|
| 45 |
+
from importlib.util import find_spec
|
| 46 |
+
|
| 47 |
+
if find_spec("bitsandbytes") is None:
|
| 48 |
+
raise RuntimeError(
|
| 49 |
+
"NF4 inference requires bitsandbytes; install requirements.txt or build the Bazza Docker image"
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
return config, state
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _load_tokenizer(saved, weights):
|
| 56 |
+
"""Verify backbone source and tokenizer identity before loading the tokenizer."""
|
| 57 |
+
from tokenizers import Tokenizer
|
| 58 |
+
|
| 59 |
+
tokenizer_path = weights / "tokenizer.json"
|
| 60 |
+
expected_tokenizer = (
|
| 61 |
+
saved.get("identity", {}).get("readout_files", {}).get("tokenizer.json")
|
| 62 |
+
)
|
| 63 |
+
if (
|
| 64 |
+
expected_tokenizer is not None
|
| 65 |
+
and file_hash(tokenizer_path) != expected_tokenizer
|
| 66 |
+
):
|
| 67 |
+
raise ValueError("Tokenizer differs from the training checkpoint")
|
| 68 |
+
expected_source = (
|
| 69 |
+
saved.get("identity", {}).get("source_sha256", {}).get("model/gemma.py")
|
| 70 |
+
)
|
| 71 |
+
if expected_source is not None and not matches_gemma_source(expected_source):
|
| 72 |
+
raise ValueError("Bundled backbone differs from the training checkpoint")
|
| 73 |
+
return Tokenizer.from_file(str(tokenizer_path))
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _load_head(saved, config, state):
|
| 77 |
+
"""Construct the decision head and restore its trained parameters."""
|
| 78 |
+
softcap = saved.get("readout", {}).get("softcap", config.get("softcap"))
|
| 79 |
+
if softcap is None:
|
| 80 |
+
raise ValueError("Checkpoint lacks the readout softcap")
|
| 81 |
+
letter = state["letter.weight"]
|
| 82 |
+
if (
|
| 83 |
+
letter.ndim != 2
|
| 84 |
+
or letter.shape[0] != 26
|
| 85 |
+
or config.get("encoder_width") != letter.shape[1]
|
| 86 |
+
):
|
| 87 |
+
raise ValueError("Checkpoint encoder/readout dimensions differ")
|
| 88 |
+
head = BazzaDecisionHead(
|
| 89 |
+
letter,
|
| 90 |
+
softcap,
|
| 91 |
+
encoder_width=letter.shape[1],
|
| 92 |
+
width=config["hidden_width"],
|
| 93 |
+
attention_heads=config["attention_heads"],
|
| 94 |
+
low_steps=config["low_steps"],
|
| 95 |
+
high_cycles=config["high_cycles"],
|
| 96 |
+
gate_width=config["gate_width"],
|
| 97 |
+
rank=config["adapter_rank"],
|
| 98 |
+
)
|
| 99 |
+
head.load_state_dict(state, strict=True)
|
| 100 |
+
|
| 101 |
+
return head
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
class SystemOne:
|
| 105 |
+
"""Wrap a loaded Bazza model and tokenizer with a typed decision API."""
|
| 106 |
+
|
| 107 |
+
def __init__(self, model, tokenizer, *, max_length=4096):
|
| 108 |
+
if max_length < 1:
|
| 109 |
+
raise ValueError("max_length must be positive")
|
| 110 |
+
self.model = model.eval()
|
| 111 |
+
self.tokenizer = tokenizer
|
| 112 |
+
self.max_length = max_length
|
| 113 |
+
self.device = self.model.head.letter.weight.device
|
| 114 |
+
if self.device.type == "cpu" and self.model.head_precision == "bf16":
|
| 115 |
+
self.model.head.float()
|
| 116 |
+
self.model.head_precision = "fp32"
|
| 117 |
+
|
| 118 |
+
@classmethod
|
| 119 |
+
def from_checkpoint(
|
| 120 |
+
cls, checkpoint, *, backbone_path, device="cuda", max_length=4096
|
| 121 |
+
):
|
| 122 |
+
"""Load a trained decision checkpoint and the bundled live Gemma backbone.
|
| 123 |
+
|
| 124 |
+
Training data, feature caches and experiment configuration files are not
|
| 125 |
+
required. Architecture settings are read from the checkpoint itself.
|
| 126 |
+
"""
|
| 127 |
+
saved = torch.load(Path(checkpoint), map_location="cpu", weights_only=True)
|
| 128 |
+
return cls._from_saved(
|
| 129 |
+
saved, backbone_path=backbone_path, device=device, max_length=max_length
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
@classmethod
|
| 133 |
+
def from_pretrained(
|
| 134 |
+
cls,
|
| 135 |
+
model_id,
|
| 136 |
+
*,
|
| 137 |
+
revision=None,
|
| 138 |
+
backbone_path=None,
|
| 139 |
+
cache_dir=None,
|
| 140 |
+
token=None,
|
| 141 |
+
local_files_only=False,
|
| 142 |
+
device="cuda",
|
| 143 |
+
max_length=4096,
|
| 144 |
+
):
|
| 145 |
+
"""Load a head release and its pinned backbone from the Hugging Face Hub.
|
| 146 |
+
|
| 147 |
+
A local release directory and optional local backbone path also work.
|
| 148 |
+
Downloaded Python code is never executed; use this installed implementation.
|
| 149 |
+
"""
|
| 150 |
+
from huggingface_hub import snapshot_download
|
| 151 |
+
from safetensors.torch import load_file
|
| 152 |
+
|
| 153 |
+
download_options = dict(
|
| 154 |
+
cache_dir=cache_dir, token=token, local_files_only=local_files_only
|
| 155 |
+
)
|
| 156 |
+
directory = Path(model_id).expanduser()
|
| 157 |
+
if not directory.is_dir():
|
| 158 |
+
directory = Path(
|
| 159 |
+
snapshot_download(
|
| 160 |
+
repo_id=str(model_id),
|
| 161 |
+
revision=revision,
|
| 162 |
+
allow_patterns=["config.json", "model.safetensors"],
|
| 163 |
+
**download_options,
|
| 164 |
+
)
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
saved = json.loads((directory / "config.json").read_text())
|
| 168 |
+
if saved.get("model_type") != "bazza" or saved.get("format_version") != 1:
|
| 169 |
+
raise ValueError("Unsupported Bazza head release format")
|
| 170 |
+
weight_path = directory / "model.safetensors"
|
| 171 |
+
expected_hash = saved.get("weights", {}).get("sha256")
|
| 172 |
+
if (
|
| 173 |
+
not isinstance(expected_hash, str)
|
| 174 |
+
or file_hash(weight_path) != expected_hash
|
| 175 |
+
):
|
| 176 |
+
raise ValueError("Bazza head weight SHA-256 mismatch")
|
| 177 |
+
saved["head"] = load_file(str(weight_path), device="cpu")
|
| 178 |
+
config, _ = _checkpoint_settings(saved)
|
| 179 |
+
|
| 180 |
+
if backbone_path is None:
|
| 181 |
+
backbone_path = snapshot_download(
|
| 182 |
+
repo_id=config["backbone_checkpoint"],
|
| 183 |
+
revision=config["backbone_revision"],
|
| 184 |
+
allow_patterns=["config.json", "tokenizer.json", "model.safetensors"],
|
| 185 |
+
**download_options,
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
return cls._from_saved(
|
| 189 |
+
saved, backbone_path=backbone_path, device=device, max_length=max_length
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
@classmethod
|
| 193 |
+
def _from_saved(cls, saved, *, backbone_path, device, max_length):
|
| 194 |
+
"""Construct inference modules from validated checkpoint metadata."""
|
| 195 |
+
from model import gemma
|
| 196 |
+
|
| 197 |
+
config, state = _checkpoint_settings(saved)
|
| 198 |
+
weights = Path(backbone_path).expanduser().resolve()
|
| 199 |
+
tokenizer = _load_tokenizer(saved, weights)
|
| 200 |
+
|
| 201 |
+
identity = dict(
|
| 202 |
+
checkpoint=gemma.CHECKPOINT,
|
| 203 |
+
revision=gemma.REVISION,
|
| 204 |
+
weight_sha256=gemma.WEIGHT_SHA256,
|
| 205 |
+
quantization=config["quantization"],
|
| 206 |
+
implementation_sha256=file_hash(Path(gemma.__file__)),
|
| 207 |
+
)
|
| 208 |
+
backbone = Gemma4(
|
| 209 |
+
weights,
|
| 210 |
+
quantization=config["quantization"],
|
| 211 |
+
selection={"identity": identity},
|
| 212 |
+
)
|
| 213 |
+
head = _load_head(saved, config, state)
|
| 214 |
+
|
| 215 |
+
precision = (
|
| 216 |
+
config.get("precision", "bf16")
|
| 217 |
+
if torch.device(device).type == "cuda"
|
| 218 |
+
else "fp32"
|
| 219 |
+
)
|
| 220 |
+
if precision == "fp32":
|
| 221 |
+
head.float()
|
| 222 |
+
model = Bazza(backbone, head, head_precision=precision).to(device).eval()
|
| 223 |
+
backbone._load()
|
| 224 |
+
return cls(model, tokenizer, max_length=max_length)
|
| 225 |
+
|
| 226 |
+
def _prepare(self, state, questions):
|
| 227 |
+
# Validate the complete request before any model execution.
|
| 228 |
+
try:
|
| 229 |
+
json.dumps(state, allow_nan=False, sort_keys=True)
|
| 230 |
+
except (TypeError, ValueError) as error:
|
| 231 |
+
raise ValueError("state must be finite JSON-compatible data") from error
|
| 232 |
+
if not isinstance(questions, Mapping) or not questions:
|
| 233 |
+
raise ValueError("questions must be a nonempty mapping")
|
| 234 |
+
return [
|
| 235 |
+
self._prepare_question(state, name, question)
|
| 236 |
+
for name, question in questions.items()
|
| 237 |
+
]
|
| 238 |
+
|
| 239 |
+
def _prepare_question(self, state, name, question):
|
| 240 |
+
"""Validate one typed question and encode its candidate decisions."""
|
| 241 |
+
if not isinstance(name, str) or not name or not isinstance(question, Mapping):
|
| 242 |
+
raise ValueError("Each question requires a nonempty name and object")
|
| 243 |
+
kind = question.get("type")
|
| 244 |
+
if kind not in ("choice", "noul", "score"):
|
| 245 |
+
raise ValueError("Unsupported question type: " + str(kind))
|
| 246 |
+
instructions, criteria = (
|
| 247 |
+
question.get("instructions"),
|
| 248 |
+
question.get("criteria"),
|
| 249 |
+
)
|
| 250 |
+
if not isinstance(instructions, str) or not instructions.strip():
|
| 251 |
+
raise ValueError("Question instructions must be nonempty text")
|
| 252 |
+
if not isinstance(criteria, Mapping) or not 2 <= len(criteria) <= 26:
|
| 253 |
+
raise ValueError("criteria must contain between 2 and 26 candidates")
|
| 254 |
+
if any(
|
| 255 |
+
not isinstance(k, str) or not k or not isinstance(v, str) or not v.strip()
|
| 256 |
+
for k, v in criteria.items()
|
| 257 |
+
):
|
| 258 |
+
raise ValueError("Criterion keys and descriptions must be nonempty strings")
|
| 259 |
+
keys = list(criteria)
|
| 260 |
+
if kind == "noul" and set(keys) != {"true", "false"}:
|
| 261 |
+
raise ValueError("noul criteria must contain true and false")
|
| 262 |
+
if kind == "score":
|
| 263 |
+
try:
|
| 264 |
+
levels = [float(k) for k in keys]
|
| 265 |
+
except ValueError as error:
|
| 266 |
+
raise ValueError("score criteria require numeric keys") from error
|
| 267 |
+
if not all(math.isfinite(v) for v in levels) or len(set(levels)) != len(
|
| 268 |
+
levels
|
| 269 |
+
):
|
| 270 |
+
raise ValueError("score criteria require distinct finite numeric keys")
|
| 271 |
+
row = dict(
|
| 272 |
+
id=name,
|
| 273 |
+
group_id=name,
|
| 274 |
+
source="systemone",
|
| 275 |
+
kind=kind,
|
| 276 |
+
state=state,
|
| 277 |
+
question=instructions,
|
| 278 |
+
options=list(criteria.values()),
|
| 279 |
+
)
|
| 280 |
+
encoded = encode_input(row, self.tokenizer, self.max_length)
|
| 281 |
+
if encoded is None:
|
| 282 |
+
raise ValueError(
|
| 283 |
+
f"Question {name!r} exceeds max_length={self.max_length}; input was not truncated"
|
| 284 |
+
)
|
| 285 |
+
|
| 286 |
+
return name, kind, keys, encoded
|
| 287 |
+
|
| 288 |
+
@torch.inference_mode()
|
| 289 |
+
def decide(self, *, state, questions):
|
| 290 |
+
"""Return answers keyed by question name; candidate order is preserved.
|
| 291 |
+
|
| 292 |
+
Each question is scored as a separate request, matching training. Values
|
| 293 |
+
in probabilities are the model's softmax scores, without calibration.
|
| 294 |
+
"""
|
| 295 |
+
prepared = self._prepare(state, questions)
|
| 296 |
+
answers = {}
|
| 297 |
+
for name, kind, keys, row in prepared:
|
| 298 |
+
batch = self.model.backbone.prepare_batch([row], "inference").to(
|
| 299 |
+
self.device
|
| 300 |
+
)
|
| 301 |
+
with autocast(self.device, self.model.head_precision):
|
| 302 |
+
logits = self.model(batch).logits[0, : len(keys)].float()
|
| 303 |
+
if not torch.isfinite(logits).all():
|
| 304 |
+
raise FloatingPointError("Non-finite decision logits")
|
| 305 |
+
probabilities = dict(zip(keys, logits.softmax(-1).cpu().tolist()))
|
| 306 |
+
chosen = keys[int(logits.argmax())]
|
| 307 |
+
answer = dict(
|
| 308 |
+
type=kind, choice=chosen, value=chosen, probabilities=probabilities
|
| 309 |
+
)
|
| 310 |
+
if kind == "noul":
|
| 311 |
+
answer["noul"] = probabilities["true"]
|
| 312 |
+
answer["value"] = probabilities["true"] > 0.5
|
| 313 |
+
elif kind == "score":
|
| 314 |
+
answer["score"] = sum(float(k) * v for k, v in probabilities.items())
|
| 315 |
+
answer["value"] = answer["score"]
|
| 316 |
+
answers[name] = answer
|
| 317 |
+
return {"answers": answers}
|
| 318 |
+
|
| 319 |
+
__call__ = decide
|
utils.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Small shared IO, identity and precision helpers."""
|
| 2 |
+
|
| 3 |
+
import hashlib
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
from contextlib import contextmanager, nullcontext
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
|
| 11 |
+
ROOT = Path(__file__).resolve().parent
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def digest(value):
|
| 15 |
+
return hashlib.sha256(
|
| 16 |
+
json.dumps(
|
| 17 |
+
value,
|
| 18 |
+
ensure_ascii=False,
|
| 19 |
+
sort_keys=True,
|
| 20 |
+
separators=(",", ":"),
|
| 21 |
+
allow_nan=False,
|
| 22 |
+
).encode()
|
| 23 |
+
).hexdigest()
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def file_hash(path):
|
| 27 |
+
result = hashlib.sha256()
|
| 28 |
+
with Path(path).open("rb") as stream:
|
| 29 |
+
for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
|
| 30 |
+
result.update(chunk)
|
| 31 |
+
return result.hexdigest()
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def atomic_json(path, value):
|
| 35 |
+
path = Path(path)
|
| 36 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 37 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 38 |
+
with temporary.open("w") as stream:
|
| 39 |
+
json.dump(value, stream, indent=2, allow_nan=False)
|
| 40 |
+
stream.write("\n")
|
| 41 |
+
stream.flush()
|
| 42 |
+
os.fsync(stream.fileno())
|
| 43 |
+
temporary.replace(path)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def atomic_save(value, path):
|
| 47 |
+
path = Path(path)
|
| 48 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 49 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 50 |
+
with temporary.open("wb") as stream:
|
| 51 |
+
torch.save(value, stream)
|
| 52 |
+
stream.flush()
|
| 53 |
+
os.fsync(stream.fileno())
|
| 54 |
+
temporary.replace(path)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def autocast(device, precision="bf16"):
|
| 58 |
+
return (
|
| 59 |
+
torch.autocast("cuda", dtype=torch.bfloat16)
|
| 60 |
+
if torch.device(device).type == "cuda" and precision == "bf16"
|
| 61 |
+
else nullcontext()
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
@contextmanager
|
| 66 |
+
def job_lock(directory):
|
| 67 |
+
import fcntl
|
| 68 |
+
|
| 69 |
+
with (Path(directory) / ".job.lock").open("a") as stream:
|
| 70 |
+
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
| 71 |
+
yield
|