Upload clef-flash-FP8-Dynamic (vLLM plugin + SystemOne server)
Browse files- .gitattributes +1 -0
- LICENSE +202 -0
- README.md +195 -0
- chat_template.jinja +154 -0
- clef_head/joint_head.safetensors +3 -0
- clef_head/joint_head_config.json +8 -0
- config.json +317 -0
- generation_config.json +6 -0
- joint_schema_model.py +576 -0
- model.safetensors +3 -0
- processor_config.json +60 -0
- recipe.yaml +8 -0
- scripts/build_data.py +99 -0
- scripts/quantize.py +70 -0
- tokenizer.json +3 -0
- tokenizer_config.json +30 -0
- vllm_plugin/clef_vllm/__init__.py +12 -0
- vllm_plugin/clef_vllm/model.py +153 -0
- vllm_plugin/clef_vllm/schema.py +9 -0
- vllm_plugin/clef_vllm/server.py +162 -0
- vllm_plugin/pyproject.toml +20 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright 2026 Alibaba Cloud
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
README.md
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
base_model: Cloudflare/clef-flash
|
| 4 |
+
base_model_relation: quantized
|
| 5 |
+
library_name: vllm
|
| 6 |
+
tags:
|
| 7 |
+
- clef
|
| 8 |
+
- systemone
|
| 9 |
+
- jev
|
| 10 |
+
- fp8
|
| 11 |
+
- compressed-tensors
|
| 12 |
+
- llm-compressor
|
| 13 |
+
- vllm
|
| 14 |
+
- qwen3.5
|
| 15 |
+
- structured-output
|
| 16 |
+
- classification
|
| 17 |
+
---
|
| 18 |
+
|
| 19 |
+
# clef-flash-FP8-Dynamic
|
| 20 |
+
|
| 21 |
+
**Unofficial** FP8_DYNAMIC quantization of [Cloudflare/clef-flash](https://huggingface.co/Cloudflare/clef-flash),
|
| 22 |
+
a 9B decision model that scores every option of every typed question (choice / score / true-false)
|
| 23 |
+
in one forward pass. This repo is not affiliated with or endorsed by Cloudflare.
|
| 24 |
+
|
| 25 |
+
It ships with a small **vLLM plugin** and a **Jev/SystemOne-compatible HTTP server** (`POST /v1/systemone`),
|
| 26 |
+
because stock vLLM cannot run Clef's custom joint-schema head. In short: near-lossless (98.8% top-1 agreement with BF16) and 1.5x BF16's two-GPU throughput on a single GPU.
|
| 27 |
+
|
| 28 |
+
> [!IMPORTANT]
|
| 29 |
+
> **Read before using**
|
| 30 |
+
> - **Text and JSON state only.** The vision tower is kept in BF16 in the checkpoint but has **not been tested**;
|
| 31 |
+
> the server rejects `images` / `videos` with HTTP 400 and runs vLLM with the vision tower disabled.
|
| 32 |
+
> - **Use the bundled vLLM plugin.** The quantized checkpoint does not run efficiently in plain `transformers`
|
| 33 |
+
> (it can only be dequantized to BF16, which removes the memory and speed benefit). The plugin is pinned to
|
| 34 |
+
> **vLLM 0.28.x**; it uses vLLM pooling internals that may change in later versions.
|
| 35 |
+
> - **Hardware:** **NVIDIA Ada, Hopper or Blackwell** (SM 8.9+: L4, L40S, RTX 40/50-series, H100/H200, B200).
|
| 36 |
+
> - **Tested on one setup only:** 2x RTX 5070 Ti 16 GB (SM 12.0, PCIe, no NVLink), vLLM 0.28.0, CUDA 13.
|
| 37 |
+
> - **Evaluated on 1,400 records from 7 public tasks** with our own harness (below), not on Cloudflare's Decision Index.
|
| 38 |
+
> Expect a similar but not identical picture on your data; check your own workload before relying on it.
|
| 39 |
+
|
| 40 |
+
## Quickstart: SystemOne API server
|
| 41 |
+
|
| 42 |
+
```bash
|
| 43 |
+
hf download kurcontko/clef-flash-FP8-Dynamic --local-dir clef-flash-FP8-Dynamic
|
| 44 |
+
|
| 45 |
+
docker run --gpus all --ipc=host -p 8000:8000 -v $PWD/clef-flash-FP8-Dynamic:/model:ro \
|
| 46 |
+
--entrypoint bash vllm/vllm-openai:v0.28.0 -c \
|
| 47 |
+
"cp -r /model/vllm_plugin /tmp/p && pip install -q /tmp/p && clef-systemone --model /model --dp 1"
|
| 48 |
+
```
|
| 49 |
+
|
| 50 |
+
`--dp N` runs one engine per GPU behind the one port (recommended over tensor parallelism on PCIe GPUs).
|
| 51 |
+
The server warms up its kernels before `/health` reports ready (about 3-4 minutes from a cold container).
|
| 52 |
+
|
| 53 |
+
```bash
|
| 54 |
+
curl localhost:8000/v1/systemone -H 'content-type: application/json' -d '{
|
| 55 |
+
"model": "clef-flash",
|
| 56 |
+
"state": "Our checkout started returning errors and orders are blocked.",
|
| 57 |
+
"questions": {
|
| 58 |
+
"department": {"type": "choice", "instructions": "Which team should handle the message?",
|
| 59 |
+
"criteria": {"billing": "Payments or invoices", "technical": "Bugs or outages"}},
|
| 60 |
+
"urgency": {"type": "score", "criteria": ["Can wait", "This week", "Today"]},
|
| 61 |
+
"outage": {"type": "noul", "instructions": "Is a service down?"}}}'
|
| 62 |
+
```
|
| 63 |
+
|
| 64 |
+
The request and response bodies are the same as the `systemone()` function in the original release:
|
| 65 |
+
`answers` keyed by question ID (`choice` with `confidence` and `probabilities`, `score` with the expected `score`
|
| 66 |
+
and `legend`, `noul` with the probability of true) and `usage`. Also served: `GET /v1/models`, `GET /health`.
|
| 67 |
+
|
| 68 |
+
## Quickstart: Python (offline vLLM)
|
| 69 |
+
|
| 70 |
+
```python
|
| 71 |
+
# pip install ./clef-flash-FP8-Dynamic/vllm_plugin (in an environment with vllm==0.28.*)
|
| 72 |
+
import sys
|
| 73 |
+
from transformers import AutoTokenizer
|
| 74 |
+
from vllm import LLM, PoolingParams
|
| 75 |
+
from clef_vllm import ARCHITECTURE, TASK, schema_layout
|
| 76 |
+
|
| 77 |
+
path = "./clef-flash-FP8-Dynamic"
|
| 78 |
+
sys.path.insert(0, path)
|
| 79 |
+
from joint_schema_model import encode_record
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def main():
|
| 83 |
+
# Keep `llm` local: vLLM shuts its engine process down when the LLM object is freed.
|
| 84 |
+
llm = LLM(model=path, hf_overrides={"architectures": [ARCHITECTURE]}, runner="pooling",
|
| 85 |
+
enable_prefix_caching=False, language_model_only=True, max_model_len=16384)
|
| 86 |
+
tokenizer = AutoTokenizer.from_pretrained(path)
|
| 87 |
+
record = {
|
| 88 |
+
"state": {"invoice": {"vendor": "Acme", "total": 1250.0, "currency": "USD", "status": "overdue"}},
|
| 89 |
+
"questions": {
|
| 90 |
+
"status": {"type": "choice", "instructions": "What is the invoice status?",
|
| 91 |
+
"criteria": {"paid": "Invoice is paid.", "overdue": "Invoice is past due.", "draft": "Not sent."}},
|
| 92 |
+
"large": {"type": "noul", "instructions": "Is the total above 1000 USD?"},
|
| 93 |
+
},
|
| 94 |
+
}
|
| 95 |
+
encoded = encode_record(tokenizer, record)
|
| 96 |
+
params = PoolingParams(task=TASK, extra_kwargs={"clef_questions": schema_layout(encoded)})
|
| 97 |
+
output = llm.encode([{"prompt_token_ids": list(encoded.input_ids)}], pooling_params=[params], pooling_task=TASK)[0]
|
| 98 |
+
|
| 99 |
+
logits, offset = output.outputs.data.float(), 0 # every option logit of every question, in order
|
| 100 |
+
for question in encoded.questions:
|
| 101 |
+
n = len(question.option_ids)
|
| 102 |
+
print(question.question_id, dict(zip(question.option_ids, logits[offset:offset + n].softmax(-1).tolist())))
|
| 103 |
+
offset += n
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
if __name__ == "__main__":
|
| 107 |
+
main()
|
| 108 |
+
```
|
| 109 |
+
|
| 110 |
+
Pass a list of prompts and a matching list of `PoolingParams` to score many records in one call; vLLM batches them.
|
| 111 |
+
|
| 112 |
+
## Results
|
| 113 |
+
|
| 114 |
+
### Accuracy and agreement with BF16
|
| 115 |
+
|
| 116 |
+
1,400 records, 200 per task, from held-out splits: BANKING77 (test), ARC-Challenge (test), MMLU (test),
|
| 117 |
+
HellaSwag (validation), ANLI R3 (test), BoolQ (validation), and a multi-question ANLI R1 set (3 questions per
|
| 118 |
+
record: choice + true/false + score). **Accuracy** is against gold labels. **Agree** is top-1 agreement with the
|
| 119 |
+
BF16 model and **KL** is mean KL(BF16 ‖ quantized) per question. BF16 is the original release run with
|
| 120 |
+
`transformers` 5.10.2; the quantized rows were measured in vLLM with real FP8/FP4 kernels.
|
| 121 |
+
|
| 122 |
+
| Task | BF16 acc | FP8 acc | FP8 agree | FP8 KL | NVFP4 acc | NVFP4 agree | NVFP4 KL |
|
| 123 |
+
|---|---:|---:|---:|---:|---:|---:|---:|
|
| 124 |
+
| BANKING77 (77-way intent) | 96.5 | 97.0 | 99.5 | 0.0005 | 97.5 | 98.0 | 0.013 |
|
| 125 |
+
| ARC-Challenge | 99.5 | 99.5 | 100.0 | 0.0006 | 99.0 | 99.5 | 0.009 |
|
| 126 |
+
| MMLU | 94.5 | 95.0 | 99.5 | 0.0035 | 93.5 | 95.5 | 0.026 |
|
| 127 |
+
| HellaSwag | 99.0 | 99.0 | 100.0 | 0.0003 | 99.0 | 100.0 | 0.002 |
|
| 128 |
+
| ANLI R3 | 50.0 | 49.5 | 97.5 | 0.0028 | 49.0 | 86.5 | 0.041 |
|
| 129 |
+
| BoolQ | 87.5 | 88.5 | 99.0 | 0.0026 | 87.5 | 98.0 | 0.018 |
|
| 130 |
+
| Multi-question ANLI R1 | 73.0 | 74.0 | 98.0 | 0.0023 | 71.5 | 90.3 | 0.034 |
|
| 131 |
+
| **All** | **84.1** | **84.6** | **98.8** | **0.0019** | **83.6** | **94.3** | **0.024** |
|
| 132 |
+
|
| 133 |
+
Accuracy differences of under 1 point are within noise at 200 records per task. The BF16 model run through the same
|
| 134 |
+
plugin in vLLM agrees with `transformers` on 99.8% of questions (KL 0.0001), so the plugin itself adds no error.
|
| 135 |
+
NVFP4 moves most on hard, near-chance cases (ANLI): treat its confidences on borderline inputs with care.
|
| 136 |
+
|
| 137 |
+
### Throughput
|
| 138 |
+
|
| 139 |
+
Same 1,400 records (717k tokens, ~512 per record), RTX 5070 Ti 16 GB GPUs.
|
| 140 |
+
|
| 141 |
+
| Setup | GPUs | Records/s | Tokens/s | vs BF16 transformers |
|
| 142 |
+
|---|---:|---:|---:|---:|
|
| 143 |
+
| BF16, `transformers` 5.10.2 (original release code, model split over 2 GPUs) | 2 | 6.1 | 3,113 | 1.0x |
|
| 144 |
+
| BF16, vLLM + plugin, tensor parallel 2 | 2 | 13.0 | 6,633 | 2.1x |
|
| 145 |
+
| **FP8**, vLLM + plugin | 1 | 19.5 | 9,986 | 3.2x |
|
| 146 |
+
| **NVFP4**, vLLM + plugin | 1 | 39.8 | 20,359 | 6.5x |
|
| 147 |
+
| FP8, one engine per GPU (`--dp 2`) | 2 | ~39 | ~20,000 | 6.4x |
|
| 148 |
+
| NVFP4, one engine per GPU (`--dp 2`) | 2 | ~81 | ~41,000 | 13x |
|
| 149 |
+
|
| 150 |
+
Through the HTTP server with 64 concurrent clients: NVFP4 on 1 GPU 39.7 req/s (p50 0.95 s, p95 5.1 s),
|
| 151 |
+
NVFP4 on 2 GPUs 78.7 req/s; with 8 concurrent clients on 2 GPUs, p50 latency is 132 ms. On PCIe GPUs, tensor
|
| 152 |
+
parallelism was slower than one engine per GPU (FP8 TP2: 8,605 tok/s; NVFP4 TP2: 10,850 tok/s).
|
| 153 |
+
|
| 154 |
+
## Quantization details
|
| 155 |
+
|
| 156 |
+
- **Tool:** llm-compressor 0.14.0 (compressed-tensors 0.19.0), `QuantizationModifier(targets="Linear", scheme="FP8_DYNAMIC")`.
|
| 157 |
+
- **Scheme:** FP8 (E4M3) W8A8: per-output-channel static weight scales, per-token dynamic activation scales.
|
| 158 |
+
- **Calibration:** None needed (weight scales from the weights, activation scales computed per token at runtime).
|
| 159 |
+
- **Kept in BF16:** `lm_head` (the joint head reads its rows as lexical option embeddings, so it is never quantized),
|
| 160 |
+
the vision tower (`model.visual.*`), the Gated DeltaNet gate projections `linear_attn.in_proj_a` / `in_proj_b`
|
| 161 |
+
(32 outputs each), embeddings, norms, and the joint schema head (`clef_head/`).
|
| 162 |
+
- **Gated DeltaNet fused scale:** vLLM fuses `linear_attn.in_proj_qkv` and `in_proj_z` into one GEMM, which needs a
|
| 163 |
+
shared NVFP4 weight global scale. llm-compressor 0.14 only shares scales for q/k/v and gate/up, so
|
| 164 |
+
`scripts/quantize.py` adds `("in_proj_qkv", "in_proj_z")` to its fused groups. Without this, vLLM applies the
|
| 165 |
+
larger scale to both halves and the qkv weights come out scaled by about 0.67 (we measured 92.9% instead of 94.3%
|
| 166 |
+
agreement). Anyone re-quantizing Qwen3.5-family models to NVFP4 for vLLM should do the same.
|
| 167 |
+
- `recipe.yaml` is the exact recipe llm-compressor saved.
|
| 168 |
+
|
| 169 |
+
## How the vLLM plugin works
|
| 170 |
+
|
| 171 |
+
`vllm_plugin/` (`pip install`-able, package `clef-vllm`) registers `ClefFlashForDecision`, a subclass of vLLM's
|
| 172 |
+
`Qwen3_5ForConditionalGeneration` that runs as a pooling model. Its pooler collects each request's final hidden
|
| 173 |
+
states across chunked prefill and runs the original `JointSchemaHead` (from `joint_schema_model.py`) on the full
|
| 174 |
+
prompt. Each request carries its schema layout (question and option token spans from `encode_record`) in
|
| 175 |
+
`PoolingParams.extra_kwargs["clef_questions"]`; the output is one flat vector of every option logit. Prefix caching
|
| 176 |
+
must be off. Under tensor parallelism the vocab-sharded `lm_head` is gathered once on rank 0 for the head's lookups.
|
| 177 |
+
|
| 178 |
+
## Files
|
| 179 |
+
|
| 180 |
+
| Path | Purpose |
|
| 181 |
+
|---|---|
|
| 182 |
+
| `model.safetensors`, `config.json`, `recipe.yaml` | Quantized backbone (12 GB, compressed-tensors format) |
|
| 183 |
+
| `clef_head/` | Joint schema head, BF16, unchanged from the original release (in a subfolder so vLLM does not load it as backbone weights) |
|
| 184 |
+
| `joint_schema_model.py` | Original release code: `encode_record`, `systemone_answer`, the head module (unchanged) |
|
| 185 |
+
| `tokenizer*.json`, `chat_template.jinja`, `processor_config.json`, `generation_config.json` | Tokenizer and processor (unchanged) |
|
| 186 |
+
| `vllm_plugin/` | vLLM plugin and `clef-systemone` server |
|
| 187 |
+
| `scripts/` | `build_data.py` (calibration/eval sets) and `quantize.py` (how this checkpoint was made) |
|
| 188 |
+
|
| 189 |
+
See also [kurcontko/clef-flash-NVFP4](https://huggingface.co/kurcontko/clef-flash-NVFP4) (2x faster again, about -0.5 points).
|
| 190 |
+
|
| 191 |
+
## License and attribution
|
| 192 |
+
|
| 193 |
+
Apache-2.0, following [Cloudflare/clef-flash](https://huggingface.co/Cloudflare/clef-flash) (itself post-trained from
|
| 194 |
+
[Qwen/Qwen3.5-9B](https://huggingface.co/Qwen/Qwen3.5-9B)). All credit for the model goes to its original authors; this
|
| 195 |
+
repository only adds the quantization, the vLLM plugin and the server. Not an official Cloudflare release.
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- set image_count = namespace(value=0) %}
|
| 2 |
+
{%- set video_count = namespace(value=0) %}
|
| 3 |
+
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
|
| 4 |
+
{%- if content is string %}
|
| 5 |
+
{{- content }}
|
| 6 |
+
{%- elif content is iterable and content is not mapping %}
|
| 7 |
+
{%- for item in content %}
|
| 8 |
+
{%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
|
| 9 |
+
{%- if is_system_content %}
|
| 10 |
+
{{- raise_exception('System message cannot contain images.') }}
|
| 11 |
+
{%- endif %}
|
| 12 |
+
{%- if do_vision_count %}
|
| 13 |
+
{%- set image_count.value = image_count.value + 1 %}
|
| 14 |
+
{%- endif %}
|
| 15 |
+
{%- if add_vision_id %}
|
| 16 |
+
{{- 'Picture ' ~ image_count.value ~ ': ' }}
|
| 17 |
+
{%- endif %}
|
| 18 |
+
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
|
| 19 |
+
{%- elif 'video' in item or item.type == 'video' %}
|
| 20 |
+
{%- if is_system_content %}
|
| 21 |
+
{{- raise_exception('System message cannot contain videos.') }}
|
| 22 |
+
{%- endif %}
|
| 23 |
+
{%- if do_vision_count %}
|
| 24 |
+
{%- set video_count.value = video_count.value + 1 %}
|
| 25 |
+
{%- endif %}
|
| 26 |
+
{%- if add_vision_id %}
|
| 27 |
+
{{- 'Video ' ~ video_count.value ~ ': ' }}
|
| 28 |
+
{%- endif %}
|
| 29 |
+
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
|
| 30 |
+
{%- elif 'text' in item %}
|
| 31 |
+
{{- item.text }}
|
| 32 |
+
{%- else %}
|
| 33 |
+
{{- raise_exception('Unexpected item type in content.') }}
|
| 34 |
+
{%- endif %}
|
| 35 |
+
{%- endfor %}
|
| 36 |
+
{%- elif content is none or content is undefined %}
|
| 37 |
+
{{- '' }}
|
| 38 |
+
{%- else %}
|
| 39 |
+
{{- raise_exception('Unexpected content type.') }}
|
| 40 |
+
{%- endif %}
|
| 41 |
+
{%- endmacro %}
|
| 42 |
+
{%- if not messages %}
|
| 43 |
+
{{- raise_exception('No messages provided.') }}
|
| 44 |
+
{%- endif %}
|
| 45 |
+
{%- if tools and tools is iterable and tools is not mapping %}
|
| 46 |
+
{{- '<|im_start|>system\n' }}
|
| 47 |
+
{{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
|
| 48 |
+
{%- for tool in tools %}
|
| 49 |
+
{{- "\n" }}
|
| 50 |
+
{{- tool | tojson }}
|
| 51 |
+
{%- endfor %}
|
| 52 |
+
{{- "\n</tools>" }}
|
| 53 |
+
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
| 54 |
+
{%- if messages[0].role == 'system' %}
|
| 55 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 56 |
+
{%- if content %}
|
| 57 |
+
{{- '\n\n' + content }}
|
| 58 |
+
{%- endif %}
|
| 59 |
+
{%- endif %}
|
| 60 |
+
{{- '<|im_end|>\n' }}
|
| 61 |
+
{%- else %}
|
| 62 |
+
{%- if messages[0].role == 'system' %}
|
| 63 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 64 |
+
{{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
|
| 65 |
+
{%- endif %}
|
| 66 |
+
{%- endif %}
|
| 67 |
+
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
| 68 |
+
{%- for message in messages[::-1] %}
|
| 69 |
+
{%- set index = (messages|length - 1) - loop.index0 %}
|
| 70 |
+
{%- if ns.multi_step_tool and message.role == "user" %}
|
| 71 |
+
{%- set content = render_content(message.content, false)|trim %}
|
| 72 |
+
{%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
|
| 73 |
+
{%- set ns.multi_step_tool = false %}
|
| 74 |
+
{%- set ns.last_query_index = index %}
|
| 75 |
+
{%- endif %}
|
| 76 |
+
{%- endif %}
|
| 77 |
+
{%- endfor %}
|
| 78 |
+
{%- if ns.multi_step_tool %}
|
| 79 |
+
{{- raise_exception('No user query found in messages.') }}
|
| 80 |
+
{%- endif %}
|
| 81 |
+
{%- for message in messages %}
|
| 82 |
+
{%- set content = render_content(message.content, true)|trim %}
|
| 83 |
+
{%- if message.role == "system" %}
|
| 84 |
+
{%- if not loop.first %}
|
| 85 |
+
{{- raise_exception('System message must be at the beginning.') }}
|
| 86 |
+
{%- endif %}
|
| 87 |
+
{%- elif message.role == "user" %}
|
| 88 |
+
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
| 89 |
+
{%- elif message.role == "assistant" %}
|
| 90 |
+
{%- set reasoning_content = '' %}
|
| 91 |
+
{%- if message.reasoning_content is string %}
|
| 92 |
+
{%- set reasoning_content = message.reasoning_content %}
|
| 93 |
+
{%- else %}
|
| 94 |
+
{%- if '</think>' in content %}
|
| 95 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 96 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 97 |
+
{%- endif %}
|
| 98 |
+
{%- endif %}
|
| 99 |
+
{%- set reasoning_content = reasoning_content|trim %}
|
| 100 |
+
{%- if loop.index0 > ns.last_query_index %}
|
| 101 |
+
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
|
| 102 |
+
{%- else %}
|
| 103 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 104 |
+
{%- endif %}
|
| 105 |
+
{%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
|
| 106 |
+
{%- for tool_call in message.tool_calls %}
|
| 107 |
+
{%- if tool_call.function is defined %}
|
| 108 |
+
{%- set tool_call = tool_call.function %}
|
| 109 |
+
{%- endif %}
|
| 110 |
+
{%- if loop.first %}
|
| 111 |
+
{%- if content|trim %}
|
| 112 |
+
{{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 113 |
+
{%- else %}
|
| 114 |
+
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 115 |
+
{%- endif %}
|
| 116 |
+
{%- else %}
|
| 117 |
+
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 118 |
+
{%- endif %}
|
| 119 |
+
{%- if tool_call.arguments is defined %}
|
| 120 |
+
{%- for args_name, args_value in tool_call.arguments|items %}
|
| 121 |
+
{{- '<parameter=' + args_name + '>\n' }}
|
| 122 |
+
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
| 123 |
+
{{- args_value }}
|
| 124 |
+
{{- '\n</parameter>\n' }}
|
| 125 |
+
{%- endfor %}
|
| 126 |
+
{%- endif %}
|
| 127 |
+
{{- '</function>\n</tool_call>' }}
|
| 128 |
+
{%- endfor %}
|
| 129 |
+
{%- endif %}
|
| 130 |
+
{{- '<|im_end|>\n' }}
|
| 131 |
+
{%- elif message.role == "tool" %}
|
| 132 |
+
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
| 133 |
+
{{- '<|im_start|>user' }}
|
| 134 |
+
{%- endif %}
|
| 135 |
+
{{- '\n<tool_response>\n' }}
|
| 136 |
+
{{- content }}
|
| 137 |
+
{{- '\n</tool_response>' }}
|
| 138 |
+
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
| 139 |
+
{{- '<|im_end|>\n' }}
|
| 140 |
+
{%- elif loop.last %}
|
| 141 |
+
{{- '<|im_end|>\n' }}
|
| 142 |
+
{%- endif %}
|
| 143 |
+
{%- else %}
|
| 144 |
+
{{- raise_exception('Unexpected message role.') }}
|
| 145 |
+
{%- endif %}
|
| 146 |
+
{%- endfor %}
|
| 147 |
+
{%- if add_generation_prompt %}
|
| 148 |
+
{{- '<|im_start|>assistant\n' }}
|
| 149 |
+
{%- if enable_thinking is defined and enable_thinking is false %}
|
| 150 |
+
{{- '<think>\n\n</think>\n\n' }}
|
| 151 |
+
{%- else %}
|
| 152 |
+
{{- '<think>\n' }}
|
| 153 |
+
{%- endif %}
|
| 154 |
+
{%- endif %}
|
clef_head/joint_head.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:19cdcec8c81dc9212be320fff47462ab342fbc1278be4368fb3da71241cf5ba0
|
| 3 |
+
size 243538016
|
clef_head/joint_head_config.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"hidden_size": 4096,
|
| 3 |
+
"width": 1024,
|
| 4 |
+
"routing_layers": 2,
|
| 5 |
+
"layers": 4,
|
| 6 |
+
"heads": 16,
|
| 7 |
+
"feedforward": 4096
|
| 8 |
+
}
|
config.json
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"Qwen3_5ForConditionalGeneration"
|
| 4 |
+
],
|
| 5 |
+
"dtype": "bfloat16",
|
| 6 |
+
"image_token_id": 248056,
|
| 7 |
+
"model_type": "qwen3_5",
|
| 8 |
+
"quantization_config": {
|
| 9 |
+
"config_groups": {
|
| 10 |
+
"group_0": {
|
| 11 |
+
"format": "float-quantized",
|
| 12 |
+
"input_activations": {
|
| 13 |
+
"actorder": null,
|
| 14 |
+
"block_structure": null,
|
| 15 |
+
"dynamic": true,
|
| 16 |
+
"group_size": null,
|
| 17 |
+
"num_bits": 8,
|
| 18 |
+
"observer": null,
|
| 19 |
+
"observer_kwargs": {},
|
| 20 |
+
"scale_dtype": null,
|
| 21 |
+
"strategy": "token",
|
| 22 |
+
"symmetric": true,
|
| 23 |
+
"type": "float",
|
| 24 |
+
"zp_dtype": null
|
| 25 |
+
},
|
| 26 |
+
"output_activations": null,
|
| 27 |
+
"targets": [
|
| 28 |
+
"Linear"
|
| 29 |
+
],
|
| 30 |
+
"weights": {
|
| 31 |
+
"actorder": null,
|
| 32 |
+
"block_structure": null,
|
| 33 |
+
"dynamic": false,
|
| 34 |
+
"group_size": null,
|
| 35 |
+
"num_bits": 8,
|
| 36 |
+
"observer": "memoryless_minmax",
|
| 37 |
+
"observer_kwargs": {},
|
| 38 |
+
"scale_dtype": null,
|
| 39 |
+
"strategy": "channel",
|
| 40 |
+
"symmetric": true,
|
| 41 |
+
"type": "float",
|
| 42 |
+
"zp_dtype": null
|
| 43 |
+
}
|
| 44 |
+
}
|
| 45 |
+
},
|
| 46 |
+
"format": "float-quantized",
|
| 47 |
+
"global_compression_ratio": null,
|
| 48 |
+
"ignore": [
|
| 49 |
+
"model.visual.blocks.0.attn.qkv",
|
| 50 |
+
"model.visual.blocks.0.attn.proj",
|
| 51 |
+
"model.visual.blocks.0.mlp.linear_fc1",
|
| 52 |
+
"model.visual.blocks.0.mlp.linear_fc2",
|
| 53 |
+
"model.visual.blocks.1.attn.qkv",
|
| 54 |
+
"model.visual.blocks.1.attn.proj",
|
| 55 |
+
"model.visual.blocks.1.mlp.linear_fc1",
|
| 56 |
+
"model.visual.blocks.1.mlp.linear_fc2",
|
| 57 |
+
"model.visual.blocks.2.attn.qkv",
|
| 58 |
+
"model.visual.blocks.2.attn.proj",
|
| 59 |
+
"model.visual.blocks.2.mlp.linear_fc1",
|
| 60 |
+
"model.visual.blocks.2.mlp.linear_fc2",
|
| 61 |
+
"model.visual.blocks.3.attn.qkv",
|
| 62 |
+
"model.visual.blocks.3.attn.proj",
|
| 63 |
+
"model.visual.blocks.3.mlp.linear_fc1",
|
| 64 |
+
"model.visual.blocks.3.mlp.linear_fc2",
|
| 65 |
+
"model.visual.blocks.4.attn.qkv",
|
| 66 |
+
"model.visual.blocks.4.attn.proj",
|
| 67 |
+
"model.visual.blocks.4.mlp.linear_fc1",
|
| 68 |
+
"model.visual.blocks.4.mlp.linear_fc2",
|
| 69 |
+
"model.visual.blocks.5.attn.qkv",
|
| 70 |
+
"model.visual.blocks.5.attn.proj",
|
| 71 |
+
"model.visual.blocks.5.mlp.linear_fc1",
|
| 72 |
+
"model.visual.blocks.5.mlp.linear_fc2",
|
| 73 |
+
"model.visual.blocks.6.attn.qkv",
|
| 74 |
+
"model.visual.blocks.6.attn.proj",
|
| 75 |
+
"model.visual.blocks.6.mlp.linear_fc1",
|
| 76 |
+
"model.visual.blocks.6.mlp.linear_fc2",
|
| 77 |
+
"model.visual.blocks.7.attn.qkv",
|
| 78 |
+
"model.visual.blocks.7.attn.proj",
|
| 79 |
+
"model.visual.blocks.7.mlp.linear_fc1",
|
| 80 |
+
"model.visual.blocks.7.mlp.linear_fc2",
|
| 81 |
+
"model.visual.blocks.8.attn.qkv",
|
| 82 |
+
"model.visual.blocks.8.attn.proj",
|
| 83 |
+
"model.visual.blocks.8.mlp.linear_fc1",
|
| 84 |
+
"model.visual.blocks.8.mlp.linear_fc2",
|
| 85 |
+
"model.visual.blocks.9.attn.qkv",
|
| 86 |
+
"model.visual.blocks.9.attn.proj",
|
| 87 |
+
"model.visual.blocks.9.mlp.linear_fc1",
|
| 88 |
+
"model.visual.blocks.9.mlp.linear_fc2",
|
| 89 |
+
"model.visual.blocks.10.attn.qkv",
|
| 90 |
+
"model.visual.blocks.10.attn.proj",
|
| 91 |
+
"model.visual.blocks.10.mlp.linear_fc1",
|
| 92 |
+
"model.visual.blocks.10.mlp.linear_fc2",
|
| 93 |
+
"model.visual.blocks.11.attn.qkv",
|
| 94 |
+
"model.visual.blocks.11.attn.proj",
|
| 95 |
+
"model.visual.blocks.11.mlp.linear_fc1",
|
| 96 |
+
"model.visual.blocks.11.mlp.linear_fc2",
|
| 97 |
+
"model.visual.blocks.12.attn.qkv",
|
| 98 |
+
"model.visual.blocks.12.attn.proj",
|
| 99 |
+
"model.visual.blocks.12.mlp.linear_fc1",
|
| 100 |
+
"model.visual.blocks.12.mlp.linear_fc2",
|
| 101 |
+
"model.visual.blocks.13.attn.qkv",
|
| 102 |
+
"model.visual.blocks.13.attn.proj",
|
| 103 |
+
"model.visual.blocks.13.mlp.linear_fc1",
|
| 104 |
+
"model.visual.blocks.13.mlp.linear_fc2",
|
| 105 |
+
"model.visual.blocks.14.attn.qkv",
|
| 106 |
+
"model.visual.blocks.14.attn.proj",
|
| 107 |
+
"model.visual.blocks.14.mlp.linear_fc1",
|
| 108 |
+
"model.visual.blocks.14.mlp.linear_fc2",
|
| 109 |
+
"model.visual.blocks.15.attn.qkv",
|
| 110 |
+
"model.visual.blocks.15.attn.proj",
|
| 111 |
+
"model.visual.blocks.15.mlp.linear_fc1",
|
| 112 |
+
"model.visual.blocks.15.mlp.linear_fc2",
|
| 113 |
+
"model.visual.blocks.16.attn.qkv",
|
| 114 |
+
"model.visual.blocks.16.attn.proj",
|
| 115 |
+
"model.visual.blocks.16.mlp.linear_fc1",
|
| 116 |
+
"model.visual.blocks.16.mlp.linear_fc2",
|
| 117 |
+
"model.visual.blocks.17.attn.qkv",
|
| 118 |
+
"model.visual.blocks.17.attn.proj",
|
| 119 |
+
"model.visual.blocks.17.mlp.linear_fc1",
|
| 120 |
+
"model.visual.blocks.17.mlp.linear_fc2",
|
| 121 |
+
"model.visual.blocks.18.attn.qkv",
|
| 122 |
+
"model.visual.blocks.18.attn.proj",
|
| 123 |
+
"model.visual.blocks.18.mlp.linear_fc1",
|
| 124 |
+
"model.visual.blocks.18.mlp.linear_fc2",
|
| 125 |
+
"model.visual.blocks.19.attn.qkv",
|
| 126 |
+
"model.visual.blocks.19.attn.proj",
|
| 127 |
+
"model.visual.blocks.19.mlp.linear_fc1",
|
| 128 |
+
"model.visual.blocks.19.mlp.linear_fc2",
|
| 129 |
+
"model.visual.blocks.20.attn.qkv",
|
| 130 |
+
"model.visual.blocks.20.attn.proj",
|
| 131 |
+
"model.visual.blocks.20.mlp.linear_fc1",
|
| 132 |
+
"model.visual.blocks.20.mlp.linear_fc2",
|
| 133 |
+
"model.visual.blocks.21.attn.qkv",
|
| 134 |
+
"model.visual.blocks.21.attn.proj",
|
| 135 |
+
"model.visual.blocks.21.mlp.linear_fc1",
|
| 136 |
+
"model.visual.blocks.21.mlp.linear_fc2",
|
| 137 |
+
"model.visual.blocks.22.attn.qkv",
|
| 138 |
+
"model.visual.blocks.22.attn.proj",
|
| 139 |
+
"model.visual.blocks.22.mlp.linear_fc1",
|
| 140 |
+
"model.visual.blocks.22.mlp.linear_fc2",
|
| 141 |
+
"model.visual.blocks.23.attn.qkv",
|
| 142 |
+
"model.visual.blocks.23.attn.proj",
|
| 143 |
+
"model.visual.blocks.23.mlp.linear_fc1",
|
| 144 |
+
"model.visual.blocks.23.mlp.linear_fc2",
|
| 145 |
+
"model.visual.blocks.24.attn.qkv",
|
| 146 |
+
"model.visual.blocks.24.attn.proj",
|
| 147 |
+
"model.visual.blocks.24.mlp.linear_fc1",
|
| 148 |
+
"model.visual.blocks.24.mlp.linear_fc2",
|
| 149 |
+
"model.visual.blocks.25.attn.qkv",
|
| 150 |
+
"model.visual.blocks.25.attn.proj",
|
| 151 |
+
"model.visual.blocks.25.mlp.linear_fc1",
|
| 152 |
+
"model.visual.blocks.25.mlp.linear_fc2",
|
| 153 |
+
"model.visual.blocks.26.attn.qkv",
|
| 154 |
+
"model.visual.blocks.26.attn.proj",
|
| 155 |
+
"model.visual.blocks.26.mlp.linear_fc1",
|
| 156 |
+
"model.visual.blocks.26.mlp.linear_fc2",
|
| 157 |
+
"model.visual.merger.linear_fc1",
|
| 158 |
+
"model.visual.merger.linear_fc2",
|
| 159 |
+
"model.language_model.layers.0.linear_attn.in_proj_b",
|
| 160 |
+
"model.language_model.layers.0.linear_attn.in_proj_a",
|
| 161 |
+
"model.language_model.layers.1.linear_attn.in_proj_b",
|
| 162 |
+
"model.language_model.layers.1.linear_attn.in_proj_a",
|
| 163 |
+
"model.language_model.layers.2.linear_attn.in_proj_b",
|
| 164 |
+
"model.language_model.layers.2.linear_attn.in_proj_a",
|
| 165 |
+
"model.language_model.layers.4.linear_attn.in_proj_b",
|
| 166 |
+
"model.language_model.layers.4.linear_attn.in_proj_a",
|
| 167 |
+
"model.language_model.layers.5.linear_attn.in_proj_b",
|
| 168 |
+
"model.language_model.layers.5.linear_attn.in_proj_a",
|
| 169 |
+
"model.language_model.layers.6.linear_attn.in_proj_b",
|
| 170 |
+
"model.language_model.layers.6.linear_attn.in_proj_a",
|
| 171 |
+
"model.language_model.layers.8.linear_attn.in_proj_b",
|
| 172 |
+
"model.language_model.layers.8.linear_attn.in_proj_a",
|
| 173 |
+
"model.language_model.layers.9.linear_attn.in_proj_b",
|
| 174 |
+
"model.language_model.layers.9.linear_attn.in_proj_a",
|
| 175 |
+
"model.language_model.layers.10.linear_attn.in_proj_b",
|
| 176 |
+
"model.language_model.layers.10.linear_attn.in_proj_a",
|
| 177 |
+
"model.language_model.layers.12.linear_attn.in_proj_b",
|
| 178 |
+
"model.language_model.layers.12.linear_attn.in_proj_a",
|
| 179 |
+
"model.language_model.layers.13.linear_attn.in_proj_b",
|
| 180 |
+
"model.language_model.layers.13.linear_attn.in_proj_a",
|
| 181 |
+
"model.language_model.layers.14.linear_attn.in_proj_b",
|
| 182 |
+
"model.language_model.layers.14.linear_attn.in_proj_a",
|
| 183 |
+
"model.language_model.layers.16.linear_attn.in_proj_b",
|
| 184 |
+
"model.language_model.layers.16.linear_attn.in_proj_a",
|
| 185 |
+
"model.language_model.layers.17.linear_attn.in_proj_b",
|
| 186 |
+
"model.language_model.layers.17.linear_attn.in_proj_a",
|
| 187 |
+
"model.language_model.layers.18.linear_attn.in_proj_b",
|
| 188 |
+
"model.language_model.layers.18.linear_attn.in_proj_a",
|
| 189 |
+
"model.language_model.layers.20.linear_attn.in_proj_b",
|
| 190 |
+
"model.language_model.layers.20.linear_attn.in_proj_a",
|
| 191 |
+
"model.language_model.layers.21.linear_attn.in_proj_b",
|
| 192 |
+
"model.language_model.layers.21.linear_attn.in_proj_a",
|
| 193 |
+
"model.language_model.layers.22.linear_attn.in_proj_b",
|
| 194 |
+
"model.language_model.layers.22.linear_attn.in_proj_a",
|
| 195 |
+
"model.language_model.layers.24.linear_attn.in_proj_b",
|
| 196 |
+
"model.language_model.layers.24.linear_attn.in_proj_a",
|
| 197 |
+
"model.language_model.layers.25.linear_attn.in_proj_b",
|
| 198 |
+
"model.language_model.layers.25.linear_attn.in_proj_a",
|
| 199 |
+
"model.language_model.layers.26.linear_attn.in_proj_b",
|
| 200 |
+
"model.language_model.layers.26.linear_attn.in_proj_a",
|
| 201 |
+
"model.language_model.layers.28.linear_attn.in_proj_b",
|
| 202 |
+
"model.language_model.layers.28.linear_attn.in_proj_a",
|
| 203 |
+
"model.language_model.layers.29.linear_attn.in_proj_b",
|
| 204 |
+
"model.language_model.layers.29.linear_attn.in_proj_a",
|
| 205 |
+
"model.language_model.layers.30.linear_attn.in_proj_b",
|
| 206 |
+
"model.language_model.layers.30.linear_attn.in_proj_a",
|
| 207 |
+
"lm_head"
|
| 208 |
+
],
|
| 209 |
+
"kv_cache_scheme": null,
|
| 210 |
+
"quant_method": "compressed-tensors",
|
| 211 |
+
"quantization_status": "compressed",
|
| 212 |
+
"sparsity_config": {},
|
| 213 |
+
"transform_config": {},
|
| 214 |
+
"version": "0.19.0"
|
| 215 |
+
},
|
| 216 |
+
"text_config": {
|
| 217 |
+
"attention_bias": false,
|
| 218 |
+
"attention_dropout": 0.0,
|
| 219 |
+
"attn_output_gate": true,
|
| 220 |
+
"bos_token_id": null,
|
| 221 |
+
"dtype": "bfloat16",
|
| 222 |
+
"eos_token_id": 248044,
|
| 223 |
+
"full_attention_interval": 4,
|
| 224 |
+
"head_dim": 256,
|
| 225 |
+
"hidden_act": "silu",
|
| 226 |
+
"hidden_size": 4096,
|
| 227 |
+
"initializer_range": 0.02,
|
| 228 |
+
"intermediate_size": 12288,
|
| 229 |
+
"layer_types": [
|
| 230 |
+
"linear_attention",
|
| 231 |
+
"linear_attention",
|
| 232 |
+
"linear_attention",
|
| 233 |
+
"full_attention",
|
| 234 |
+
"linear_attention",
|
| 235 |
+
"linear_attention",
|
| 236 |
+
"linear_attention",
|
| 237 |
+
"full_attention",
|
| 238 |
+
"linear_attention",
|
| 239 |
+
"linear_attention",
|
| 240 |
+
"linear_attention",
|
| 241 |
+
"full_attention",
|
| 242 |
+
"linear_attention",
|
| 243 |
+
"linear_attention",
|
| 244 |
+
"linear_attention",
|
| 245 |
+
"full_attention",
|
| 246 |
+
"linear_attention",
|
| 247 |
+
"linear_attention",
|
| 248 |
+
"linear_attention",
|
| 249 |
+
"full_attention",
|
| 250 |
+
"linear_attention",
|
| 251 |
+
"linear_attention",
|
| 252 |
+
"linear_attention",
|
| 253 |
+
"full_attention",
|
| 254 |
+
"linear_attention",
|
| 255 |
+
"linear_attention",
|
| 256 |
+
"linear_attention",
|
| 257 |
+
"full_attention",
|
| 258 |
+
"linear_attention",
|
| 259 |
+
"linear_attention",
|
| 260 |
+
"linear_attention",
|
| 261 |
+
"full_attention"
|
| 262 |
+
],
|
| 263 |
+
"linear_conv_kernel_dim": 4,
|
| 264 |
+
"linear_key_head_dim": 128,
|
| 265 |
+
"linear_num_key_heads": 16,
|
| 266 |
+
"linear_num_value_heads": 32,
|
| 267 |
+
"linear_value_head_dim": 128,
|
| 268 |
+
"mamba_ssm_dtype": "float32",
|
| 269 |
+
"max_position_embeddings": 262144,
|
| 270 |
+
"mlp_only_layers": [],
|
| 271 |
+
"model_type": "qwen3_5_text",
|
| 272 |
+
"mtp_num_hidden_layers": 0,
|
| 273 |
+
"mtp_use_dedicated_embeddings": false,
|
| 274 |
+
"num_attention_heads": 16,
|
| 275 |
+
"num_hidden_layers": 32,
|
| 276 |
+
"num_key_value_heads": 4,
|
| 277 |
+
"pad_token_id": null,
|
| 278 |
+
"partial_rotary_factor": 0.25,
|
| 279 |
+
"rms_norm_eps": 1e-06,
|
| 280 |
+
"rope_parameters": {
|
| 281 |
+
"mrope_interleaved": true,
|
| 282 |
+
"mrope_section": [
|
| 283 |
+
11,
|
| 284 |
+
11,
|
| 285 |
+
10
|
| 286 |
+
],
|
| 287 |
+
"partial_rotary_factor": 0.25,
|
| 288 |
+
"rope_theta": 10000000,
|
| 289 |
+
"rope_type": "default"
|
| 290 |
+
},
|
| 291 |
+
"tie_word_embeddings": false,
|
| 292 |
+
"use_cache": true,
|
| 293 |
+
"vocab_size": 248320
|
| 294 |
+
},
|
| 295 |
+
"tie_word_embeddings": false,
|
| 296 |
+
"transformers_version": "5.10.2",
|
| 297 |
+
"video_token_id": 248057,
|
| 298 |
+
"vision_config": {
|
| 299 |
+
"deepstack_visual_indexes": [],
|
| 300 |
+
"depth": 27,
|
| 301 |
+
"dtype": "bfloat16",
|
| 302 |
+
"hidden_act": "gelu_pytorch_tanh",
|
| 303 |
+
"hidden_size": 1152,
|
| 304 |
+
"in_channels": 3,
|
| 305 |
+
"initializer_range": 0.02,
|
| 306 |
+
"intermediate_size": 4304,
|
| 307 |
+
"model_type": "qwen3_5_vision",
|
| 308 |
+
"num_heads": 16,
|
| 309 |
+
"num_position_embeddings": 2304,
|
| 310 |
+
"out_hidden_size": 4096,
|
| 311 |
+
"patch_size": 16,
|
| 312 |
+
"spatial_merge_size": 2,
|
| 313 |
+
"temporal_patch_size": 2
|
| 314 |
+
},
|
| 315 |
+
"vision_end_token_id": 248054,
|
| 316 |
+
"vision_start_token_id": 248053
|
| 317 |
+
}
|
generation_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"eos_token_id": 248044,
|
| 4 |
+
"transformers_version": "5.10.2",
|
| 5 |
+
"use_cache": true
|
| 6 |
+
}
|
joint_schema_model.py
ADDED
|
@@ -0,0 +1,576 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Clef: a multimodal Qwen backbone with a joint schema head for typed decisions.
|
| 2 |
+
|
| 3 |
+
A record provides a ``state`` (any JSON value), optional ``images`` and ``videos``,
|
| 4 |
+
and ``questions``. Each question has a ``type`` (``noul``, ``choice``, or ``score``),
|
| 5 |
+
``instructions``, and, for ``choice`` and ``score``, ``criteria`` describing the
|
| 6 |
+
allowed options. The model returns one logit per allowed option for every question.
|
| 7 |
+
``systemone`` answers a Jev/SystemOne ``/v1/systemone`` request body with the same
|
| 8 |
+
response body.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import json
|
| 14 |
+
import math
|
| 15 |
+
from dataclasses import dataclass
|
| 16 |
+
from dataclasses import field as dataclass_field
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
from typing import Any
|
| 19 |
+
|
| 20 |
+
import torch
|
| 21 |
+
import torch.nn.functional as functional
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
SYSTEM_PROMPT = (
|
| 25 |
+
"Read the complete state and schema. Decide every field jointly. Each answer "
|
| 26 |
+
"must be exactly one of that field's allowed options."
|
| 27 |
+
)
|
| 28 |
+
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
|
| 29 |
+
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
|
| 30 |
+
MEDIA_BATCH_KEYS = ("pixel_values", "image_grid_thw", "pixel_values_videos", "video_grid_thw")
|
| 31 |
+
MEDIA_TOKEN_KEYS = ("mm_token_type_ids",)
|
| 32 |
+
QUESTION_TYPES = {"noul": 0, "choice": 1, "score": 2}
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def render(value: Any) -> str:
|
| 36 |
+
if isinstance(value, str):
|
| 37 |
+
return value
|
| 38 |
+
return json.dumps(
|
| 39 |
+
value,
|
| 40 |
+
ensure_ascii=False,
|
| 41 |
+
separators=(",", ":"),
|
| 42 |
+
sort_keys=True,
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def question_options(question: dict[str, Any]) -> list[tuple[str, Any]]:
|
| 47 |
+
question_type = str(question["type"])
|
| 48 |
+
if question_type == "noul":
|
| 49 |
+
criteria = {
|
| 50 |
+
"true": "The proposition is true or the answer is yes.",
|
| 51 |
+
"false": "The proposition is false or the answer is no.",
|
| 52 |
+
}
|
| 53 |
+
criteria.update(question.get("criteria") or {})
|
| 54 |
+
return [(key, criteria[key]) for key in ("true", "false")]
|
| 55 |
+
if question_type == "choice":
|
| 56 |
+
return sorted((str(key), value) for key, value in question["criteria"].items())
|
| 57 |
+
return [(str(index), value) for index, value in enumerate(question["criteria"])]
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
@dataclass(frozen=True)
|
| 61 |
+
class EncodedQuestion:
|
| 62 |
+
question_id: str
|
| 63 |
+
question_type: int
|
| 64 |
+
question_span: tuple[int, int]
|
| 65 |
+
option_spans: tuple[tuple[int, int], ...]
|
| 66 |
+
option_ids: tuple[str, ...]
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
@dataclass(frozen=True)
|
| 70 |
+
class EncodedRecord:
|
| 71 |
+
input_ids: tuple[int, ...]
|
| 72 |
+
questions: tuple[EncodedQuestion, ...]
|
| 73 |
+
record_id: str
|
| 74 |
+
media: dict[str, Any] | None = dataclass_field(default=None, compare=False, repr=False)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _tokens(tokenizer: Any, text: str) -> list[int]:
|
| 78 |
+
return tokenizer(text, add_special_tokens=False).input_ids
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _encode_media(processor: Any, record: dict[str, Any]) -> tuple[list[int], dict[str, Any] | None]:
|
| 82 |
+
images = list(record.get("images") or [])
|
| 83 |
+
videos = list(record.get("videos") or [])
|
| 84 |
+
if not images and not videos:
|
| 85 |
+
return [], None
|
| 86 |
+
if processor is None:
|
| 87 |
+
raise ValueError("records with images or videos require a processor")
|
| 88 |
+
text = IMAGE_PLACEHOLDER * len(images) + VIDEO_PLACEHOLDER * len(videos) + "\n"
|
| 89 |
+
encoded = processor(
|
| 90 |
+
text=[text],
|
| 91 |
+
images=images or None,
|
| 92 |
+
videos=videos or None,
|
| 93 |
+
return_tensors="pt",
|
| 94 |
+
**(record.get("media_kwargs") or {}),
|
| 95 |
+
)
|
| 96 |
+
media = {key: encoded[key] for key in MEDIA_BATCH_KEYS if key in encoded}
|
| 97 |
+
for key in MEDIA_TOKEN_KEYS:
|
| 98 |
+
if key in encoded:
|
| 99 |
+
media[key] = encoded[key][0].tolist()
|
| 100 |
+
return encoded["input_ids"][0].tolist(), media
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def encode_record(
|
| 104 |
+
tokenizer: Any,
|
| 105 |
+
record: dict[str, Any],
|
| 106 |
+
max_length: int = 16384,
|
| 107 |
+
max_state_tokens: int | None = None,
|
| 108 |
+
processor: Any | None = None,
|
| 109 |
+
) -> EncodedRecord:
|
| 110 |
+
schema_ids = _tokens(tokenizer, "\n\nSCHEMA FIELDS:\n")
|
| 111 |
+
questions: list[EncodedQuestion] = []
|
| 112 |
+
for question_index, (question_id, question) in enumerate(record["questions"].items()):
|
| 113 |
+
schema_ids.extend(
|
| 114 |
+
_tokens(
|
| 115 |
+
tokenizer,
|
| 116 |
+
f"\nFIELD {question_index + 1}\nID: {question_id}\nTYPE: {question['type']}\nINSTRUCTION: ",
|
| 117 |
+
)
|
| 118 |
+
)
|
| 119 |
+
question_start = len(schema_ids)
|
| 120 |
+
instructions = question.get("instructions")
|
| 121 |
+
if instructions is None or instructions == "":
|
| 122 |
+
instructions = str(question_id)
|
| 123 |
+
schema_ids.extend(_tokens(tokenizer, render(instructions)))
|
| 124 |
+
question_end = len(schema_ids)
|
| 125 |
+
schema_ids.extend(_tokens(tokenizer, "\nALLOWED OPTIONS:\n"))
|
| 126 |
+
|
| 127 |
+
option_spans: list[tuple[int, int]] = []
|
| 128 |
+
option_ids: list[str] = []
|
| 129 |
+
for option_index, (option_id, description) in enumerate(question_options(question)):
|
| 130 |
+
schema_ids.extend(_tokens(tokenizer, f"OPTION {option_index + 1}: "))
|
| 131 |
+
option_start = len(schema_ids)
|
| 132 |
+
semantics = {"option_id": option_id}
|
| 133 |
+
if description is not None:
|
| 134 |
+
semantics["description"] = description
|
| 135 |
+
schema_ids.extend(_tokens(tokenizer, render(semantics)))
|
| 136 |
+
option_spans.append((option_start, len(schema_ids)))
|
| 137 |
+
option_ids.append(option_id)
|
| 138 |
+
schema_ids.extend(_tokens(tokenizer, "\n"))
|
| 139 |
+
schema_ids.extend(_tokens(tokenizer, "END FIELD\n"))
|
| 140 |
+
questions.append(
|
| 141 |
+
EncodedQuestion(
|
| 142 |
+
question_id=str(question_id),
|
| 143 |
+
question_type=QUESTION_TYPES[str(question["type"])],
|
| 144 |
+
question_span=(question_start, question_end),
|
| 145 |
+
option_spans=tuple(option_spans),
|
| 146 |
+
option_ids=tuple(option_ids),
|
| 147 |
+
)
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
prefix_ids = _tokens(
|
| 151 |
+
tokenizer,
|
| 152 |
+
f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n<|im_start|>user\nSTATE:\n",
|
| 153 |
+
)
|
| 154 |
+
suffix_ids = _tokens(
|
| 155 |
+
tokenizer,
|
| 156 |
+
"\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\nJOINT SCHEMA DECISIONS:",
|
| 157 |
+
)
|
| 158 |
+
media_ids, media = _encode_media(processor, record)
|
| 159 |
+
if media is not None:
|
| 160 |
+
media["token_offset"] = len(prefix_ids)
|
| 161 |
+
prefix_ids = prefix_ids + media_ids
|
| 162 |
+
state_ids = _tokens(tokenizer, render(record["state"]))
|
| 163 |
+
if max_state_tokens is not None:
|
| 164 |
+
state_ids = state_ids[:max_state_tokens]
|
| 165 |
+
fixed_length = len(prefix_ids) + len(schema_ids) + len(suffix_ids)
|
| 166 |
+
if fixed_length > max_length:
|
| 167 |
+
raise ValueError(
|
| 168 |
+
f"schema requires {fixed_length} tokens before state; maximum is {max_length}"
|
| 169 |
+
)
|
| 170 |
+
state_ids = state_ids[: max_length - fixed_length]
|
| 171 |
+
schema_offset = len(prefix_ids) + len(state_ids)
|
| 172 |
+
shifted_questions = tuple(
|
| 173 |
+
EncodedQuestion(
|
| 174 |
+
question_id=question.question_id,
|
| 175 |
+
question_type=question.question_type,
|
| 176 |
+
question_span=(
|
| 177 |
+
question.question_span[0] + schema_offset,
|
| 178 |
+
question.question_span[1] + schema_offset,
|
| 179 |
+
),
|
| 180 |
+
option_spans=tuple(
|
| 181 |
+
(start + schema_offset, end + schema_offset)
|
| 182 |
+
for start, end in question.option_spans
|
| 183 |
+
),
|
| 184 |
+
option_ids=question.option_ids,
|
| 185 |
+
)
|
| 186 |
+
for question in questions
|
| 187 |
+
)
|
| 188 |
+
input_ids = tuple(prefix_ids + state_ids + schema_ids + suffix_ids)
|
| 189 |
+
if not input_ids or not shifted_questions:
|
| 190 |
+
raise ValueError("record produced no model input or questions")
|
| 191 |
+
return EncodedRecord(
|
| 192 |
+
input_ids=input_ids,
|
| 193 |
+
questions=shifted_questions,
|
| 194 |
+
record_id=str(record.get("id", "unknown")),
|
| 195 |
+
media=media,
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def collate_records(
|
| 200 |
+
records: list[EncodedRecord],
|
| 201 |
+
pad_token_id: int,
|
| 202 |
+
device: torch.device,
|
| 203 |
+
) -> dict[str, Any]:
|
| 204 |
+
maximum_length = max(len(record.input_ids) for record in records)
|
| 205 |
+
input_ids = torch.full(
|
| 206 |
+
(len(records), maximum_length),
|
| 207 |
+
pad_token_id,
|
| 208 |
+
dtype=torch.long,
|
| 209 |
+
device=device,
|
| 210 |
+
)
|
| 211 |
+
attention_mask = torch.zeros(
|
| 212 |
+
(len(records), maximum_length),
|
| 213 |
+
dtype=torch.long,
|
| 214 |
+
device=device,
|
| 215 |
+
)
|
| 216 |
+
for index, record in enumerate(records):
|
| 217 |
+
length = len(record.input_ids)
|
| 218 |
+
input_ids[index, :length] = torch.tensor(record.input_ids, device=device)
|
| 219 |
+
attention_mask[index, :length] = 1
|
| 220 |
+
media: dict[str, torch.Tensor] = {}
|
| 221 |
+
for key in MEDIA_BATCH_KEYS:
|
| 222 |
+
values = [record.media[key] for record in records if record.media and key in record.media]
|
| 223 |
+
if values:
|
| 224 |
+
media[key] = torch.cat(values, dim=0).to(device)
|
| 225 |
+
for key in MEDIA_TOKEN_KEYS:
|
| 226 |
+
if any(record.media and key in record.media for record in records):
|
| 227 |
+
token_values = torch.zeros((len(records), maximum_length), dtype=torch.long, device=device)
|
| 228 |
+
for index, record in enumerate(records):
|
| 229 |
+
if record.media and key in record.media:
|
| 230 |
+
offset = record.media["token_offset"]
|
| 231 |
+
values = torch.tensor(record.media[key], dtype=torch.long, device=device)
|
| 232 |
+
token_values[index, offset : offset + len(values)] = values
|
| 233 |
+
media[key] = token_values
|
| 234 |
+
return {
|
| 235 |
+
"input_ids": input_ids,
|
| 236 |
+
"attention_mask": attention_mask,
|
| 237 |
+
"records": records,
|
| 238 |
+
"media": media,
|
| 239 |
+
}
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
class EvidenceRoutingLayer(torch.nn.Module):
|
| 243 |
+
def __init__(
|
| 244 |
+
self,
|
| 245 |
+
width: int,
|
| 246 |
+
heads: int,
|
| 247 |
+
feedforward: int,
|
| 248 |
+
dropout: float = 0.0,
|
| 249 |
+
) -> None:
|
| 250 |
+
super().__init__()
|
| 251 |
+
self.query_norm = torch.nn.LayerNorm(width)
|
| 252 |
+
self.memory_norm = torch.nn.LayerNorm(width)
|
| 253 |
+
self.attention = torch.nn.MultiheadAttention(
|
| 254 |
+
width,
|
| 255 |
+
heads,
|
| 256 |
+
dropout=dropout,
|
| 257 |
+
batch_first=True,
|
| 258 |
+
)
|
| 259 |
+
self.attention_dropout = torch.nn.Dropout(dropout)
|
| 260 |
+
self.feedforward_norm = torch.nn.LayerNorm(width)
|
| 261 |
+
self.feedforward = torch.nn.Sequential(
|
| 262 |
+
torch.nn.Linear(width, feedforward),
|
| 263 |
+
torch.nn.GELU(),
|
| 264 |
+
torch.nn.Dropout(dropout),
|
| 265 |
+
torch.nn.Linear(feedforward, width),
|
| 266 |
+
torch.nn.Dropout(dropout),
|
| 267 |
+
)
|
| 268 |
+
|
| 269 |
+
def forward(self, queries: torch.Tensor, memory: torch.Tensor) -> torch.Tensor:
|
| 270 |
+
normalized_queries = self.query_norm(queries)
|
| 271 |
+
routed, _ = self.attention(
|
| 272 |
+
normalized_queries,
|
| 273 |
+
self.memory_norm(memory),
|
| 274 |
+
self.memory_norm(memory),
|
| 275 |
+
need_weights=False,
|
| 276 |
+
)
|
| 277 |
+
queries = queries + self.attention_dropout(routed)
|
| 278 |
+
return queries + self.feedforward(self.feedforward_norm(queries))
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
class JointSchemaHead(torch.nn.Module):
|
| 282 |
+
def __init__(
|
| 283 |
+
self,
|
| 284 |
+
hidden_size: int,
|
| 285 |
+
width: int,
|
| 286 |
+
routing_layers: int,
|
| 287 |
+
layers: int,
|
| 288 |
+
heads: int,
|
| 289 |
+
feedforward: int,
|
| 290 |
+
dropout: float = 0.0,
|
| 291 |
+
) -> None:
|
| 292 |
+
super().__init__()
|
| 293 |
+
self.hidden_norm = torch.nn.LayerNorm(hidden_size)
|
| 294 |
+
self.memory_projection = torch.nn.Linear(hidden_size, width, bias=False)
|
| 295 |
+
self.question_projection = torch.nn.Linear(hidden_size, width, bias=False)
|
| 296 |
+
self.option_question_projection = torch.nn.Linear(hidden_size, width, bias=False)
|
| 297 |
+
self.global_projection = torch.nn.Linear(hidden_size, width, bias=False)
|
| 298 |
+
self.option_context_projection = torch.nn.Linear(hidden_size, width, bias=False)
|
| 299 |
+
self.option_lexical_projection = torch.nn.Linear(hidden_size, width, bias=False)
|
| 300 |
+
self.type_embedding = torch.nn.Embedding(3, width)
|
| 301 |
+
self.evidence_layers = torch.nn.ModuleList(
|
| 302 |
+
[
|
| 303 |
+
EvidenceRoutingLayer(
|
| 304 |
+
width=width,
|
| 305 |
+
heads=heads,
|
| 306 |
+
feedforward=feedforward,
|
| 307 |
+
dropout=dropout,
|
| 308 |
+
)
|
| 309 |
+
for _ in range(routing_layers)
|
| 310 |
+
]
|
| 311 |
+
)
|
| 312 |
+
self.option_summary_norm = torch.nn.LayerNorm(width)
|
| 313 |
+
self.layers = torch.nn.ModuleList(
|
| 314 |
+
[
|
| 315 |
+
torch.nn.TransformerDecoderLayer(
|
| 316 |
+
d_model=width,
|
| 317 |
+
nhead=heads,
|
| 318 |
+
dim_feedforward=feedforward,
|
| 319 |
+
dropout=dropout,
|
| 320 |
+
activation="gelu",
|
| 321 |
+
batch_first=True,
|
| 322 |
+
norm_first=True,
|
| 323 |
+
)
|
| 324 |
+
for _ in range(layers)
|
| 325 |
+
]
|
| 326 |
+
)
|
| 327 |
+
self.field_norm = torch.nn.LayerNorm(width)
|
| 328 |
+
self.option_norm = torch.nn.LayerNorm(width)
|
| 329 |
+
self.residual_scorer = torch.nn.Sequential(
|
| 330 |
+
torch.nn.Linear(width * 4, width),
|
| 331 |
+
torch.nn.GELU(),
|
| 332 |
+
torch.nn.Dropout(dropout),
|
| 333 |
+
torch.nn.Linear(width, 1),
|
| 334 |
+
)
|
| 335 |
+
self.prior_logit_scale = torch.nn.Parameter(torch.zeros(()))
|
| 336 |
+
self.joint_logit_scale = torch.nn.Parameter(torch.zeros(()))
|
| 337 |
+
self.residual_gate = torch.nn.Parameter(torch.zeros(()))
|
| 338 |
+
|
| 339 |
+
@staticmethod
|
| 340 |
+
def _mean_span(values: torch.Tensor, span: tuple[int, int]) -> torch.Tensor:
|
| 341 |
+
start, end = span
|
| 342 |
+
return values[start:end].mean(dim=0)
|
| 343 |
+
|
| 344 |
+
def forward(
|
| 345 |
+
self,
|
| 346 |
+
hidden_states: torch.Tensor,
|
| 347 |
+
input_ids: torch.Tensor,
|
| 348 |
+
attention_mask: torch.Tensor,
|
| 349 |
+
records: list[EncodedRecord],
|
| 350 |
+
output_embedding_weight: torch.Tensor,
|
| 351 |
+
) -> list[list[torch.Tensor]]:
|
| 352 |
+
results: list[list[torch.Tensor]] = []
|
| 353 |
+
normalized_hidden = self.hidden_norm(hidden_states)
|
| 354 |
+
for batch_index, record in enumerate(records):
|
| 355 |
+
sequence_length = int(attention_mask[batch_index].sum().item())
|
| 356 |
+
sequence_hidden = normalized_hidden[batch_index, :sequence_length]
|
| 357 |
+
memory = self.memory_projection(sequence_hidden).unsqueeze(0)
|
| 358 |
+
global_vector = sequence_hidden[-1]
|
| 359 |
+
question_vectors = torch.stack(
|
| 360 |
+
[
|
| 361 |
+
self._mean_span(sequence_hidden, question.question_span)
|
| 362 |
+
for question in record.questions
|
| 363 |
+
]
|
| 364 |
+
)
|
| 365 |
+
type_ids = torch.tensor(
|
| 366 |
+
[question.question_type for question in record.questions],
|
| 367 |
+
device=hidden_states.device,
|
| 368 |
+
)
|
| 369 |
+
option_contexts: list[torch.Tensor] = []
|
| 370 |
+
lexical_options: list[torch.Tensor] = []
|
| 371 |
+
option_counts = []
|
| 372 |
+
for question in record.questions:
|
| 373 |
+
context_vectors = torch.stack(
|
| 374 |
+
[
|
| 375 |
+
self._mean_span(sequence_hidden, span)
|
| 376 |
+
for span in question.option_spans
|
| 377 |
+
]
|
| 378 |
+
)
|
| 379 |
+
lexical_vectors = []
|
| 380 |
+
for start, end in question.option_spans:
|
| 381 |
+
token_ids = input_ids[batch_index, start:end]
|
| 382 |
+
lexical_vectors.append(output_embedding_weight[token_ids].mean(dim=0))
|
| 383 |
+
lexical = torch.stack(lexical_vectors)
|
| 384 |
+
option_contexts.append(context_vectors)
|
| 385 |
+
lexical_options.append(lexical)
|
| 386 |
+
option_counts.append(len(question.option_spans))
|
| 387 |
+
|
| 388 |
+
option_queries = []
|
| 389 |
+
for question_index, (context_vectors, lexical) in enumerate(
|
| 390 |
+
zip(option_contexts, lexical_options)
|
| 391 |
+
):
|
| 392 |
+
option_queries.append(
|
| 393 |
+
self.option_context_projection(context_vectors)
|
| 394 |
+
+ self.option_lexical_projection(lexical)
|
| 395 |
+
+ self.option_question_projection(
|
| 396 |
+
question_vectors[question_index]
|
| 397 |
+
).unsqueeze(0)
|
| 398 |
+
)
|
| 399 |
+
routed_options = torch.cat(option_queries, dim=0).unsqueeze(0)
|
| 400 |
+
for layer in self.evidence_layers:
|
| 401 |
+
routed_options = layer(routed_options, memory)
|
| 402 |
+
routed_options = routed_options[0]
|
| 403 |
+
split_options = list(torch.split(routed_options, option_counts, dim=0))
|
| 404 |
+
|
| 405 |
+
base_fields = self.question_projection(question_vectors)
|
| 406 |
+
option_summaries = []
|
| 407 |
+
for field, options in zip(base_fields, split_options):
|
| 408 |
+
routing_weights = torch.softmax(
|
| 409 |
+
torch.matmul(options, field) / math.sqrt(options.shape[-1]),
|
| 410 |
+
dim=0,
|
| 411 |
+
)
|
| 412 |
+
option_summaries.append(
|
| 413 |
+
torch.sum(routing_weights.unsqueeze(-1) * options, dim=0)
|
| 414 |
+
)
|
| 415 |
+
fields = (
|
| 416 |
+
base_fields
|
| 417 |
+
+ self.option_summary_norm(torch.stack(option_summaries))
|
| 418 |
+
+ self.global_projection(global_vector).unsqueeze(0)
|
| 419 |
+
+ self.type_embedding(type_ids)
|
| 420 |
+
)
|
| 421 |
+
fields = fields.unsqueeze(0)
|
| 422 |
+
for layer in self.layers:
|
| 423 |
+
fields = layer(fields, memory)
|
| 424 |
+
fields = self.field_norm(fields[0])
|
| 425 |
+
|
| 426 |
+
record_logits: list[torch.Tensor] = []
|
| 427 |
+
for field, question, lexical, routed in zip(
|
| 428 |
+
fields,
|
| 429 |
+
record.questions,
|
| 430 |
+
lexical_options,
|
| 431 |
+
split_options,
|
| 432 |
+
):
|
| 433 |
+
anchor = functional.normalize(
|
| 434 |
+
question_vectors[len(record_logits)] + global_vector,
|
| 435 |
+
dim=-1,
|
| 436 |
+
)
|
| 437 |
+
lexical_anchor = functional.normalize(lexical, dim=-1)
|
| 438 |
+
prior_scale = self.prior_logit_scale.clamp(max=math.log(100.0)).exp()
|
| 439 |
+
prior = prior_scale * torch.matmul(lexical_anchor, anchor)
|
| 440 |
+
options = self.option_norm(routed)
|
| 441 |
+
repeated_field = field.unsqueeze(0).expand_as(options)
|
| 442 |
+
cosine = functional.cosine_similarity(repeated_field, options, dim=-1)
|
| 443 |
+
features = torch.cat(
|
| 444 |
+
[
|
| 445 |
+
repeated_field,
|
| 446 |
+
options,
|
| 447 |
+
repeated_field * options,
|
| 448 |
+
torch.abs(repeated_field - options),
|
| 449 |
+
],
|
| 450 |
+
dim=-1,
|
| 451 |
+
)
|
| 452 |
+
residual = self.residual_scorer(features).squeeze(-1)
|
| 453 |
+
joint_scale = self.joint_logit_scale.clamp(max=math.log(100.0)).exp()
|
| 454 |
+
joint = joint_scale * cosine + residual
|
| 455 |
+
record_logits.append(
|
| 456 |
+
prior + torch.sigmoid(self.residual_gate) * joint
|
| 457 |
+
)
|
| 458 |
+
results.append(record_logits)
|
| 459 |
+
return results
|
| 460 |
+
|
| 461 |
+
|
| 462 |
+
class ClefModel(torch.nn.Module):
|
| 463 |
+
def __init__(self, language_model: Any, head: JointSchemaHead) -> None:
|
| 464 |
+
super().__init__()
|
| 465 |
+
self.language_model = language_model
|
| 466 |
+
self.head = head
|
| 467 |
+
|
| 468 |
+
def forward(self, batch: dict[str, Any]) -> list[list[torch.Tensor]]:
|
| 469 |
+
base_model = (
|
| 470 |
+
self.language_model.get_base_model()
|
| 471 |
+
if hasattr(self.language_model, "get_base_model")
|
| 472 |
+
else self.language_model
|
| 473 |
+
)
|
| 474 |
+
media = batch.get("media") or {}
|
| 475 |
+
text_model = base_model.model
|
| 476 |
+
if not media and hasattr(text_model, "language_model"):
|
| 477 |
+
text_model = text_model.language_model
|
| 478 |
+
outputs = text_model(
|
| 479 |
+
input_ids=batch["input_ids"],
|
| 480 |
+
attention_mask=batch["attention_mask"],
|
| 481 |
+
use_cache=False,
|
| 482 |
+
return_dict=True,
|
| 483 |
+
**media,
|
| 484 |
+
)
|
| 485 |
+
return self.head(
|
| 486 |
+
outputs.last_hidden_state,
|
| 487 |
+
batch["input_ids"],
|
| 488 |
+
batch["attention_mask"],
|
| 489 |
+
batch["records"],
|
| 490 |
+
base_model.get_output_embeddings().weight,
|
| 491 |
+
)
|
| 492 |
+
|
| 493 |
+
|
| 494 |
+
def load_release_model(
|
| 495 |
+
model_path: str | Path,
|
| 496 |
+
device: str | torch.device = "cuda",
|
| 497 |
+
dtype: torch.dtype = torch.bfloat16,
|
| 498 |
+
**from_pretrained_kwargs: Any,
|
| 499 |
+
) -> tuple[ClefModel, Any]:
|
| 500 |
+
"""Load a Clef release (merged backbone, joint schema head, and processor)."""
|
| 501 |
+
from huggingface_hub import snapshot_download
|
| 502 |
+
from safetensors.torch import load_file
|
| 503 |
+
from transformers import AutoProcessor, Qwen3_5ForConditionalGeneration
|
| 504 |
+
|
| 505 |
+
path = Path(model_path)
|
| 506 |
+
if not path.is_dir():
|
| 507 |
+
path = Path(snapshot_download(str(model_path)))
|
| 508 |
+
backbone = Qwen3_5ForConditionalGeneration.from_pretrained(
|
| 509 |
+
path,
|
| 510 |
+
dtype=dtype,
|
| 511 |
+
device_map={"": str(device)},
|
| 512 |
+
**from_pretrained_kwargs,
|
| 513 |
+
)
|
| 514 |
+
backbone.config.use_cache = False
|
| 515 |
+
head_config = json.loads((path / "joint_head_config.json").read_text())
|
| 516 |
+
head = JointSchemaHead(**head_config)
|
| 517 |
+
head.load_state_dict(load_file(path / "joint_head.safetensors"), strict=True)
|
| 518 |
+
head = head.to(device=device, dtype=dtype)
|
| 519 |
+
processor = AutoProcessor.from_pretrained(path)
|
| 520 |
+
return ClefModel(backbone, head).eval(), processor
|
| 521 |
+
|
| 522 |
+
|
| 523 |
+
def systemone_answer(question: dict[str, Any], probabilities: dict[str, float]) -> dict[str, Any]:
|
| 524 |
+
"""Convert per-option probabilities for one question into a SystemOne answer."""
|
| 525 |
+
if question["type"] == "noul":
|
| 526 |
+
return {"type": "noul", "noul": round(probabilities["true"], 4)}
|
| 527 |
+
if question["type"] == "choice":
|
| 528 |
+
options = [str(option) for option in question["criteria"]]
|
| 529 |
+
choice = max(options, key=probabilities.__getitem__)
|
| 530 |
+
return {
|
| 531 |
+
"type": "choice",
|
| 532 |
+
"choice": choice,
|
| 533 |
+
"confidence": round(probabilities[choice], 4),
|
| 534 |
+
"probabilities": {option: round(probabilities[option], 4) for option in options},
|
| 535 |
+
}
|
| 536 |
+
levels = [str(index) for index in range(len(question["criteria"]))]
|
| 537 |
+
return {
|
| 538 |
+
"type": "score",
|
| 539 |
+
"score": round(sum(index * probabilities[level] for index, level in enumerate(levels)), 4),
|
| 540 |
+
"confidence": round(max(probabilities[level] for level in levels), 4),
|
| 541 |
+
"legend": dict(zip(levels, question["criteria"])),
|
| 542 |
+
"probabilities": {level: round(probabilities[level], 4) for level in levels},
|
| 543 |
+
}
|
| 544 |
+
|
| 545 |
+
|
| 546 |
+
@torch.inference_mode()
|
| 547 |
+
def systemone(model: ClefModel, processor: Any, request: dict[str, Any], max_length: int = 16384) -> dict[str, Any]:
|
| 548 |
+
"""Answer a Jev/SystemOne ``/v1/systemone`` request body with a SystemOne response body.
|
| 549 |
+
|
| 550 |
+
The request has ``model``, ``state``, and ``questions``, plus optional ``images`` and ``videos``.
|
| 551 |
+
"""
|
| 552 |
+
questions = request.get("questions")
|
| 553 |
+
if not isinstance(request.get("model"), str) or "state" not in request:
|
| 554 |
+
raise ValueError("model and state are required")
|
| 555 |
+
if not isinstance(questions, dict) or not questions:
|
| 556 |
+
raise ValueError("at least one question is required")
|
| 557 |
+
for question_id, question in questions.items():
|
| 558 |
+
if question.get("type") not in QUESTION_TYPES:
|
| 559 |
+
raise ValueError(f"{question_id}: type must be noul, choice, or score")
|
| 560 |
+
if question["type"] != "noul" and not question.get("criteria"):
|
| 561 |
+
raise ValueError(f"{question_id}: criteria must not be empty")
|
| 562 |
+
encoded = encode_record(processor.tokenizer, request, max_length=max_length, processor=processor)
|
| 563 |
+
device = next(model.parameters()).device
|
| 564 |
+
logits = model(collate_records([encoded], processor.tokenizer.pad_token_id, device))[0]
|
| 565 |
+
answers = {
|
| 566 |
+
question.question_id: systemone_answer(
|
| 567 |
+
questions[question.question_id],
|
| 568 |
+
dict(zip(question.option_ids, question_logits.float().softmax(-1).tolist())),
|
| 569 |
+
)
|
| 570 |
+
for question, question_logits in zip(encoded.questions, logits)
|
| 571 |
+
}
|
| 572 |
+
return {
|
| 573 |
+
"model": request["model"],
|
| 574 |
+
"answers": answers,
|
| 575 |
+
"usage": {"input_tokens": len(encoded.input_ids), "output_tokens": 0},
|
| 576 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7799d5c3a47b8f249d667a040e3fd538ccbf47bd958dc6f2aa52af8346aefa6b
|
| 3 |
+
size 11910387016
|
processor_config.json
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"image_processor": {
|
| 3 |
+
"do_convert_rgb": true,
|
| 4 |
+
"do_normalize": true,
|
| 5 |
+
"do_rescale": true,
|
| 6 |
+
"do_resize": true,
|
| 7 |
+
"image_mean": [
|
| 8 |
+
0.5,
|
| 9 |
+
0.5,
|
| 10 |
+
0.5
|
| 11 |
+
],
|
| 12 |
+
"image_processor_type": "Qwen2VLImageProcessor",
|
| 13 |
+
"image_std": [
|
| 14 |
+
0.5,
|
| 15 |
+
0.5,
|
| 16 |
+
0.5
|
| 17 |
+
],
|
| 18 |
+
"merge_size": 2,
|
| 19 |
+
"patch_size": 16,
|
| 20 |
+
"resample": 3,
|
| 21 |
+
"rescale_factor": 0.00392156862745098,
|
| 22 |
+
"size": {
|
| 23 |
+
"longest_edge": 16777216,
|
| 24 |
+
"shortest_edge": 65536
|
| 25 |
+
},
|
| 26 |
+
"temporal_patch_size": 2
|
| 27 |
+
},
|
| 28 |
+
"processor_class": "Qwen3VLProcessor",
|
| 29 |
+
"video_processor": {
|
| 30 |
+
"do_convert_rgb": true,
|
| 31 |
+
"do_normalize": true,
|
| 32 |
+
"do_rescale": true,
|
| 33 |
+
"do_resize": true,
|
| 34 |
+
"do_sample_frames": true,
|
| 35 |
+
"fps": 2,
|
| 36 |
+
"image_mean": [
|
| 37 |
+
0.5,
|
| 38 |
+
0.5,
|
| 39 |
+
0.5
|
| 40 |
+
],
|
| 41 |
+
"image_std": [
|
| 42 |
+
0.5,
|
| 43 |
+
0.5,
|
| 44 |
+
0.5
|
| 45 |
+
],
|
| 46 |
+
"max_frames": 768,
|
| 47 |
+
"merge_size": 2,
|
| 48 |
+
"min_frames": 4,
|
| 49 |
+
"patch_size": 16,
|
| 50 |
+
"resample": 3,
|
| 51 |
+
"rescale_factor": 0.00392156862745098,
|
| 52 |
+
"return_metadata": false,
|
| 53 |
+
"size": {
|
| 54 |
+
"longest_edge": 25165824,
|
| 55 |
+
"shortest_edge": 4096
|
| 56 |
+
},
|
| 57 |
+
"temporal_patch_size": 2,
|
| 58 |
+
"video_processor_type": "Qwen3VLVideoProcessor"
|
| 59 |
+
}
|
| 60 |
+
}
|
recipe.yaml
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
default_stage:
|
| 2 |
+
default_modifiers:
|
| 3 |
+
QuantizationModifier:
|
| 4 |
+
targets: [Linear]
|
| 5 |
+
ignore: [lm_head, 're:.*visual.*', 're:.*linear_attn.in_proj_a$', 're:.*linear_attn.in_proj_b$']
|
| 6 |
+
scheme: FP8_DYNAMIC
|
| 7 |
+
bypass_divisibility_checks: false
|
| 8 |
+
requires_calibration_data: false
|
scripts/build_data.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Build Clef-format records from public labeled datasets.
|
| 2 |
+
|
| 3 |
+
calib.jsonl — from train splits, used for NVFP4/FP8-static calibration
|
| 4 |
+
eval.jsonl — from test/validation splits, scored against gold labels and BF16
|
| 5 |
+
Each line: {"task", "gold": {question_id: option_id}, "record": {state, questions}}
|
| 6 |
+
"""
|
| 7 |
+
import json
|
| 8 |
+
import random
|
| 9 |
+
|
| 10 |
+
from datasets import load_dataset
|
| 11 |
+
|
| 12 |
+
random.seed(0)
|
| 13 |
+
LETTERS = "ABCDEFGHIJ"
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def banking77(split):
|
| 17 |
+
ds = load_dataset("mteb/banking77", split=split)
|
| 18 |
+
names = sorted(set(ds["label_text"]))
|
| 19 |
+
criteria = {n: n.replace("_", " ") for n in names}
|
| 20 |
+
for r in ds:
|
| 21 |
+
yield {"state": r["text"], "questions": {"intent": {
|
| 22 |
+
"type": "choice", "instructions": "Which banking intent does the customer message express?",
|
| 23 |
+
"criteria": criteria}}}, {"intent": r["label_text"]}
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def arc(split):
|
| 27 |
+
for r in load_dataset("allenai/ai2_arc", "ARC-Challenge", split=split):
|
| 28 |
+
crit = dict(zip(r["choices"]["label"], r["choices"]["text"]))
|
| 29 |
+
if r["answerKey"] not in crit:
|
| 30 |
+
continue
|
| 31 |
+
yield {"state": {"question": r["question"]}, "questions": {"answer": {
|
| 32 |
+
"type": "choice", "instructions": "Which option correctly answers the question?",
|
| 33 |
+
"criteria": crit}}}, {"answer": r["answerKey"]}
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def mmlu(split):
|
| 37 |
+
for r in load_dataset("cais/mmlu", "all", split=split):
|
| 38 |
+
yield {"state": {"subject": r["subject"], "question": r["question"]}, "questions": {"answer": {
|
| 39 |
+
"type": "choice", "instructions": "Which option correctly answers the question?",
|
| 40 |
+
"criteria": dict(zip(LETTERS, r["choices"]))}}}, {"answer": LETTERS[r["answer"]]}
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def hellaswag(split):
|
| 44 |
+
for r in load_dataset("Rowan/hellaswag", split=split):
|
| 45 |
+
if r["label"] == "":
|
| 46 |
+
continue
|
| 47 |
+
yield {"state": {"context": r["ctx"]}, "questions": {"ending": {
|
| 48 |
+
"type": "choice", "instructions": "Which ending most plausibly continues the context?",
|
| 49 |
+
"criteria": dict(zip(LETTERS, r["endings"]))}}}, {"ending": LETTERS[int(r["label"])]}
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def anli(split):
|
| 53 |
+
names = ["entailment", "neutral", "contradiction"]
|
| 54 |
+
for r in load_dataset("facebook/anli", split=split):
|
| 55 |
+
yield {"state": {"premise": r["premise"], "hypothesis": r["hypothesis"]}, "questions": {"relation": {
|
| 56 |
+
"type": "choice", "instructions": "How does the premise relate to the hypothesis?",
|
| 57 |
+
"criteria": {"entailment": "The premise entails the hypothesis.",
|
| 58 |
+
"neutral": "The premise neither entails nor contradicts the hypothesis.",
|
| 59 |
+
"contradiction": "The premise contradicts the hypothesis."}}}}, {"relation": names[r["label"]]}
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def boolq(split):
|
| 63 |
+
for r in load_dataset("google/boolq", split=split):
|
| 64 |
+
yield {"state": {"passage": r["passage"]}, "questions": {"answer": {
|
| 65 |
+
"type": "noul", "instructions": r["question"].rstrip("?") + "?"}}}, {"answer": "true" if r["answer"] else "false"}
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def multi(split):
|
| 69 |
+
"""Multi-question records: ANLI relation + a score + a noul over the same state."""
|
| 70 |
+
names = ["entailment", "neutral", "contradiction"]
|
| 71 |
+
for r in load_dataset("facebook/anli", split=split):
|
| 72 |
+
yield {"state": {"premise": r["premise"], "hypothesis": r["hypothesis"]}, "questions": {
|
| 73 |
+
"relation": {"type": "choice", "instructions": "How does the premise relate to the hypothesis?",
|
| 74 |
+
"criteria": {"entailment": "Entails.", "neutral": "Neither.", "contradiction": "Contradicts."}},
|
| 75 |
+
"supported": {"type": "noul", "instructions": "Does the premise support the hypothesis?"},
|
| 76 |
+
"overlap": {"type": "score", "instructions": "How much topical overlap do premise and hypothesis have?",
|
| 77 |
+
"criteria": ["None", "Some", "A lot"]},
|
| 78 |
+
}}, {"relation": names[r["label"]], "supported": "true" if r["label"] == 0 else "false"}
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
TASKS = {
|
| 82 |
+
"banking77": (banking77, "train", "test"),
|
| 83 |
+
"arc_challenge": (arc, "train", "test"),
|
| 84 |
+
"mmlu": (mmlu, "auxiliary_train", "test"),
|
| 85 |
+
"hellaswag": (hellaswag, "train", "validation"),
|
| 86 |
+
"anli": (anli, "train_r3", "test_r3"),
|
| 87 |
+
"boolq": (boolq, "train", "validation"),
|
| 88 |
+
"multi_anli": (multi, "train_r1", "test_r1"),
|
| 89 |
+
}
|
| 90 |
+
CALIB_PER_TASK, EVAL_PER_TASK = 96, 200
|
| 91 |
+
|
| 92 |
+
for out, idx, n in (("calib.jsonl", 1, CALIB_PER_TASK), ("eval.jsonl", 2, EVAL_PER_TASK)):
|
| 93 |
+
with open(out, "w") as f:
|
| 94 |
+
for task, spec in TASKS.items():
|
| 95 |
+
rows = list(spec[0](spec[idx]))
|
| 96 |
+
random.shuffle(rows)
|
| 97 |
+
for record, gold in rows[:n]:
|
| 98 |
+
f.write(json.dumps({"task": task, "gold": gold, "record": record}) + "\n")
|
| 99 |
+
print(out, task, min(n, len(rows)))
|
scripts/quantize.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Quantize the Clef-Flash backbone with llm-compressor (0.14, transformers 5.17).
|
| 2 |
+
|
| 3 |
+
python build_data.py # writes calib.jsonl / eval.jsonl
|
| 4 |
+
python quantize.py --scheme FP8_DYNAMIC # no calibration needed
|
| 5 |
+
python quantize.py --scheme NVFP4 # calibrates on calib.jsonl
|
| 6 |
+
The joint head is not quantized; copy clef_head/ from the release next to the output.
|
| 7 |
+
"""
|
| 8 |
+
import argparse
|
| 9 |
+
import json
|
| 10 |
+
import shutil
|
| 11 |
+
import sys
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
import torch
|
| 15 |
+
from datasets import Dataset
|
| 16 |
+
from huggingface_hub import snapshot_download
|
| 17 |
+
from llmcompressor import oneshot
|
| 18 |
+
from llmcompressor.modifiers.quantization import QuantizationModifier
|
| 19 |
+
from llmcompressor.observers.helpers import FUSED_LAYER_NAMES
|
| 20 |
+
from transformers import AutoProcessor, Qwen3_5ForConditionalGeneration
|
| 21 |
+
|
| 22 |
+
ap = argparse.ArgumentParser()
|
| 23 |
+
ap.add_argument("--release", default="Cloudflare/clef-flash", help="hub id or local path of the BF16 release")
|
| 24 |
+
ap.add_argument("--scheme", required=True, choices=["FP8_DYNAMIC", "FP8", "NVFP4"])
|
| 25 |
+
ap.add_argument("--calib", default="calib.jsonl")
|
| 26 |
+
ap.add_argument("--num-samples", type=int, default=672)
|
| 27 |
+
ap.add_argument("--pipeline", default="sequential")
|
| 28 |
+
args = ap.parse_args()
|
| 29 |
+
|
| 30 |
+
RELEASE = Path(args.release) if Path(args.release).is_dir() else Path(snapshot_download(args.release))
|
| 31 |
+
sys.path.insert(0, str(RELEASE))
|
| 32 |
+
from joint_schema_model import encode_record # noqa: E402
|
| 33 |
+
|
| 34 |
+
IGNORE = [
|
| 35 |
+
"lm_head", # head reads it as lexical option embeddings
|
| 36 |
+
"re:.*visual.*", # vision tower
|
| 37 |
+
"re:.*linear_attn.in_proj_a$", # GDN gate projections, 32 outputs each
|
| 38 |
+
"re:.*linear_attn.in_proj_b$",
|
| 39 |
+
]
|
| 40 |
+
|
| 41 |
+
# vLLM fuses Qwen3.5 GDN in_proj_qkv + in_proj_z into one in_proj_qkvz GEMM, so NVFP4 needs a
|
| 42 |
+
# shared weight global scale for them (llm-compressor only knows q/k/v and gate/up). Without
|
| 43 |
+
# this vLLM takes the max scale and the qkv weights come out scaled by g_qkv / g_z.
|
| 44 |
+
FUSED_LAYER_NAMES.append(("in_proj_qkv", "in_proj_z"))
|
| 45 |
+
|
| 46 |
+
processor = AutoProcessor.from_pretrained(RELEASE)
|
| 47 |
+
model = Qwen3_5ForConditionalGeneration.from_pretrained(RELEASE, dtype=torch.bfloat16)
|
| 48 |
+
|
| 49 |
+
dataset = None
|
| 50 |
+
if args.scheme != "FP8_DYNAMIC":
|
| 51 |
+
rows = [json.loads(l) for l in open(args.calib)][: args.num_samples]
|
| 52 |
+
ids = [list(encode_record(processor.tokenizer, r["record"], processor=processor).input_ids) for r in rows]
|
| 53 |
+
dataset = Dataset.from_dict({"input_ids": ids, "attention_mask": [[1] * len(x) for x in ids]})
|
| 54 |
+
|
| 55 |
+
oneshot(
|
| 56 |
+
model=model,
|
| 57 |
+
dataset=dataset,
|
| 58 |
+
recipe=QuantizationModifier(targets="Linear", scheme=args.scheme, ignore=IGNORE),
|
| 59 |
+
max_seq_length=16384,
|
| 60 |
+
num_calibration_samples=len(dataset) if dataset is not None else 0,
|
| 61 |
+
pipeline=args.pipeline if dataset is not None else "datafree",
|
| 62 |
+
sequential_targets=["Qwen3_5DecoderLayer"],
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
out = Path("quant") / f"clef-flash-{args.scheme.lower()}"
|
| 66 |
+
model.save_pretrained(out, save_compressed=True)
|
| 67 |
+
for f in ["tokenizer.json", "tokenizer_config.json", "chat_template.jinja", "processor_config.json",
|
| 68 |
+
"generation_config.json"]:
|
| 69 |
+
shutil.copy(RELEASE / f, out / f)
|
| 70 |
+
print("saved", out)
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523
|
| 3 |
+
size 19989325
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"audio_bos_token": "<|audio_start|>",
|
| 4 |
+
"audio_eos_token": "<|audio_end|>",
|
| 5 |
+
"audio_token": "<|audio_pad|>",
|
| 6 |
+
"backend": "tokenizers",
|
| 7 |
+
"bos_token": null,
|
| 8 |
+
"clean_up_tokenization_spaces": false,
|
| 9 |
+
"eos_token": "<|im_end|>",
|
| 10 |
+
"errors": "replace",
|
| 11 |
+
"image_token": "<|image_pad|>",
|
| 12 |
+
"model_max_length": 262144,
|
| 13 |
+
"model_specific_special_tokens": {
|
| 14 |
+
"audio_bos_token": "<|audio_start|>",
|
| 15 |
+
"audio_eos_token": "<|audio_end|>",
|
| 16 |
+
"audio_token": "<|audio_pad|>",
|
| 17 |
+
"image_token": "<|image_pad|>",
|
| 18 |
+
"video_token": "<|video_pad|>",
|
| 19 |
+
"vision_bos_token": "<|vision_start|>",
|
| 20 |
+
"vision_eos_token": "<|vision_end|>"
|
| 21 |
+
},
|
| 22 |
+
"pad_token": "<|endoftext|>",
|
| 23 |
+
"pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
| 24 |
+
"split_special_tokens": false,
|
| 25 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
| 26 |
+
"unk_token": null,
|
| 27 |
+
"video_token": "<|video_pad|>",
|
| 28 |
+
"vision_bos_token": "<|vision_start|>",
|
| 29 |
+
"vision_eos_token": "<|vision_end|>"
|
| 30 |
+
}
|
vllm_plugin/clef_vllm/__init__.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""vLLM plugin for Clef decision models (Cloudflare/clef-flash and its quantizations)."""
|
| 2 |
+
|
| 3 |
+
from clef_vllm.schema import ARCHITECTURE, TASK, schema_layout
|
| 4 |
+
|
| 5 |
+
__all__ = ["ARCHITECTURE", "TASK", "schema_layout", "register"]
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def register() -> None:
|
| 9 |
+
from vllm import ModelRegistry
|
| 10 |
+
|
| 11 |
+
if ARCHITECTURE not in ModelRegistry.get_supported_archs():
|
| 12 |
+
ModelRegistry.register_model(ARCHITECTURE, "clef_vllm.model:ClefFlashForDecision")
|
vllm_plugin/clef_vllm/model.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Clef-Flash as a vLLM pooling model.
|
| 2 |
+
|
| 3 |
+
The Qwen3.5 backbone runs in vLLM; the release's JointSchemaHead runs in the pooler on the
|
| 4 |
+
full prompt's final hidden states. Each request carries its schema layout in
|
| 5 |
+
PoolingParams.extra_kwargs["clef_questions"]: a list of [type_id, [q_start, q_end],
|
| 6 |
+
[[opt_start, opt_end], ...]] built by joint_schema_model.encode_record on the client.
|
| 7 |
+
The output is one flat float32 tensor: every option logit of every question, in order.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import importlib.util
|
| 12 |
+
import json
|
| 13 |
+
import sys
|
| 14 |
+
from collections.abc import Iterable, Set
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
from safetensors.torch import load_file
|
| 19 |
+
from vllm.config import VllmConfig, get_current_vllm_config
|
| 20 |
+
from vllm.distributed import (
|
| 21 |
+
get_tensor_model_parallel_rank,
|
| 22 |
+
get_tensor_model_parallel_world_size,
|
| 23 |
+
tensor_model_parallel_all_gather,
|
| 24 |
+
)
|
| 25 |
+
from vllm.model_executor.layers.pooler import Pooler, PoolingParamsUpdate
|
| 26 |
+
from vllm.model_executor.models.qwen3_5 import (
|
| 27 |
+
Qwen3_5ForConditionalGeneration,
|
| 28 |
+
Qwen3_5ProcessingInfo,
|
| 29 |
+
)
|
| 30 |
+
from vllm.model_executor.models.qwen3_vl import (
|
| 31 |
+
Qwen3VLDummyInputsBuilder,
|
| 32 |
+
Qwen3VLMultiModalProcessor,
|
| 33 |
+
)
|
| 34 |
+
from vllm.multimodal import MULTIMODAL_REGISTRY
|
| 35 |
+
from vllm.tasks import PoolingTask
|
| 36 |
+
from vllm.v1.pool.metadata import PoolingMetadata
|
| 37 |
+
|
| 38 |
+
from clef_vllm.schema import TASK
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _import_release_module(model_dir: Path):
|
| 42 |
+
spec = importlib.util.spec_from_file_location("joint_schema_model", model_dir / "joint_schema_model.py")
|
| 43 |
+
module = importlib.util.module_from_spec(spec)
|
| 44 |
+
sys.modules[spec.name] = module # dataclasses resolve annotations via sys.modules
|
| 45 |
+
spec.loader.exec_module(module)
|
| 46 |
+
return module
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class ClefPooler(Pooler):
|
| 50 |
+
def __init__(self, model: "ClefFlashForDecision") -> None:
|
| 51 |
+
super().__init__()
|
| 52 |
+
# Not a submodule: the head and lm_head are owned (and loaded) by the model.
|
| 53 |
+
self._model = [model]
|
| 54 |
+
scheduler_config = get_current_vllm_config().scheduler_config
|
| 55 |
+
self.enable_chunked_prefill = scheduler_config.enable_chunked_prefill
|
| 56 |
+
|
| 57 |
+
def get_supported_tasks(self) -> Set[PoolingTask]:
|
| 58 |
+
return {TASK}
|
| 59 |
+
|
| 60 |
+
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
| 61 |
+
return PoolingParamsUpdate(requires_token_ids=True)
|
| 62 |
+
|
| 63 |
+
def forward(self, hidden_states: torch.Tensor, pooling_metadata: PoolingMetadata):
|
| 64 |
+
model = self._model[0]
|
| 65 |
+
cursor = pooling_metadata.get_pooling_cursor()
|
| 66 |
+
chunks = list(torch.split(hidden_states, cursor.num_scheduled_tokens_cpu.tolist()))
|
| 67 |
+
if self.enable_chunked_prefill:
|
| 68 |
+
finished = cursor.is_finished().tolist()
|
| 69 |
+
full: list[torch.Tensor | None] = []
|
| 70 |
+
for state, chunk, done in zip(pooling_metadata.pooling_states, chunks, finished):
|
| 71 |
+
state.hidden_states_cache.append(chunk if done and not state.hidden_states_cache else chunk.clone())
|
| 72 |
+
if done:
|
| 73 |
+
cache = state.hidden_states_cache
|
| 74 |
+
full.append(cache[0] if len(cache) == 1 else torch.cat(cache, dim=0))
|
| 75 |
+
state.clean()
|
| 76 |
+
else:
|
| 77 |
+
full.append(None)
|
| 78 |
+
else:
|
| 79 |
+
full = chunks
|
| 80 |
+
|
| 81 |
+
token_ids = pooling_metadata.get_prompt_token_ids_cpu()
|
| 82 |
+
outputs: list[torch.Tensor | None] = []
|
| 83 |
+
for hidden, ids, params in zip(full, token_ids, pooling_metadata.pooling_params):
|
| 84 |
+
if hidden is None:
|
| 85 |
+
outputs.append(None)
|
| 86 |
+
continue
|
| 87 |
+
layout = (params.extra_kwargs or {}).get("clef_questions")
|
| 88 |
+
if layout is None or model.lexical_weight is None:
|
| 89 |
+
# vLLM's profiling dummy run (no schema), or a TP rank that does not pool.
|
| 90 |
+
layout = [[1, [0, 1], [[0, 1], [0, 1]]]]
|
| 91 |
+
if model.lexical_weight is None:
|
| 92 |
+
outputs.append(torch.zeros(2, device=hidden.device))
|
| 93 |
+
continue
|
| 94 |
+
outputs.append(model.score(hidden, ids, layout))
|
| 95 |
+
return outputs
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
@MULTIMODAL_REGISTRY.register_processor(
|
| 99 |
+
Qwen3VLMultiModalProcessor,
|
| 100 |
+
info=Qwen3_5ProcessingInfo,
|
| 101 |
+
dummy_inputs=Qwen3VLDummyInputsBuilder,
|
| 102 |
+
)
|
| 103 |
+
class ClefFlashForDecision(Qwen3_5ForConditionalGeneration):
|
| 104 |
+
is_pooling_model = True
|
| 105 |
+
default_seq_pooling_type = "LAST"
|
| 106 |
+
default_tok_pooling_type = "ALL"
|
| 107 |
+
|
| 108 |
+
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
|
| 109 |
+
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
| 110 |
+
self.model_dir = Path(vllm_config.model_config.model)
|
| 111 |
+
self.release = _import_release_module(self.model_dir)
|
| 112 |
+
head_config = json.loads((self.model_dir / "clef_head" / "joint_head_config.json").read_text())
|
| 113 |
+
self.head = self.release.JointSchemaHead(**head_config)
|
| 114 |
+
self.pooler = ClefPooler(self)
|
| 115 |
+
|
| 116 |
+
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
| 117 |
+
loaded = super().load_weights(weights)
|
| 118 |
+
state = load_file(self.model_dir / "clef_head" / "joint_head.safetensors")
|
| 119 |
+
self.head.load_state_dict(state, strict=True)
|
| 120 |
+
lm_head = self.language_model.lm_head.weight
|
| 121 |
+
self.head.to(device=lm_head.device, dtype=lm_head.dtype).eval()
|
| 122 |
+
# The head indexes lm_head rows by token id; under TP the weight is vocab-sharded,
|
| 123 |
+
# so gather it once. The pooler only runs on TP rank 0, so only rank 0 keeps it.
|
| 124 |
+
self.lexical_weight = lm_head.data # plain tensor, not a registered Parameter
|
| 125 |
+
if get_tensor_model_parallel_world_size() > 1:
|
| 126 |
+
gathered = tensor_model_parallel_all_gather(lm_head.data.contiguous(), dim=0)
|
| 127 |
+
self.lexical_weight = gathered if get_tensor_model_parallel_rank() == 0 else None
|
| 128 |
+
return loaded | {f"head.{name}" for name in state}
|
| 129 |
+
|
| 130 |
+
@torch.inference_mode()
|
| 131 |
+
def score(self, hidden: torch.Tensor, ids: torch.Tensor, layout: list) -> torch.Tensor:
|
| 132 |
+
release = self.release
|
| 133 |
+
questions = tuple(
|
| 134 |
+
release.EncodedQuestion(
|
| 135 |
+
question_id=str(index),
|
| 136 |
+
question_type=int(type_id),
|
| 137 |
+
question_span=tuple(question_span),
|
| 138 |
+
option_spans=tuple(tuple(span) for span in option_spans),
|
| 139 |
+
option_ids=tuple(str(i) for i in range(len(option_spans))),
|
| 140 |
+
)
|
| 141 |
+
for index, (type_id, question_span, option_spans) in enumerate(layout)
|
| 142 |
+
)
|
| 143 |
+
record = release.EncodedRecord(input_ids=tuple(), questions=questions, record_id="vllm")
|
| 144 |
+
device = hidden.device
|
| 145 |
+
length = hidden.shape[0]
|
| 146 |
+
logits = self.head(
|
| 147 |
+
hidden.unsqueeze(0),
|
| 148 |
+
ids.to(device).unsqueeze(0),
|
| 149 |
+
torch.ones((1, length), dtype=torch.long, device=device),
|
| 150 |
+
[record],
|
| 151 |
+
self.lexical_weight,
|
| 152 |
+
)[0]
|
| 153 |
+
return torch.cat([question_logits.float() for question_logits in logits])
|
vllm_plugin/clef_vllm/schema.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Client-side constants and helpers. No vLLM imports, so clients can use them cheaply."""
|
| 2 |
+
|
| 3 |
+
ARCHITECTURE = "ClefFlashForDecision"
|
| 4 |
+
TASK = "token_classify"
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def schema_layout(encoded) -> list:
|
| 8 |
+
"""PoolingParams.extra_kwargs["clef_questions"] for an EncodedRecord from encode_record."""
|
| 9 |
+
return [[q.question_type, list(q.question_span), [list(s) for s in q.option_spans]] for q in encoded.questions]
|
vllm_plugin/clef_vllm/server.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Jev/SystemOne-compatible HTTP API for Clef models on vLLM.
|
| 2 |
+
|
| 3 |
+
POST /v1/systemone same request/response bodies as joint_schema_model.systemone()
|
| 4 |
+
GET /v1/models, /health
|
| 5 |
+
|
| 6 |
+
Requests are encoded with the release's encode_record and scored by vLLM, which batches
|
| 7 |
+
concurrent requests continuously. --dp N runs one engine per GPU behind one port.
|
| 8 |
+
|
| 9 |
+
clef-systemone --model ./clef-flash-NVFP4 --dp 2 --port 8000
|
| 10 |
+
"""
|
| 11 |
+
import argparse
|
| 12 |
+
import asyncio
|
| 13 |
+
import importlib.util
|
| 14 |
+
import sys
|
| 15 |
+
import uuid
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
import uvicorn
|
| 19 |
+
from fastapi import FastAPI, HTTPException
|
| 20 |
+
from fastapi.responses import JSONResponse
|
| 21 |
+
from transformers import AutoTokenizer
|
| 22 |
+
from vllm import PoolingParams
|
| 23 |
+
from vllm.engine.arg_utils import AsyncEngineArgs
|
| 24 |
+
from vllm.v1.engine.async_llm import AsyncLLM
|
| 25 |
+
|
| 26 |
+
from clef_vllm.schema import ARCHITECTURE, TASK, schema_layout
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def parse_args(argv=None):
|
| 30 |
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
| 31 |
+
ap.add_argument("--model", required=True, help="local path of a Clef checkpoint (snapshot directory)")
|
| 32 |
+
ap.add_argument("--served-model-name", default="clef-flash")
|
| 33 |
+
ap.add_argument("--host", default="0.0.0.0")
|
| 34 |
+
ap.add_argument("--port", type=int, default=8000)
|
| 35 |
+
ap.add_argument("--dp", type=int, default=1, help="data-parallel engines, one per GPU")
|
| 36 |
+
ap.add_argument("--tp", type=int, default=1, help="tensor-parallel GPUs per engine")
|
| 37 |
+
ap.add_argument("--max-model-len", type=int, default=16384)
|
| 38 |
+
ap.add_argument("--max-num-seqs", type=int, default=64)
|
| 39 |
+
ap.add_argument("--max-num-batched-tokens", type=int, default=16384)
|
| 40 |
+
ap.add_argument("--gpu-memory-utilization", type=float, default=0.90)
|
| 41 |
+
ap.add_argument("--no-warmup", action="store_true",
|
| 42 |
+
help="skip the startup warm-up (first requests then pay kernel JIT/autotune time)")
|
| 43 |
+
return ap.parse_args(argv)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _release_module(model_dir: Path):
|
| 47 |
+
spec = importlib.util.spec_from_file_location("joint_schema_model", model_dir / "joint_schema_model.py")
|
| 48 |
+
module = importlib.util.module_from_spec(spec)
|
| 49 |
+
sys.modules[spec.name] = module
|
| 50 |
+
spec.loader.exec_module(module)
|
| 51 |
+
return module
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def create_app(args) -> FastAPI:
|
| 55 |
+
model_dir = Path(args.model)
|
| 56 |
+
release = _release_module(model_dir)
|
| 57 |
+
tokenizer = AutoTokenizer.from_pretrained(model_dir)
|
| 58 |
+
app = FastAPI(title="Clef SystemOne API")
|
| 59 |
+
state = {"engine": None, "ready": False}
|
| 60 |
+
|
| 61 |
+
async def score(encoded):
|
| 62 |
+
params = PoolingParams(task=TASK, extra_kwargs={"clef_questions": schema_layout(encoded)})
|
| 63 |
+
final = None
|
| 64 |
+
async for output in state["engine"].encode(
|
| 65 |
+
{"prompt_token_ids": list(encoded.input_ids)}, params, f"so-{uuid.uuid4().hex}"
|
| 66 |
+
):
|
| 67 |
+
final = output
|
| 68 |
+
if final is None:
|
| 69 |
+
raise HTTPException(500, "engine returned no output")
|
| 70 |
+
return final.outputs.data.float()
|
| 71 |
+
|
| 72 |
+
async def warmup() -> None:
|
| 73 |
+
"""Run batches of several sizes so kernel JIT/autotuning happens before serving."""
|
| 74 |
+
question = {"type": "choice", "instructions": "Pick one.", "criteria": {"a": "First.", "b": "Second."}}
|
| 75 |
+
for words, concurrent in ((50, 1), (400, 8), (1500, 32), (6000, 4), (200, args.max_num_seqs)):
|
| 76 |
+
record = {"state": "warm up " * words, "questions": {"q": question, "n": {"type": "noul"}}}
|
| 77 |
+
encoded = release.encode_record(tokenizer, record, args.max_model_len)
|
| 78 |
+
await asyncio.gather(*(score(encoded) for _ in range(concurrent)))
|
| 79 |
+
|
| 80 |
+
def validate(request: dict) -> None:
|
| 81 |
+
"""The checks of joint_schema_model.systemone(), plus text-only input."""
|
| 82 |
+
questions = request.get("questions")
|
| 83 |
+
if not isinstance(request.get("model"), str) or "state" not in request:
|
| 84 |
+
raise ValueError("model and state are required")
|
| 85 |
+
if not isinstance(questions, dict) or not questions:
|
| 86 |
+
raise ValueError("at least one question is required")
|
| 87 |
+
for question_id, question in questions.items():
|
| 88 |
+
if not isinstance(question, dict) or question.get("type") not in release.QUESTION_TYPES:
|
| 89 |
+
raise ValueError(f"{question_id}: type must be noul, choice, or score")
|
| 90 |
+
if question["type"] != "noul" and not question.get("criteria"):
|
| 91 |
+
raise ValueError(f"{question_id}: criteria must not be empty")
|
| 92 |
+
if request.get("images") or request.get("videos"):
|
| 93 |
+
raise ValueError("images and videos are not supported by this server (text and JSON state only)")
|
| 94 |
+
|
| 95 |
+
@app.on_event("startup")
|
| 96 |
+
async def start_engine() -> None:
|
| 97 |
+
state["engine"] = AsyncLLM.from_engine_args(AsyncEngineArgs(
|
| 98 |
+
model=str(model_dir),
|
| 99 |
+
hf_overrides={"architectures": [ARCHITECTURE]},
|
| 100 |
+
runner="pooling",
|
| 101 |
+
tensor_parallel_size=args.tp,
|
| 102 |
+
data_parallel_size=args.dp,
|
| 103 |
+
enable_prefix_caching=False,
|
| 104 |
+
language_model_only=True,
|
| 105 |
+
max_model_len=args.max_model_len,
|
| 106 |
+
max_num_seqs=args.max_num_seqs,
|
| 107 |
+
max_num_batched_tokens=args.max_num_batched_tokens,
|
| 108 |
+
gpu_memory_utilization=args.gpu_memory_utilization,
|
| 109 |
+
disable_log_stats=True,
|
| 110 |
+
))
|
| 111 |
+
if not args.no_warmup:
|
| 112 |
+
await warmup()
|
| 113 |
+
state["ready"] = True
|
| 114 |
+
|
| 115 |
+
@app.on_event("shutdown")
|
| 116 |
+
async def stop_engine() -> None:
|
| 117 |
+
if state["engine"] is not None:
|
| 118 |
+
state["engine"].shutdown()
|
| 119 |
+
|
| 120 |
+
@app.post("/v1/systemone")
|
| 121 |
+
async def systemone(request: dict):
|
| 122 |
+
try:
|
| 123 |
+
validate(request)
|
| 124 |
+
encoded = await asyncio.to_thread(release.encode_record, tokenizer, request, args.max_model_len)
|
| 125 |
+
except (ValueError, TypeError, KeyError) as exc:
|
| 126 |
+
return JSONResponse(status_code=400, content={"error": {"message": str(exc), "type": "invalid_request_error"}})
|
| 127 |
+
|
| 128 |
+
flat = await score(encoded)
|
| 129 |
+
answers, offset = {}, 0
|
| 130 |
+
for question in encoded.questions:
|
| 131 |
+
count = len(question.option_ids)
|
| 132 |
+
probabilities = flat[offset:offset + count].softmax(-1).tolist()
|
| 133 |
+
offset += count
|
| 134 |
+
answers[question.question_id] = release.systemone_answer(
|
| 135 |
+
request["questions"][question.question_id], dict(zip(question.option_ids, probabilities)))
|
| 136 |
+
return {
|
| 137 |
+
"model": request["model"],
|
| 138 |
+
"answers": answers,
|
| 139 |
+
"usage": {"input_tokens": len(encoded.input_ids), "output_tokens": 0},
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
@app.get("/v1/models")
|
| 143 |
+
async def models():
|
| 144 |
+
return {"object": "list", "data": [{"id": args.served_model_name, "object": "model", "owned_by": "clef"}]}
|
| 145 |
+
|
| 146 |
+
@app.get("/health")
|
| 147 |
+
async def health():
|
| 148 |
+
engine = state["engine"]
|
| 149 |
+
if engine is None or engine.errored or not state["ready"]:
|
| 150 |
+
raise HTTPException(503, "engine not ready")
|
| 151 |
+
return {"status": "ok"}
|
| 152 |
+
|
| 153 |
+
return app
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def main(argv=None) -> None:
|
| 157 |
+
args = parse_args(argv)
|
| 158 |
+
uvicorn.run(create_app(args), host=args.host, port=args.port, log_level="info")
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
if __name__ == "__main__":
|
| 162 |
+
main()
|
vllm_plugin/pyproject.toml
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "clef-vllm"
|
| 3 |
+
version = "0.1.0"
|
| 4 |
+
description = "vLLM plugin + Jev/SystemOne API server for Clef decision models"
|
| 5 |
+
license = "Apache-2.0"
|
| 6 |
+
requires-python = ">=3.10"
|
| 7 |
+
dependencies = ["vllm>=0.28,<0.29", "fastapi", "uvicorn"]
|
| 8 |
+
|
| 9 |
+
[project.scripts]
|
| 10 |
+
clef-systemone = "clef_vllm.server:main"
|
| 11 |
+
|
| 12 |
+
[project.entry-points."vllm.general_plugins"]
|
| 13 |
+
clef = "clef_vllm:register"
|
| 14 |
+
|
| 15 |
+
[build-system]
|
| 16 |
+
requires = ["setuptools>=68"]
|
| 17 |
+
build-backend = "setuptools.build_meta"
|
| 18 |
+
|
| 19 |
+
[tool.setuptools]
|
| 20 |
+
packages = ["clef_vllm"]
|