Add validated MLX 4-bit MetaEncoder quantization
Browse files- .gitattributes +1 -0
- LICENSE +202 -0
- NOTICE +60 -0
- README.md +78 -0
- USAGE_POLICY.md +46 -0
- chat_template.jinja +211 -0
- config.json +236 -0
- conversion.json +12 -0
- convert_metaencoder.py +131 -0
- generation_config.json +14 -0
- metaencoder_mlx.py +132 -0
- model-00001-of-00010.safetensors +3 -0
- model-00002-of-00010.safetensors +3 -0
- model-00003-of-00010.safetensors +3 -0
- model-00004-of-00010.safetensors +3 -0
- model-00005-of-00010.safetensors +3 -0
- model-00006-of-00010.safetensors +3 -0
- model-00007-of-00010.safetensors +3 -0
- model-00008-of-00010.safetensors +3 -0
- model-00009-of-00010.safetensors +3 -0
- model-00010-of-00010.safetensors +3 -0
- model.safetensors.index.json +0 -0
- processor_config.json +53 -0
- requirements.txt +65 -0
- source-sha256.json +4 -0
- tokenizer.json +3 -0
- tokenizer_config.json +2104 -0
- upstream_metaencoder.py +369 -0
- validate_metaencoder.py +57 -0
- validation.json +104 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
NOTICE
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MetaEncoder-30B
|
| 2 |
+
===============
|
| 3 |
+
|
| 4 |
+
This work is a derivative of Muse Glimmer-30B, (c) Meta Platforms, Inc., obtained from
|
| 5 |
+
https://huggingface.co/meta-models/Muse-Glimmer-30B and licensed under the Apache License,
|
| 6 |
+
Version 2.0. A copy of that license is distributed with this work as LICENSE.
|
| 7 |
+
|
| 8 |
+
Trademarks
|
| 9 |
+
----------
|
| 10 |
+
|
| 11 |
+
The Apache License, Version 2.0 grants no trademark rights (Section 6). "Meta", "Muse", and
|
| 12 |
+
"Muse Glimmer" are trademarks of Meta Platforms, Inc. The name "MetaEncoder" is used here only
|
| 13 |
+
to identify this derivative work and does not indicate that it is an official Meta product or
|
| 14 |
+
that Meta Platforms, Inc. endorses it.
|
| 15 |
+
|
| 16 |
+
Statement of changes (Apache License 2.0, Section 4(b))
|
| 17 |
+
-------------------------------------------------------
|
| 18 |
+
|
| 19 |
+
The following files differ from the corresponding files in the base model:
|
| 20 |
+
|
| 21 |
+
model-00001-of-00002.safetensors
|
| 22 |
+
model-00002-of-00002.safetensors
|
| 23 |
+
model.safetensors.index.json
|
| 24 |
+
The weights of the language tower have been modified by contrastive fine-tuning for
|
| 25 |
+
embedding extraction. A LoRA adapter (rank 128, alpha 256) trained on the
|
| 26 |
+
self_attn.{q,k,v,o,gate}_proj and mlp.{gate,up,down}_proj projections of all 52 language
|
| 27 |
+
layers was merged into the base weights. The perception encoder, token embeddings,
|
| 28 |
+
lm_head, and all normalisation layers are unmodified and remain bit-identical to the
|
| 29 |
+
base model.
|
| 30 |
+
|
| 31 |
+
config.json
|
| 32 |
+
Regenerated on save. Differs from the base only in recorded token ids, dtype fields,
|
| 33 |
+
use_cache, and the transformers version string.
|
| 34 |
+
|
| 35 |
+
processor_config.json
|
| 36 |
+
max_image_tokens reduced from 4096 to 1280 to match the budget the model was trained
|
| 37 |
+
under.
|
| 38 |
+
|
| 39 |
+
README.md
|
| 40 |
+
Replaced with a model card for this derivative.
|
| 41 |
+
|
| 42 |
+
The following files are new in this work and are not part of the base model:
|
| 43 |
+
|
| 44 |
+
modeling_metaencoder.py
|
| 45 |
+
Inference wrapper.
|
| 46 |
+
|
| 47 |
+
USAGE.md
|
| 48 |
+
Usage guide for this derivative.
|
| 49 |
+
|
| 50 |
+
requirements.txt
|
| 51 |
+
.gitattributes
|
| 52 |
+
Packaging metadata for this derivative.
|
| 53 |
+
|
| 54 |
+
NOTICE
|
| 55 |
+
This file.
|
| 56 |
+
|
| 57 |
+
All other files (tokenizer.json, tokenizer_config.json, chat_template.jinja,
|
| 58 |
+
generation_config.json, LICENSE, USAGE_POLICY.md) are unmodified copies from the base model.
|
| 59 |
+
|
| 60 |
+
MLX conversion: edgefloor, 2026-10-09. Language weights quantized to affine 4-bit/group-64; vision weights preserved. Added MLX inference and conversion scripts.
|
README.md
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
base_model: facebook/meta-encoder
|
| 4 |
+
base_model_relation: quantized
|
| 5 |
+
library_name: mlx
|
| 6 |
+
pipeline_tag: feature-extraction
|
| 7 |
+
tags:
|
| 8 |
+
- mlx
|
| 9 |
+
- mlx-vlm
|
| 10 |
+
- muse_glimmer
|
| 11 |
+
- metaencoder
|
| 12 |
+
- multimodal
|
| 13 |
+
- embeddings
|
| 14 |
+
- quantized
|
| 15 |
+
- 4-bit
|
| 16 |
+
---
|
| 17 |
+
|
| 18 |
+
# MetaEncoder MLX 4-bit
|
| 19 |
+
|
| 20 |
+
An MLX quantization of [facebook/meta-encoder](https://huggingface.co/facebook/meta-encoder), a 30B multimodal encoder that ranks candidates against a natural-language task.
|
| 21 |
+
|
| 22 |
+
Converted from source revision `3d0df704aa5bf66d9c49da1393181f33e893e5f7`. Language linear layers, token embeddings and the language head use 4-bit affine quantization with group size 64. The vision tower, vision adapter and vision projection retain their original BF16 weights. Normalization weights also retain BF16. Weight files total 19.51 GB, or 18.17 GiB.
|
| 23 |
+
|
| 24 |
+
## Use on Apple silicon
|
| 25 |
+
|
| 26 |
+
Download the repository and install the tested dependencies:
|
| 27 |
+
|
| 28 |
+
```bash
|
| 29 |
+
hf download edgefloor/meta-encoder-mlx-4bit --local-dir meta-encoder-mlx-4bit
|
| 30 |
+
cd meta-encoder-mlx-4bit
|
| 31 |
+
python -m pip install -r requirements.txt
|
| 32 |
+
python metaencoder_mlx.py --model . --task "Which animal says meow?" --candidates cat dog horse
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
For embeddings and candidate ranking:
|
| 36 |
+
|
| 37 |
+
```python
|
| 38 |
+
from metaencoder_mlx import MetaEncoderMLX
|
| 39 |
+
|
| 40 |
+
model = MetaEncoderMLX(".")
|
| 41 |
+
embeddings = model.encode(["cat", "dog", "horse"])
|
| 42 |
+
matches = model.match("Which animal says meow?", ["cat", "dog", "horse"])
|
| 43 |
+
scores = model.score("Which animal says meow?", embeddings)
|
| 44 |
+
```
|
| 45 |
+
|
| 46 |
+
Text, PIL images and dictionaries with `text`, `instruction`, `image`, `images` or `video` are accepted. A video is a list of pre-sampled PIL frames at 2 FPS, matching the source wrapper. Each item is encoded individually to avoid padding entering MLX-VLM's decoder attention masks. Outputs are L2-normalized float32 vectors with 6,656 dimensions. Scores are inner products of these vectors.
|
| 47 |
+
|
| 48 |
+
```python
|
| 49 |
+
from PIL import Image
|
| 50 |
+
|
| 51 |
+
matches = model.match(
|
| 52 |
+
{"text": "Which description matches the image?", "image": Image.open("photo.jpg")},
|
| 53 |
+
["a red bicycle", "a blue car"],
|
| 54 |
+
)
|
| 55 |
+
```
|
| 56 |
+
|
| 57 |
+
The included runner uses the original Transformers processor and the source wrapper's prompt construction. It pools the last token's final hidden state, avoids vocabulary logits and rounds rotary frequencies to BF16 as the source MetaEncoder wrapper does. PyTorch is used for processing support; model inference runs through MLX. The original PyTorch wrapper cannot directly load these quantized weights.
|
| 58 |
+
|
| 59 |
+
## Local validation
|
| 60 |
+
|
| 61 |
+
The full checkpoint loaded with MLX-VLM's strict weight validation. Text, image and two-frame video smoke checks returned finite unit-length embeddings. The model selected the expected first candidate in 4 of 4 simple checks.
|
| 62 |
+
|
| 63 |
+
One text-task embedding had cosine similarity 0.972983 to a streamed BF16 MLX reference from the original weights. This checks quantization drift for that input; it is not a benchmark. Full-model parity with the official Transformers implementation was not measured. See `validation.json` for prompts, scores, timings and memory measurements.
|
| 64 |
+
|
| 65 |
+
A separate small-architecture test verified that every saved quantized tensor exactly matched `mlx.core.quantize`, that unquantized tensors were preserved exactly and that both variants reloaded strictly. A small unquantized architecture also exceeded 0.999 cosine agreement with Transformers on text hidden states and image features.
|
| 66 |
+
|
| 67 |
+
## Reproduce
|
| 68 |
+
|
| 69 |
+
```bash
|
| 70 |
+
hf download facebook/meta-encoder --revision 3d0df704aa5bf66d9c49da1393181f33e893e5f7 --local-dir source
|
| 71 |
+
python convert_metaencoder.py --source source --output converted --revision 3d0df704aa5bf66d9c49da1393181f33e893e5f7
|
| 72 |
+
```
|
| 73 |
+
|
| 74 |
+
The converter writes both 4-bit and 8-bit variants incrementally and validates the source tensor names and shapes against MLX-VLM's Muse Glimmer architecture. Conversion metadata is in `conversion.json`.
|
| 75 |
+
|
| 76 |
+
## License
|
| 77 |
+
|
| 78 |
+
The source model is Apache-2.0 and remains subject to its base model usage policy. `LICENSE`, `NOTICE` and `USAGE_POLICY.md` are preserved. This conversion changes weight storage and adds an MLX inference wrapper; it does not fine-tune the model. The upstream PyTorch helper source is preserved as `upstream_metaencoder.py`.
|
USAGE_POLICY.md
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Muse Glimmer Usage Policy
|
| 2 |
+
|
| 3 |
+
Meta is committed to promoting safe and fair use of its tools and features, including Muse Glimmer. This Usage Policy (“Policy”) applies to your access or use of Muse Glimmer. The most recent copy of this policy can be found at: [https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/main/USAGE\_POLICY](https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/main/USAGE_POLICY.md)
|
| 4 |
+
|
| 5 |
+
## Prohibited Uses
|
| 6 |
+
|
| 7 |
+
We want everyone to use Muse Glimmer safely and responsibly. Muse Glimmer is not intended for individuals under the age of 18\.
|
| 8 |
+
|
| 9 |
+
You should not use, or allow others to use, Muse Glimmer to:
|
| 10 |
+
|
| 11 |
+
1. **Violate the law or others' rights, including to:**
|
| 12 |
+
1. Engage in, promote, generate, contribute to, encourage, plan, incite, or further illegal or unlawful activity or content, such as:
|
| 13 |
+
1. Violence or terrorism
|
| 14 |
+
2. Exploitation or harm to children, including the solicitation, creation, acquisition, or dissemination of child exploitative content or failure to report Child Sexual Abuse Material
|
| 15 |
+
3. Human trafficking, exploitation, and sexual violence
|
| 16 |
+
4. The illegal distribution of information or materials to minors, including obscene materials, or failure to employ legally required age-gating in connection with such information or materials
|
| 17 |
+
5. Sexual solicitation
|
| 18 |
+
6. Any other criminal activity
|
| 19 |
+
2. Engage in, promote, incite, or facilitate the harassment, abuse, threatening, or bullying of individuals or groups of individuals
|
| 20 |
+
3. Engage in, promote, incite, or facilitate discrimination or other unlawful or harmful conduct in the provision of employment, employment benefits, credit, housing, other economic benefits, or other essential goods and services
|
| 21 |
+
4. Engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or related professional practices
|
| 22 |
+
5. Collect, process, disclose, generate, or infer private or sensitive information about individuals, including information about individuals' identity, health, or demographic information, unless you have obtained the right to do so in accordance with applicable law
|
| 23 |
+
6. Engage in or facilitate any action or generate any content that infringes, misappropriates, or otherwise violates any third-party rights, including the outputs or results of any products or services using Muse Glimmer
|
| 24 |
+
7. Create, generate, or facilitate the creation of malicious code, malware, computer viruses or do anything else that could disable, overburden, interfere with or impair the proper working, integrity, operation or appearance of a website or computer system
|
| 25 |
+
8. Engage in any action, or facilitate any action, to intentionally circumvent or remove usage restrictions or other safety measures, or to enable functionality disabled by Meta
|
| 26 |
+
2. **Engage in, promote, incite, facilitate, or assist in the planning or development of activities that present a risk of death or bodily harm to individuals, including use of Muse Glimmer related to the following:**
|
| 27 |
+
1. Military, warfare, nuclear industries or applications, espionage, use for materials or activities that are subject to the International Traffic Arms Regulations (ITAR) maintained by the United States Department of State or to the U.S. Biological Weapons Anti-Terrorism Act of 1989 or the Chemical Weapons Convention Implementation Act of 1997
|
| 28 |
+
2. Guns and illegal weapons (including weapon development)
|
| 29 |
+
3. Illegal drugs and regulated/controlled substances
|
| 30 |
+
4. Operation of critical infrastructure, transportation technologies, or heavy machinery
|
| 31 |
+
5. Self-harm or harm to others, including suicide, cutting, and eating disorders
|
| 32 |
+
6. Any content intended to incite or promote violence, abuse, or any infliction of bodily harm to an individual
|
| 33 |
+
3. **Intentionally deceive or mislead others, including use of Muse Glimmer related to the following:**
|
| 34 |
+
1. Generating, promoting, or furthering fraud or the creation or promotion of disinformation
|
| 35 |
+
2. Generating, promoting, or furthering defamatory content, including the creation of defamatory statements, images, or other content
|
| 36 |
+
3. Generating, promoting, or further distributing spam
|
| 37 |
+
4. Impersonating another individual without consent, authorization, or legal right
|
| 38 |
+
5. Representing that the use of Muse Glimmer or outputs are human generated
|
| 39 |
+
6. Generating or facilitating false online engagement, including fake reviews and other means of fake online engagement
|
| 40 |
+
4. **Fail to appropriately disclose to end users any known dangers of your AI system**
|
| 41 |
+
5. **Interact with third party tools, models, or software designed to generate unlawful content or engage in unlawful or harmful conduct and/or represent that the outputs of such tools, models, or software are associated with Meta or Muse Glimmer**
|
| 42 |
+
|
| 43 |
+
Please report any violation of this Policy, software "bug," or other problems that could lead to a violation of this Policy through one of the following means:
|
| 44 |
+
|
| 45 |
+
* Reporting issues with the model: [https://huggingface.co/meta-models/](https://huggingface.co/meta-models/)
|
| 46 |
+
* Reporting bugs and security concerns: [https://facebook.com/whitehat/info](https://facebook.com/whitehat/info)
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{#
|
| 2 |
+
Template: Muse Glimmer ATEM Chat Template
|
| 3 |
+
Renders the ATEM tool-calling protocol: reasoning channel (to=self), tool
|
| 4 |
+
channels (to=<tool>), and the user channel, plus tool definitions and the
|
| 5 |
+
valid-recipient list in the system block.
|
| 6 |
+
|
| 7 |
+
Whitespace note: every tag uses the {%- -%} / {{- -}} stripping markers, so
|
| 8 |
+
the indentation below is purely for readability and contributes nothing to
|
| 9 |
+
the rendered output.
|
| 10 |
+
#}
|
| 11 |
+
{%- macro render_content(content) -%}
|
| 12 |
+
{%- if content is string -%}
|
| 13 |
+
{{- content -}}
|
| 14 |
+
{%- elif content is not none -%}
|
| 15 |
+
{%- for part in content -%}
|
| 16 |
+
{%- if part['type'] == 'image' -%}
|
| 17 |
+
{{- '<|patch|>' -}}
|
| 18 |
+
{%- elif part['type'] == 'video' -%}
|
| 19 |
+
{{- '<|video|>' -}}
|
| 20 |
+
{%- elif part['type'] == 'text' -%}
|
| 21 |
+
{{- part['text'] -}}
|
| 22 |
+
{%- endif -%}
|
| 23 |
+
{%- endfor -%}
|
| 24 |
+
{%- endif -%}
|
| 25 |
+
{%- endmacro -%}
|
| 26 |
+
{%- macro render_atem(tc) -%}
|
| 27 |
+
{%- set args = tc.function.arguments -%}
|
| 28 |
+
{%- if args is not mapping -%}
|
| 29 |
+
{{- raise_exception('Muse Glimmer ATEM chat template requires tool_call.function.arguments to be a dict (mapping); a JSON string cannot be parsed in the HF jinja sandbox.') -}}
|
| 30 |
+
{%- endif -%}
|
| 31 |
+
{{- '<atem:function_calls>\n<atem:invoke name="' + tc.function.name + '">\n' -}}
|
| 32 |
+
{%- for k, v in args.items() -%}
|
| 33 |
+
{{- '<atem:parameter name="' + k + '">' -}}
|
| 34 |
+
{%- if v is boolean -%}
|
| 35 |
+
{%- if v -%}
|
| 36 |
+
true
|
| 37 |
+
{%- else -%}
|
| 38 |
+
false
|
| 39 |
+
{%- endif -%}
|
| 40 |
+
{%- elif v is none -%}
|
| 41 |
+
null
|
| 42 |
+
{%- elif v is mapping or (v is iterable and v is not string) -%}
|
| 43 |
+
{{- v | tojson -}}
|
| 44 |
+
{%- else -%}
|
| 45 |
+
{{- v -}}
|
| 46 |
+
{%- endif -%}
|
| 47 |
+
{{- '</atem:parameter>\n' -}}
|
| 48 |
+
{%- endfor -%}
|
| 49 |
+
{{- '</atem:invoke>\n</atem:function_calls>' -}}
|
| 50 |
+
{%- endmacro -%}
|
| 51 |
+
{%- macro render_tool_defs(tools) -%}
|
| 52 |
+
{{- 'In this environment you have access to a set of tools you can use to answer the user\'s question.\n\n' -}}
|
| 53 |
+
{{- 'You can invoke a function by writing a "<atem:function_calls>" block like the following:\n' -}}
|
| 54 |
+
{{- '<atem:function_calls>\n<atem:invoke name="$FUNCTION_NAME">\n<atem:parameter name="$PARAMETER_NAME">$PARAMETER_VALUE</atem:parameter>\n...\n</atem:invoke>\n</atem:function_calls>\n\n' -}}
|
| 55 |
+
{{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\n' -}}
|
| 56 |
+
{{- 'Here are the functions available in JSONSchema format:\n' -}}
|
| 57 |
+
{{- '// Tool metadata\n' -}}
|
| 58 |
+
{%- set nsns = namespace(seen=[]) -%}
|
| 59 |
+
{%- for tool in tools -%}
|
| 60 |
+
{%- set fn = tool.function if tool.function is defined else tool -%}
|
| 61 |
+
{%- set tns = fn.name.split('.')[0] -%}
|
| 62 |
+
{%- if tns not in nsns.seen -%}
|
| 63 |
+
{%- set nsns.seen = nsns.seen + [tns] -%}
|
| 64 |
+
{%- endif -%}
|
| 65 |
+
{%- endfor -%}
|
| 66 |
+
{%- set nd = tool_namespace_descriptions if tool_namespace_descriptions is defined else {} -%}
|
| 67 |
+
{%- for tns in nsns.seen -%}
|
| 68 |
+
{{- '{"name": ' + (tns | tojson) + ', "description": ' + ((nd[tns] if tns in nd else '') | tojson) + '}\n' -}}
|
| 69 |
+
{%- endfor -%}
|
| 70 |
+
{{- '// Function schemas' -}}
|
| 71 |
+
{%- for tool in tools -%}
|
| 72 |
+
{%- set fn = tool.function if tool.function is defined else tool -%}
|
| 73 |
+
{{- '\n{"name": ' + (fn.name | tojson) + ', "description": ' + (fn.description | tojson) + ', "parameters": ' + (fn.parameters | tojson) + '}' -}}
|
| 74 |
+
{%- endfor -%}
|
| 75 |
+
{{- '\n\nHere\'s an example of how to call a function in the tool set:\n' -}}
|
| 76 |
+
{{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name` rather than `example_tool_name.example_function_name`)\n\n' -}}
|
| 77 |
+
{{- 'to=example_tool_name.example_function_name\n\n' -}}
|
| 78 |
+
{{- '<atem:function_calls>\n<atem:invoke name="example_tool_name.example_function_name">\n' -}}
|
| 79 |
+
{{- '<atem:parameter name="example_parameter_1">value_1</atem:parameter>\n' -}}
|
| 80 |
+
{{- '<atem:parameter name="example_parameter_2">This is the value for the second parameter\nthat can span\n"multiple" lines\n</atem:parameter>\n' -}}
|
| 81 |
+
{{- '</atem:invoke>\n</atem:function_calls>' -}}
|
| 82 |
+
{%- endmacro -%}
|
| 83 |
+
{%- macro render_reasoning() -%}
|
| 84 |
+
{%- set rs = reasoning_strength if reasoning_strength is defined and reasoning_strength else 'high' -%}
|
| 85 |
+
{{- 'Reasoning strength: ' + rs + '.' -}}
|
| 86 |
+
{%- endmacro -%}
|
| 87 |
+
{%- macro render_system_meta(tools) -%}
|
| 88 |
+
{%- set rns = namespace(recipients=['"self"'], nslist=[]) -%}
|
| 89 |
+
{%- if tools -%}
|
| 90 |
+
{%- for tool in tools -%}
|
| 91 |
+
{%- set fn = tool.function if tool.function is defined else tool -%}
|
| 92 |
+
{%- set tns = fn.name.split('.')[0] -%}
|
| 93 |
+
{%- if tns not in rns.nslist -%}
|
| 94 |
+
{%- set rns.nslist = rns.nslist + [tns] -%}
|
| 95 |
+
{%- endif -%}
|
| 96 |
+
{%- endfor -%}
|
| 97 |
+
{%- for tns in rns.nslist -%}
|
| 98 |
+
{%- set rns.recipients = rns.recipients + ['"' + tns + '.*"'] -%}
|
| 99 |
+
{%- endfor -%}
|
| 100 |
+
{%- endif -%}
|
| 101 |
+
{%- set rns.recipients = rns.recipients + ['"user"'] -%}
|
| 102 |
+
{{- '# Valid recipients: ' + rns.recipients | join(', ') + '.' -}}
|
| 103 |
+
{%- endmacro -%}
|
| 104 |
+
{{- bos_token -}}
|
| 105 |
+
{%- set ns = namespace(has_system=false) -%}
|
| 106 |
+
{%- for m in messages -%}
|
| 107 |
+
{%- if m['role'] == 'system' -%}
|
| 108 |
+
{%- set ns.has_system = true -%}
|
| 109 |
+
{%- endif -%}
|
| 110 |
+
{%- endfor -%}
|
| 111 |
+
{%- if not ns.has_system -%}
|
| 112 |
+
{{- '<|start|>system<|message|>You are a helpful AI assistant.' -}}
|
| 113 |
+
{%- set kc = knowledge_cutoff if knowledge_cutoff is defined and knowledge_cutoff else '2026-01-04' -%}
|
| 114 |
+
{{- '\nKnowledge cutoff: ' + kc + '.' -}}
|
| 115 |
+
{%- if current_date is defined and current_date -%}
|
| 116 |
+
{{- '\nCurrent date: ' + current_date + '.' -}}
|
| 117 |
+
{%- elif strftime_now is defined -%}
|
| 118 |
+
{{- '\nCurrent date: ' + strftime_now('%Y-%m-%d') + '.' -}}
|
| 119 |
+
{%- endif -%}
|
| 120 |
+
{{- '\n\n' -}}
|
| 121 |
+
{{- render_reasoning() -}}
|
| 122 |
+
{%- if tools -%}
|
| 123 |
+
{{- '\n\n' -}}
|
| 124 |
+
{{- render_tool_defs(tools) -}}
|
| 125 |
+
{%- endif -%}
|
| 126 |
+
{{- '\n\n' -}}
|
| 127 |
+
{{- render_system_meta(tools) -}}
|
| 128 |
+
{{- '<|eot|>' -}}
|
| 129 |
+
{%- endif -%}
|
| 130 |
+
{%- for message in messages -%}
|
| 131 |
+
{%- set role = message['role'] -%}
|
| 132 |
+
{%- set end_token = '<|eom|>' if (not loop.last and messages[loop.index0 + 1]['role'] == role) else '<|eot|>' -%}
|
| 133 |
+
{%- if role == 'system' -%}
|
| 134 |
+
{#- Callers sometimes write the directive into the system prompt themselves.
|
| 135 |
+
Normalise "Reasoning effort" to "Reasoning strength" (jinja has no
|
| 136 |
+
case-insensitive replace, hence the four realistic casings), then skip
|
| 137 |
+
the kwarg-driven line below if the prompt already carries one. -#}
|
| 138 |
+
{%- set sys_text = render_content(message['content'])
|
| 139 |
+
| replace('Reasoning effort', 'Reasoning strength')
|
| 140 |
+
| replace('Reasoning Effort', 'Reasoning Strength')
|
| 141 |
+
| replace('reasoning effort', 'reasoning strength')
|
| 142 |
+
| replace('REASONING EFFORT', 'REASONING STRENGTH') -%}
|
| 143 |
+
{{- '<|start|>system<|message|>' -}}
|
| 144 |
+
{{- sys_text -}}
|
| 145 |
+
{%- if 'reasoning strength' not in (sys_text | lower) -%}
|
| 146 |
+
{{- '\n\n' -}}
|
| 147 |
+
{{- render_reasoning() -}}
|
| 148 |
+
{%- endif -%}
|
| 149 |
+
{%- if tools -%}
|
| 150 |
+
{{- '\n\n' -}}
|
| 151 |
+
{{- render_tool_defs(tools) -}}
|
| 152 |
+
{%- endif -%}
|
| 153 |
+
{{- '\n\n' -}}
|
| 154 |
+
{{- render_system_meta(tools) -}}
|
| 155 |
+
{{- '<|eot|>' -}}
|
| 156 |
+
{%- elif role == 'user' -%}
|
| 157 |
+
{{- '<|start|>user<|message|>' -}}
|
| 158 |
+
{{- render_content(message['content']) -}}
|
| 159 |
+
{{- '<|eot|>' -}}
|
| 160 |
+
{%- elif role == 'tool' -%}
|
| 161 |
+
{%- set tname = message.get('name') -%}
|
| 162 |
+
{%- if not tname -%}
|
| 163 |
+
{%- set tcid = message.get('tool_call_id') -%}
|
| 164 |
+
{%- set rns = namespace(name=tcid if tcid else '') -%}
|
| 165 |
+
{%- for m in messages -%}
|
| 166 |
+
{%- if m.get('tool_calls') -%}
|
| 167 |
+
{%- for tc in m['tool_calls'] -%}
|
| 168 |
+
{%- if tcid is not none and tc.id is defined and tc.id == tcid -%}
|
| 169 |
+
{%- set rns.name = tc.function.name -%}
|
| 170 |
+
{%- endif -%}
|
| 171 |
+
{%- endfor -%}
|
| 172 |
+
{%- endif -%}
|
| 173 |
+
{%- endfor -%}
|
| 174 |
+
{%- set tname = rns.name -%}
|
| 175 |
+
{%- endif -%}
|
| 176 |
+
{{- '<|start|>tool ' + tname + '<|message|><tool_output name="' + tname + '">\n' -}}
|
| 177 |
+
{{- render_content(message['content']) -}}
|
| 178 |
+
{{- '\n</tool_output><|eot|>' -}}
|
| 179 |
+
{%- elif role == 'assistant' -%}
|
| 180 |
+
{%- if message.get('reasoning_content') -%}
|
| 181 |
+
{{- '<|start|>assistant to=self<|message|>' + message['reasoning_content'] + '<|eom|>' -}}
|
| 182 |
+
{%- endif -%}
|
| 183 |
+
{%- if message.get('tool_calls') -%}
|
| 184 |
+
{%- for tc in message['tool_calls'] -%}
|
| 185 |
+
{{- '<|start|>assistant to=' + tc.function.name + '<|message|>' -}}
|
| 186 |
+
{{- render_atem(tc) -}}
|
| 187 |
+
{%- if loop.last -%}
|
| 188 |
+
{{- end_token -}}
|
| 189 |
+
{%- else -%}
|
| 190 |
+
{{- '<|eom|>' -}}
|
| 191 |
+
{%- endif -%}
|
| 192 |
+
{%- endfor -%}
|
| 193 |
+
{%- else -%}
|
| 194 |
+
{%- set recipient = message.get('recipient') or 'user' -%}
|
| 195 |
+
{%- set end_turn = message.get('end_turn') -%}
|
| 196 |
+
{%- if end_turn is none -%}
|
| 197 |
+
{%- set end_turn = not (recipient and recipient != 'user') -%}
|
| 198 |
+
{%- endif -%}
|
| 199 |
+
{{- '<|start|>assistant' -}}
|
| 200 |
+
{%- if recipient -%}
|
| 201 |
+
{{- ' to=' + recipient -}}
|
| 202 |
+
{%- endif -%}
|
| 203 |
+
{{- '<|message|>' -}}
|
| 204 |
+
{{- render_content(message['content']) -}}
|
| 205 |
+
{{- ('<|eot|>' if end_turn else '<|eom|>') -}}
|
| 206 |
+
{%- endif -%}
|
| 207 |
+
{%- endif -%}
|
| 208 |
+
{%- endfor -%}
|
| 209 |
+
{%- if add_generation_prompt -%}
|
| 210 |
+
{{- '<|start|>assistant' -}}
|
| 211 |
+
{%- endif -%}
|
config.json
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"MuseGlimmerForConditionalGeneration"
|
| 4 |
+
],
|
| 5 |
+
"dtype": "bfloat16",
|
| 6 |
+
"image_token_id": 200092,
|
| 7 |
+
"model_type": "muse_glimmer",
|
| 8 |
+
"out_hidden_size": 6144,
|
| 9 |
+
"projector_hidden_act": "gelu",
|
| 10 |
+
"projector_hidden_size": 4096,
|
| 11 |
+
"text_config": {
|
| 12 |
+
"attention_bias": false,
|
| 13 |
+
"attention_dropout": 0.0,
|
| 14 |
+
"bos_token_id": 200000,
|
| 15 |
+
"dtype": "bfloat16",
|
| 16 |
+
"eos_token_id": 200001,
|
| 17 |
+
"final_logit_softcapping": 20.0,
|
| 18 |
+
"head_dim": 128,
|
| 19 |
+
"hidden_activation": "silu",
|
| 20 |
+
"hidden_size": 6656,
|
| 21 |
+
"initializer_range": 0.02,
|
| 22 |
+
"intermediate_size": 19968,
|
| 23 |
+
"layer_rope_theta": [
|
| 24 |
+
500000.0,
|
| 25 |
+
500000.0,
|
| 26 |
+
500000.0,
|
| 27 |
+
0,
|
| 28 |
+
500000.0,
|
| 29 |
+
500000.0,
|
| 30 |
+
500000.0,
|
| 31 |
+
0,
|
| 32 |
+
500000.0,
|
| 33 |
+
500000.0,
|
| 34 |
+
500000.0,
|
| 35 |
+
0,
|
| 36 |
+
500000.0,
|
| 37 |
+
500000.0,
|
| 38 |
+
500000.0,
|
| 39 |
+
0,
|
| 40 |
+
500000.0,
|
| 41 |
+
500000.0,
|
| 42 |
+
500000.0,
|
| 43 |
+
0,
|
| 44 |
+
500000.0,
|
| 45 |
+
500000.0,
|
| 46 |
+
500000.0,
|
| 47 |
+
0,
|
| 48 |
+
500000.0,
|
| 49 |
+
500000.0,
|
| 50 |
+
500000.0,
|
| 51 |
+
0,
|
| 52 |
+
500000.0,
|
| 53 |
+
500000.0,
|
| 54 |
+
500000.0,
|
| 55 |
+
0,
|
| 56 |
+
500000.0,
|
| 57 |
+
500000.0,
|
| 58 |
+
500000.0,
|
| 59 |
+
0,
|
| 60 |
+
500000.0,
|
| 61 |
+
500000.0,
|
| 62 |
+
500000.0,
|
| 63 |
+
0,
|
| 64 |
+
500000.0,
|
| 65 |
+
500000.0,
|
| 66 |
+
500000.0,
|
| 67 |
+
0,
|
| 68 |
+
500000.0,
|
| 69 |
+
500000.0,
|
| 70 |
+
500000.0,
|
| 71 |
+
0,
|
| 72 |
+
500000.0,
|
| 73 |
+
500000.0,
|
| 74 |
+
500000.0,
|
| 75 |
+
0
|
| 76 |
+
],
|
| 77 |
+
"layer_types": [
|
| 78 |
+
"sliding_attention",
|
| 79 |
+
"sliding_attention",
|
| 80 |
+
"sliding_attention",
|
| 81 |
+
"full_attention",
|
| 82 |
+
"sliding_attention",
|
| 83 |
+
"sliding_attention",
|
| 84 |
+
"sliding_attention",
|
| 85 |
+
"full_attention",
|
| 86 |
+
"sliding_attention",
|
| 87 |
+
"sliding_attention",
|
| 88 |
+
"sliding_attention",
|
| 89 |
+
"full_attention",
|
| 90 |
+
"sliding_attention",
|
| 91 |
+
"sliding_attention",
|
| 92 |
+
"sliding_attention",
|
| 93 |
+
"full_attention",
|
| 94 |
+
"sliding_attention",
|
| 95 |
+
"sliding_attention",
|
| 96 |
+
"sliding_attention",
|
| 97 |
+
"full_attention",
|
| 98 |
+
"sliding_attention",
|
| 99 |
+
"sliding_attention",
|
| 100 |
+
"sliding_attention",
|
| 101 |
+
"full_attention",
|
| 102 |
+
"sliding_attention",
|
| 103 |
+
"sliding_attention",
|
| 104 |
+
"sliding_attention",
|
| 105 |
+
"full_attention",
|
| 106 |
+
"sliding_attention",
|
| 107 |
+
"sliding_attention",
|
| 108 |
+
"sliding_attention",
|
| 109 |
+
"full_attention",
|
| 110 |
+
"sliding_attention",
|
| 111 |
+
"sliding_attention",
|
| 112 |
+
"sliding_attention",
|
| 113 |
+
"full_attention",
|
| 114 |
+
"sliding_attention",
|
| 115 |
+
"sliding_attention",
|
| 116 |
+
"sliding_attention",
|
| 117 |
+
"full_attention",
|
| 118 |
+
"sliding_attention",
|
| 119 |
+
"sliding_attention",
|
| 120 |
+
"sliding_attention",
|
| 121 |
+
"full_attention",
|
| 122 |
+
"sliding_attention",
|
| 123 |
+
"sliding_attention",
|
| 124 |
+
"sliding_attention",
|
| 125 |
+
"full_attention",
|
| 126 |
+
"sliding_attention",
|
| 127 |
+
"sliding_attention",
|
| 128 |
+
"sliding_attention",
|
| 129 |
+
"full_attention"
|
| 130 |
+
],
|
| 131 |
+
"max_position_embeddings": 131072,
|
| 132 |
+
"model_type": "muse_glimmer_text",
|
| 133 |
+
"num_attention_heads": 32,
|
| 134 |
+
"num_hidden_layers": 52,
|
| 135 |
+
"num_key_value_heads": 2,
|
| 136 |
+
"output_multiplier": 0.19611613513818404,
|
| 137 |
+
"pad_token_id": null,
|
| 138 |
+
"post_norm_eps": 1e-08,
|
| 139 |
+
"qk_scale_factor": 3.87,
|
| 140 |
+
"rms_norm_eps": 1e-05,
|
| 141 |
+
"rope_parameters": {
|
| 142 |
+
"rope_theta": 500000.0,
|
| 143 |
+
"rope_type": "default"
|
| 144 |
+
},
|
| 145 |
+
"sliding_window": 2048,
|
| 146 |
+
"tie_word_embeddings": false,
|
| 147 |
+
"use_cache": true,
|
| 148 |
+
"vocab_size": 202048
|
| 149 |
+
},
|
| 150 |
+
"transformers_version": "5.15.1",
|
| 151 |
+
"use_cache": false,
|
| 152 |
+
"video_token_id": 200091,
|
| 153 |
+
"vision_config": {
|
| 154 |
+
"dtype": "bfloat16",
|
| 155 |
+
"hidden_act": "gelu",
|
| 156 |
+
"hidden_size": 1536,
|
| 157 |
+
"intermediate_size": 8960,
|
| 158 |
+
"layer_norm_eps": 1e-05,
|
| 159 |
+
"layer_types": [
|
| 160 |
+
"window_attention",
|
| 161 |
+
"window_attention",
|
| 162 |
+
"window_attention",
|
| 163 |
+
"full_attention",
|
| 164 |
+
"window_attention",
|
| 165 |
+
"window_attention",
|
| 166 |
+
"window_attention",
|
| 167 |
+
"full_attention",
|
| 168 |
+
"window_attention",
|
| 169 |
+
"window_attention",
|
| 170 |
+
"window_attention",
|
| 171 |
+
"full_attention",
|
| 172 |
+
"window_attention",
|
| 173 |
+
"window_attention",
|
| 174 |
+
"window_attention",
|
| 175 |
+
"full_attention",
|
| 176 |
+
"window_attention",
|
| 177 |
+
"window_attention",
|
| 178 |
+
"window_attention",
|
| 179 |
+
"full_attention",
|
| 180 |
+
"window_attention",
|
| 181 |
+
"window_attention",
|
| 182 |
+
"window_attention",
|
| 183 |
+
"full_attention",
|
| 184 |
+
"window_attention",
|
| 185 |
+
"window_attention",
|
| 186 |
+
"window_attention",
|
| 187 |
+
"full_attention",
|
| 188 |
+
"window_attention",
|
| 189 |
+
"window_attention",
|
| 190 |
+
"window_attention",
|
| 191 |
+
"full_attention",
|
| 192 |
+
"window_attention",
|
| 193 |
+
"window_attention",
|
| 194 |
+
"window_attention",
|
| 195 |
+
"full_attention",
|
| 196 |
+
"window_attention",
|
| 197 |
+
"window_attention",
|
| 198 |
+
"window_attention",
|
| 199 |
+
"full_attention",
|
| 200 |
+
"window_attention",
|
| 201 |
+
"window_attention",
|
| 202 |
+
"window_attention",
|
| 203 |
+
"full_attention",
|
| 204 |
+
"window_attention",
|
| 205 |
+
"window_attention",
|
| 206 |
+
"window_attention",
|
| 207 |
+
"full_attention",
|
| 208 |
+
"window_attention",
|
| 209 |
+
"full_attention"
|
| 210 |
+
],
|
| 211 |
+
"max_position_embeddings": 1024,
|
| 212 |
+
"merge_size": 2,
|
| 213 |
+
"model_type": "muse_glimmer_vision",
|
| 214 |
+
"num_attention_heads": 16,
|
| 215 |
+
"num_hidden_layers": 50,
|
| 216 |
+
"patch_size": 14,
|
| 217 |
+
"patch_temporal": 2,
|
| 218 |
+
"pos_emb_height": 32,
|
| 219 |
+
"pos_emb_width": 32,
|
| 220 |
+
"rope_parameters": {
|
| 221 |
+
"rope_theta": 10000.0,
|
| 222 |
+
"rope_type": "default"
|
| 223 |
+
},
|
| 224 |
+
"skip_vision": true
|
| 225 |
+
},
|
| 226 |
+
"quantization": {
|
| 227 |
+
"group_size": 64,
|
| 228 |
+
"bits": 4,
|
| 229 |
+
"mode": "affine"
|
| 230 |
+
},
|
| 231 |
+
"quantization_config": {
|
| 232 |
+
"group_size": 64,
|
| 233 |
+
"bits": 4,
|
| 234 |
+
"mode": "affine"
|
| 235 |
+
}
|
| 236 |
+
}
|
conversion.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"source": "facebook/meta-encoder",
|
| 3 |
+
"revision": "3d0df704aa5bf66d9c49da1393181f33e893e5f7",
|
| 4 |
+
"bits": 4,
|
| 5 |
+
"group_size": 64,
|
| 6 |
+
"mode": "affine",
|
| 7 |
+
"vision": "unchanged BF16",
|
| 8 |
+
"source_tensors": 1436,
|
| 9 |
+
"quantized_modules": 418,
|
| 10 |
+
"weight_bytes": 19514005504,
|
| 11 |
+
"mlx": "0.32.3"
|
| 12 |
+
}
|
convert_metaencoder.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Stream the official checkpoint into MLX-VLM-compatible 4/8-bit shards."""
|
| 2 |
+
import argparse
|
| 3 |
+
import copy
|
| 4 |
+
import json
|
| 5 |
+
import shutil
|
| 6 |
+
import struct
|
| 7 |
+
import time
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
import mlx.core as mx
|
| 11 |
+
import mlx.nn as nn
|
| 12 |
+
import numpy as np
|
| 13 |
+
from mlx.utils import tree_flatten
|
| 14 |
+
from mlx_vlm.models.muse_glimmer import Model, ModelConfig
|
| 15 |
+
from mlx_vlm.utils import skip_multimodal_module
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def main():
|
| 19 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 20 |
+
parser.add_argument("--source", type=Path, required=True)
|
| 21 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 22 |
+
parser.add_argument("--revision", required=True)
|
| 23 |
+
parser.add_argument("--wait-for-download", action="store_true")
|
| 24 |
+
args = parser.parse_args()
|
| 25 |
+
config = json.loads((args.source / "config.json").read_text())
|
| 26 |
+
model = Model(ModelConfig.from_dict(config))
|
| 27 |
+
shapes = {k: v.shape for k, v in tree_flatten(model.parameters())}
|
| 28 |
+
quantized_paths = {
|
| 29 |
+
path for path, module in model.named_modules()
|
| 30 |
+
if isinstance(module, (nn.Linear, nn.Embedding))
|
| 31 |
+
and not skip_multimodal_module(path)
|
| 32 |
+
and module.weight.shape[-1] % 64 == 0
|
| 33 |
+
}
|
| 34 |
+
source_index = json.loads((args.source / "model.safetensors.index.json").read_text())
|
| 35 |
+
mapped = model.sanitize({key: None for key in source_index["weight_map"]})
|
| 36 |
+
if set(mapped) != set(shapes):
|
| 37 |
+
raise ValueError(f"Schema mismatch: missing={set(shapes)-set(mapped)}, extra={set(mapped)-set(shapes)}")
|
| 38 |
+
del model
|
| 39 |
+
mx.clear_cache()
|
| 40 |
+
mx.set_cache_limit(256 * 1024**2)
|
| 41 |
+
states = {}
|
| 42 |
+
for bits in (4, 8):
|
| 43 |
+
folder = args.output / f"meta-encoder-mlx-{bits}bit"
|
| 44 |
+
folder.mkdir(parents=True, exist_ok=True)
|
| 45 |
+
if list(folder.glob("*.safetensors")):
|
| 46 |
+
raise FileExistsError(f"Refusing to overwrite existing weights in {folder}")
|
| 47 |
+
states[bits] = dict(folder=folder, pending={}, bytes=0, total=0, index={}, shards=[])
|
| 48 |
+
|
| 49 |
+
def flush(state):
|
| 50 |
+
if not state["pending"]:
|
| 51 |
+
return
|
| 52 |
+
name = f"part-{len(state['shards'])+1:05d}.safetensors"
|
| 53 |
+
mx.save_safetensors(str(state["folder"] / name), state["pending"], metadata={"format": "mlx"})
|
| 54 |
+
state["index"].update({key: name for key in state["pending"]})
|
| 55 |
+
state["shards"].append(name)
|
| 56 |
+
state["pending"].clear()
|
| 57 |
+
state["bytes"] = 0
|
| 58 |
+
|
| 59 |
+
count = 0
|
| 60 |
+
for shard_name in sorted(set(source_index["weight_map"].values()), reverse=True):
|
| 61 |
+
if args.wait_for_download:
|
| 62 |
+
while not (args.source / shard_name).exists():
|
| 63 |
+
print(f"Waiting for download: {shard_name}", flush=True)
|
| 64 |
+
time.sleep(30)
|
| 65 |
+
with (args.source / shard_name).open("rb") as source:
|
| 66 |
+
header_size = struct.unpack("<Q", source.read(8))[0]
|
| 67 |
+
header = json.loads(source.read(header_size))
|
| 68 |
+
for original, info in header.items():
|
| 69 |
+
if original == "__metadata__":
|
| 70 |
+
continue
|
| 71 |
+
key = next(iter(Model.sanitize(None, {original: None})))
|
| 72 |
+
if tuple(info["shape"]) != tuple(shapes[key]):
|
| 73 |
+
raise ValueError(f"Shape mismatch for {key}")
|
| 74 |
+
if info["dtype"] != "BF16":
|
| 75 |
+
raise ValueError(f"Unexpected dtype {info['dtype']} for {key}")
|
| 76 |
+
start, end = info["data_offsets"]
|
| 77 |
+
source.seek(8 + header_size + start)
|
| 78 |
+
raw = source.read(end - start)
|
| 79 |
+
if len(raw) != end - start:
|
| 80 |
+
raise EOFError(original)
|
| 81 |
+
weight = mx.array(np.frombuffer(raw, dtype=np.uint16).reshape(info["shape"])).view(mx.bfloat16)
|
| 82 |
+
mx.eval(weight)
|
| 83 |
+
del raw
|
| 84 |
+
for bits, state in states.items():
|
| 85 |
+
if key.removesuffix(".weight") in quantized_paths and key.endswith(".weight"):
|
| 86 |
+
packed, scales, biases = mx.quantize(weight, group_size=64, bits=bits, mode="affine")
|
| 87 |
+
arrays = {key: packed, key.removesuffix(".weight") + ".scales": scales,
|
| 88 |
+
key.removesuffix(".weight") + ".biases": biases}
|
| 89 |
+
else:
|
| 90 |
+
arrays = {key: weight}
|
| 91 |
+
mx.eval(list(arrays.values()))
|
| 92 |
+
size = sum(value.nbytes for value in arrays.values())
|
| 93 |
+
if state["bytes"] + size > 2 * 1024**3:
|
| 94 |
+
flush(state)
|
| 95 |
+
state["pending"].update(arrays)
|
| 96 |
+
state["bytes"] += size
|
| 97 |
+
state["total"] += size
|
| 98 |
+
del arrays, weight
|
| 99 |
+
count += 1
|
| 100 |
+
if count % 100 == 0:
|
| 101 |
+
print(f"Converted {count}/{len(shapes)} tensors", flush=True)
|
| 102 |
+
for bits, state in states.items():
|
| 103 |
+
flush(state)
|
| 104 |
+
names = {}
|
| 105 |
+
for number, old in enumerate(state["shards"], 1):
|
| 106 |
+
new = f"model-{number:05d}-of-{len(state['shards']):05d}.safetensors"
|
| 107 |
+
(state["folder"] / old).rename(state["folder"] / new)
|
| 108 |
+
names[old] = new
|
| 109 |
+
index = {"metadata": {"total_size": state["total"]},
|
| 110 |
+
"weight_map": {k: names[v] for k, v in sorted(state["index"].items())}}
|
| 111 |
+
(state["folder"] / "model.safetensors.index.json").write_text(json.dumps(index, indent=2) + "\n")
|
| 112 |
+
for name in ("tokenizer.json", "tokenizer_config.json", "processor_config.json",
|
| 113 |
+
"generation_config.json", "chat_template.jinja", "LICENSE", "NOTICE", "USAGE_POLICY.md"):
|
| 114 |
+
if (args.source / name).exists():
|
| 115 |
+
shutil.copy2(args.source / name, state["folder"] / name)
|
| 116 |
+
out_config = copy.deepcopy(config)
|
| 117 |
+
out_config["quantization"] = {"group_size": 64, "bits": bits, "mode": "affine"}
|
| 118 |
+
out_config["quantization_config"] = copy.deepcopy(out_config["quantization"])
|
| 119 |
+
out_config["vision_config"]["skip_vision"] = True
|
| 120 |
+
(state["folder"] / "config.json").write_text(json.dumps(out_config, indent=2) + "\n")
|
| 121 |
+
provenance = {"source": "facebook/meta-encoder", "revision": args.revision,
|
| 122 |
+
"bits": bits, "group_size": 64, "mode": "affine",
|
| 123 |
+
"vision": "unchanged BF16", "source_tensors": count,
|
| 124 |
+
"quantized_modules": len(quantized_paths), "weight_bytes": state["total"],
|
| 125 |
+
"mlx": mx.__version__}
|
| 126 |
+
(state["folder"] / "conversion.json").write_text(json.dumps(provenance, indent=2) + "\n")
|
| 127 |
+
print(json.dumps(provenance), flush=True)
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
if __name__ == "__main__":
|
| 131 |
+
main()
|
generation_config.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token_id": 200000,
|
| 3 |
+
"do_sample": true,
|
| 4 |
+
"eos_token_id": [
|
| 5 |
+
200001,
|
| 6 |
+
200008
|
| 7 |
+
],
|
| 8 |
+
"max_length": 131072,
|
| 9 |
+
"pad_token_id": 200018,
|
| 10 |
+
"temperature": 1.0,
|
| 11 |
+
"top_k": 64,
|
| 12 |
+
"top_p": 0.95,
|
| 13 |
+
"transformers_version": "5.15.1"
|
| 14 |
+
}
|
metaencoder_mlx.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MLX inference for MetaEncoder with its official prompt and last-token pooling."""
|
| 2 |
+
import argparse
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import MethodType
|
| 6 |
+
|
| 7 |
+
import mlx.core as mx
|
| 8 |
+
import numpy as np
|
| 9 |
+
from mlx_vlm.utils import load_model
|
| 10 |
+
from PIL import Image
|
| 11 |
+
from transformers.models.muse_glimmer.processing_muse_glimmer import MuseGlimmerProcessor
|
| 12 |
+
|
| 13 |
+
from upstream_metaencoder import (
|
| 14 |
+
DEFAULT_SYSTEM_INSTRUCTION, VIDEO_FPS, _as_items, _user_text,
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _reference_vision_rotary(vision, positions):
|
| 19 |
+
inv_freq = 1.0 / (vision.rope_theta ** (
|
| 20 |
+
mx.arange(0, vision.rope_spatial_dim, 2, dtype=mx.float32) / vision.rope_spatial_dim))
|
| 21 |
+
inv_freq = inv_freq.astype(mx.bfloat16).astype(mx.float32)
|
| 22 |
+
width = positions[:, 0].astype(mx.float32)[:, None] * inv_freq[None, :]
|
| 23 |
+
height = positions[:, 1].astype(mx.float32)[:, None] * inv_freq[None, :]
|
| 24 |
+
frequencies = mx.concatenate([width, height, width, height], axis=-1)
|
| 25 |
+
return mx.cos(frequencies).astype(mx.bfloat16), mx.sin(frequencies).astype(mx.bfloat16)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class MetaEncoderMLX:
|
| 29 |
+
def __init__(self, path):
|
| 30 |
+
self.model = load_model(Path(path), lazy=False)
|
| 31 |
+
self.model.eval()
|
| 32 |
+
# Use the original Transformers processor for exact image/video token routing.
|
| 33 |
+
self.processor = MuseGlimmerProcessor.from_pretrained(path)
|
| 34 |
+
self.processor.tokenizer.padding_side = "left"
|
| 35 |
+
# The reference MetaEncoder rounds rotary frequencies to BF16 at load time.
|
| 36 |
+
for layer in self.model.language_model.model.layers:
|
| 37 |
+
rope = layer.self_attn.rope
|
| 38 |
+
rope._frequencies = rope._frequencies.astype(mx.bfloat16).astype(mx.float32)
|
| 39 |
+
self.model.vision_tower._rotary = MethodType(_reference_vision_rotary, self.model.vision_tower)
|
| 40 |
+
|
| 41 |
+
@property
|
| 42 |
+
def embedding_dim(self):
|
| 43 |
+
return self.model.config.text_config.hidden_size
|
| 44 |
+
|
| 45 |
+
def encode(self, inputs, instruction=None, max_length=8192):
|
| 46 |
+
items = _as_items(inputs, instruction)
|
| 47 |
+
vectors = []
|
| 48 |
+
# Individual forwards avoid padding entering the decoder's attention masks.
|
| 49 |
+
for item in items:
|
| 50 |
+
content = [{"type": "image", "image": image} for image in item["images"] or []]
|
| 51 |
+
if item["video"]:
|
| 52 |
+
content.append({"type": "video", "video": item["video"]})
|
| 53 |
+
text = _user_text(item["instruction"], item["text"])
|
| 54 |
+
if text or not content:
|
| 55 |
+
content.append({"type": "text", "text": text or " "})
|
| 56 |
+
conversation = [
|
| 57 |
+
{"role": "system", "content": [{"type": "text", "text": DEFAULT_SYSTEM_INSTRUCTION}]},
|
| 58 |
+
{"role": "user", "content": content},
|
| 59 |
+
]
|
| 60 |
+
prompt = self.processor.apply_chat_template(conversation, add_generation_prompt=False, tokenize=False)
|
| 61 |
+
has_media = bool(item["images"] or item["video"])
|
| 62 |
+
kwargs = dict(text=[prompt], images=item["images"], videos=[item["video"]] if item["video"] else None,
|
| 63 |
+
return_tensors="np", add_special_tokens=False, padding=False,
|
| 64 |
+
truncation=not has_media)
|
| 65 |
+
if not has_media:
|
| 66 |
+
kwargs["max_length"] = max_length
|
| 67 |
+
if item["video"]:
|
| 68 |
+
from transformers.video_utils import VideoMetadata
|
| 69 |
+
frames = item["video"]
|
| 70 |
+
kwargs.update(do_sample_frames=False, video_metadata=[VideoMetadata(
|
| 71 |
+
total_num_frames=len(frames), fps=VIDEO_FPS,
|
| 72 |
+
duration=len(frames)/VIDEO_FPS, frames_indices=list(range(len(frames))))])
|
| 73 |
+
processed = self.processor(**kwargs)
|
| 74 |
+
for media, pixel_key in ((item["images"], "pixel_values"), (item["video"], "pixel_values_videos")):
|
| 75 |
+
if media and pixel_key not in processed:
|
| 76 |
+
raise ValueError(f"Processor dropped media: missing {pixel_key}")
|
| 77 |
+
tensors = {key: mx.array(np.asarray(value)) for key, value in processed.items()
|
| 78 |
+
if key in {"input_ids", "pixel_values", "pixel_values_videos", "image_grid_thw", "video_grid_thw"}}
|
| 79 |
+
ids = tensors["input_ids"].astype(mx.int32)
|
| 80 |
+
language = self.model.language_model.model
|
| 81 |
+
embeddings = language.embed_norm(language.embed_tokens(ids))
|
| 82 |
+
for pixel_key, grid_key, token_id in (
|
| 83 |
+
("pixel_values", "image_grid_thw", self.model.config.image_token_id),
|
| 84 |
+
("pixel_values_videos", "video_grid_thw", self.model.config.video_token_id),
|
| 85 |
+
):
|
| 86 |
+
if pixel_key not in tensors:
|
| 87 |
+
continue
|
| 88 |
+
features = self.model._encode_image(tensors[pixel_key], tensors[grid_key]).astype(embeddings.dtype)
|
| 89 |
+
positions = np.flatnonzero(np.asarray(ids[0]) == token_id)
|
| 90 |
+
if len(positions) != features.shape[0]:
|
| 91 |
+
raise ValueError(f"Media token count {len(positions)} differs from feature count {features.shape[0]}")
|
| 92 |
+
embeddings[0, mx.array(positions.astype(np.int32))] = features
|
| 93 |
+
hidden = language(ids, inputs_embeds=embeddings)
|
| 94 |
+
pooled = hidden[:, -1, :].astype(mx.float32)
|
| 95 |
+
pooled = pooled / mx.maximum(mx.linalg.norm(pooled, axis=-1, keepdims=True), 1e-12)
|
| 96 |
+
mx.eval(pooled)
|
| 97 |
+
vectors.append(pooled)
|
| 98 |
+
return mx.concatenate(vectors, axis=0) if vectors else mx.zeros((0, self.embedding_dim))
|
| 99 |
+
|
| 100 |
+
def score(self, task, candidates):
|
| 101 |
+
task_vector = self.encode(task)
|
| 102 |
+
candidate_vectors = candidates if isinstance(candidates, mx.array) else self.encode(candidates)
|
| 103 |
+
if candidate_vectors.ndim != 2 or candidate_vectors.shape[1] != self.embedding_dim:
|
| 104 |
+
raise ValueError(f"Expected candidate embeddings of shape (n, {self.embedding_dim})")
|
| 105 |
+
scores = (task_vector @ candidate_vectors.T)[0]
|
| 106 |
+
mx.eval(scores)
|
| 107 |
+
return scores
|
| 108 |
+
|
| 109 |
+
def match(self, task, candidates, top_k=None):
|
| 110 |
+
scores = np.asarray(self.score(task, candidates))
|
| 111 |
+
order = np.argsort(-scores, kind="stable")
|
| 112 |
+
if top_k is not None:
|
| 113 |
+
if top_k < 0:
|
| 114 |
+
raise ValueError("top_k must be nonnegative")
|
| 115 |
+
order = order[:top_k]
|
| 116 |
+
return [{"index": int(index), "score": float(scores[index]),
|
| 117 |
+
"candidate": None if isinstance(candidates, mx.array) else candidates[int(index)]}
|
| 118 |
+
for index in order]
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def main():
|
| 122 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 123 |
+
parser.add_argument("--model", required=True)
|
| 124 |
+
parser.add_argument("--task", required=True)
|
| 125 |
+
parser.add_argument("--candidates", nargs="+", required=True)
|
| 126 |
+
args = parser.parse_args()
|
| 127 |
+
model = MetaEncoderMLX(args.model)
|
| 128 |
+
print(json.dumps(model.match(args.task, args.candidates), indent=2))
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
if __name__ == "__main__":
|
| 132 |
+
main()
|
model-00001-of-00010.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f7c44d6022c2f8ac4d1dfe3641af5d3c23c85e8931c85a03bec9b8357442e9bc
|
| 3 |
+
size 2146561182
|
model-00002-of-00010.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a3a30b9d196464cc6ea259d2d8134ee990681316cc91ab4e284ee0c07bed5c10
|
| 3 |
+
size 2145187111
|
model-00003-of-00010.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cf3dc03558ecc72388a2711570be08106c59d2079d7d0d5d080b847145fd1da5
|
| 3 |
+
size 1942065880
|
model-00004-of-00010.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1cbcb381e609ba53479dc80be31087989986290f7d95ba657ef6c6097cb07d52
|
| 3 |
+
size 2117784741
|
model-00005-of-00010.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:16a68f2ffbdacc9638cb356acef5c83ce7022f1badc7864280226b8124457720
|
| 3 |
+
size 2146441412
|
model-00006-of-00010.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:846d39f1644380f29eaf58b72d2135aa45663a363bc46beacef3cd5e81846593
|
| 3 |
+
size 2086989785
|
model-00007-of-00010.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a030b542c9761c3def391d9bf6c8ef80c39b8f10233d306520e18a9ab4ffecd8
|
| 3 |
+
size 2103324751
|
model-00008-of-00010.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c8b08350a6c9497b39ea04c9f0eb2096ac4f14b93c697daca683e32016e0c5a9
|
| 3 |
+
size 2103324683
|
model-00009-of-00010.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:742cbc081524b9c910287505f6cd5d7eac1bbef125ea1a025d101e4df250c76b
|
| 3 |
+
size 2146454789
|
model-00010-of-00010.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:84b15c3067a65493dc0b4d548a0d75ecdc74660585df0ded07ba12eff04b69c3
|
| 3 |
+
size 576151680
|
model.safetensors.index.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
processor_config.json
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"image_processor": {
|
| 3 |
+
"do_convert_rgb": true,
|
| 4 |
+
"do_normalize": true,
|
| 5 |
+
"do_rescale": true,
|
| 6 |
+
"do_resize": true,
|
| 7 |
+
"image_mean": [
|
| 8 |
+
0.5,
|
| 9 |
+
0.5,
|
| 10 |
+
0.5
|
| 11 |
+
],
|
| 12 |
+
"image_processor_type": "MuseGlimmerImageProcessor",
|
| 13 |
+
"image_std": [
|
| 14 |
+
0.5,
|
| 15 |
+
0.5,
|
| 16 |
+
0.5
|
| 17 |
+
],
|
| 18 |
+
"max_image_tokens": 1280,
|
| 19 |
+
"merge_size": 2,
|
| 20 |
+
"patch_size": 14,
|
| 21 |
+
"resample": 1,
|
| 22 |
+
"rescale_factor": 0.00392156862745098,
|
| 23 |
+
"temporal_patch_size": 2
|
| 24 |
+
},
|
| 25 |
+
"processor_class": "MuseGlimmerProcessor",
|
| 26 |
+
"video_processor": {
|
| 27 |
+
"do_convert_rgb": true,
|
| 28 |
+
"do_normalize": true,
|
| 29 |
+
"do_rescale": true,
|
| 30 |
+
"do_resize": true,
|
| 31 |
+
"do_sample_frames": true,
|
| 32 |
+
"fps": 2.0,
|
| 33 |
+
"image_mean": [
|
| 34 |
+
0.5,
|
| 35 |
+
0.5,
|
| 36 |
+
0.5
|
| 37 |
+
],
|
| 38 |
+
"image_std": [
|
| 39 |
+
0.5,
|
| 40 |
+
0.5,
|
| 41 |
+
0.5
|
| 42 |
+
],
|
| 43 |
+
"max_video_frame_tokens": 144,
|
| 44 |
+
"merge_size": 2,
|
| 45 |
+
"num_frames": 96,
|
| 46 |
+
"patch_size": 14,
|
| 47 |
+
"resample": 1,
|
| 48 |
+
"rescale_factor": 0.00392156862745098,
|
| 49 |
+
"return_metadata": true,
|
| 50 |
+
"temporal_patch_size": 2,
|
| 51 |
+
"video_processor_type": "MuseGlimmerVideoProcessor"
|
| 52 |
+
}
|
| 53 |
+
}
|
requirements.txt
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
accelerate==1.15.0
|
| 2 |
+
annotated-doc==0.0.5
|
| 3 |
+
annotated-types==0.8.0
|
| 4 |
+
anyio==4.15.1
|
| 5 |
+
certifi==2026.7.22
|
| 6 |
+
cffi==2.1.1
|
| 7 |
+
charset-normalizer==3.5.2
|
| 8 |
+
click==8.5.0
|
| 9 |
+
cryptography==50.0.2
|
| 10 |
+
fastapi==0.143.0
|
| 11 |
+
filelock==4.0.12
|
| 12 |
+
fsspec==2026.9.0
|
| 13 |
+
h11==0.16.0
|
| 14 |
+
hf-xet==1.7.0
|
| 15 |
+
httpcore2==2.13.1
|
| 16 |
+
httpx2==2.13.1
|
| 17 |
+
huggingface-hub==2.2.0
|
| 18 |
+
idna==3.20
|
| 19 |
+
jinja2==3.1.6
|
| 20 |
+
llguidance==1.9.1
|
| 21 |
+
markdown-it-py==4.2.0
|
| 22 |
+
markupsafe==3.0.4
|
| 23 |
+
mdurl==0.1.2
|
| 24 |
+
miniaudio==1.71
|
| 25 |
+
mlx==0.32.3
|
| 26 |
+
mlx-audio==0.5.8
|
| 27 |
+
mlx-metal==0.32.3
|
| 28 |
+
mlx-vlm==0.7.6
|
| 29 |
+
mpmath==1.3.0
|
| 30 |
+
networkx==3.7
|
| 31 |
+
numpy==2.5.3
|
| 32 |
+
opencv-python==5.0.0.93
|
| 33 |
+
opentelemetry-api==1.45.1
|
| 34 |
+
packaging==26.3
|
| 35 |
+
pillow==12.3.0
|
| 36 |
+
psutil==7.2.2
|
| 37 |
+
pycparser==3.11
|
| 38 |
+
pydantic==2.14.0
|
| 39 |
+
pydantic-core==2.50.0
|
| 40 |
+
pygments==2.21.0
|
| 41 |
+
python-multipart==0.0.32
|
| 42 |
+
pyyaml==6.0.3
|
| 43 |
+
regex==2026.9.29
|
| 44 |
+
requests==2.34.2
|
| 45 |
+
rich==15.0.0
|
| 46 |
+
safetensors==0.8.0
|
| 47 |
+
scipy==1.18.1
|
| 48 |
+
sentencepiece==0.2.2
|
| 49 |
+
setuptools==84.0.0
|
| 50 |
+
shellingham==1.5.4
|
| 51 |
+
sounddevice==0.5.6
|
| 52 |
+
starlette==1.7.0
|
| 53 |
+
sympy==1.14.0
|
| 54 |
+
tokenizers==0.23.3
|
| 55 |
+
torch==2.14.1
|
| 56 |
+
torchvision==0.29.1
|
| 57 |
+
tqdm==4.70.1
|
| 58 |
+
transformers==5.19.0
|
| 59 |
+
truststore==0.10.4
|
| 60 |
+
typer==0.27.3
|
| 61 |
+
typing-extensions==4.16.0
|
| 62 |
+
typing-inspection==0.4.4
|
| 63 |
+
urllib3==2.8.0
|
| 64 |
+
uvicorn==0.54.0
|
| 65 |
+
websockets==17.2
|
source-sha256.json
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model-00002-of-00002.safetensors": "ced1649f8cf4ab4852f68fad42656a60022a9271b4319a606534f5ff03ca3aca",
|
| 3 |
+
"model-00001-of-00002.safetensors": "b1479b3c58364b9135e28ccac7b6dd5e99482662f4a3ce82cae5ef25f819d6f7"
|
| 4 |
+
}
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c9dbee66967b58f31a7c27f723c3760da3526ccd0427578e8905b0abb0031c4d
|
| 3 |
+
size 28129897
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,2104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"backend": "tokenizers",
|
| 3 |
+
"bos_token": "<|begin_of_text|>",
|
| 4 |
+
"eos_token": "<|end_of_text|>",
|
| 5 |
+
"extra_special_tokens": [
|
| 6 |
+
"<|begin_of_text|>",
|
| 7 |
+
"<|end_of_text|>",
|
| 8 |
+
"<|reserved_special_token_2|>",
|
| 9 |
+
"<|reserved_special_token_3|>",
|
| 10 |
+
"<|reserved_special_token_4|>",
|
| 11 |
+
"<|reserved_special_token_5|>",
|
| 12 |
+
"<|reserved_special_token_6|>",
|
| 13 |
+
"<|eom|>",
|
| 14 |
+
"<|eot|>",
|
| 15 |
+
"<|reserved_special_token_9|>",
|
| 16 |
+
"<|reserved_special_token_10|>",
|
| 17 |
+
"<|reserved_special_token_11|>",
|
| 18 |
+
"<|reserved_special_token_12|>",
|
| 19 |
+
"<|reserved_special_token_13|>",
|
| 20 |
+
"<|reserved_special_token_14|>",
|
| 21 |
+
"<|reserved_special_token_15|>",
|
| 22 |
+
"<|reserved_special_token_16|>",
|
| 23 |
+
"<|reserved_special_token_17|>",
|
| 24 |
+
"<|finetune_right_pad|>",
|
| 25 |
+
"<|reserved_special_token_19|>",
|
| 26 |
+
"<|reserved_special_token_20|>",
|
| 27 |
+
"<|reserved_special_token_21|>",
|
| 28 |
+
"<|start|>",
|
| 29 |
+
"<|message|>",
|
| 30 |
+
"<|reserved_special_token_24|>",
|
| 31 |
+
"<|reserved_special_token_25|>",
|
| 32 |
+
"<|reserved_special_token_26|>",
|
| 33 |
+
"<|reserved_special_token_27|>",
|
| 34 |
+
"<|reserved_special_token_28|>",
|
| 35 |
+
"<|reserved_special_token_29|>",
|
| 36 |
+
"<|reserved_special_token_30|>",
|
| 37 |
+
"<|reserved_special_token_31|>",
|
| 38 |
+
"<|reserved_special_token_32|>",
|
| 39 |
+
"<|reserved_special_token_33|>",
|
| 40 |
+
"<|reserved_special_token_34|>",
|
| 41 |
+
"<|reserved_special_token_35|>",
|
| 42 |
+
"<|reserved_special_token_36|>",
|
| 43 |
+
"<|reserved_special_token_37|>",
|
| 44 |
+
"<|reserved_special_token_38|>",
|
| 45 |
+
"<|reserved_special_token_39|>",
|
| 46 |
+
"<|reserved_special_token_40|>",
|
| 47 |
+
"<|reserved_special_token_41|>",
|
| 48 |
+
"<|reserved_special_token_42|>",
|
| 49 |
+
"<|reserved_special_token_43|>",
|
| 50 |
+
"<|reserved_special_token_44|>",
|
| 51 |
+
"<|reserved_special_token_45|>",
|
| 52 |
+
"<|reserved_special_token_46|>",
|
| 53 |
+
"<|reserved_special_token_47|>",
|
| 54 |
+
"<|reserved_special_token_48|>",
|
| 55 |
+
"<|reserved_special_token_49|>",
|
| 56 |
+
"<|reserved_special_token_50|>",
|
| 57 |
+
"<|reserved_special_token_51|>",
|
| 58 |
+
"<|reserved_special_token_52|>",
|
| 59 |
+
"<|reserved_special_token_53|>",
|
| 60 |
+
"<|reserved_special_token_54|>",
|
| 61 |
+
"<|reserved_special_token_55|>",
|
| 62 |
+
"<|reserved_special_token_56|>",
|
| 63 |
+
"<|reserved_special_token_57|>",
|
| 64 |
+
"<|reserved_special_token_58|>",
|
| 65 |
+
"<|reserved_special_token_59|>",
|
| 66 |
+
"<|reserved_special_token_60|>",
|
| 67 |
+
"<|reserved_special_token_61|>",
|
| 68 |
+
"<|reserved_special_token_62|>",
|
| 69 |
+
"<|reserved_special_token_63|>",
|
| 70 |
+
"<|reserved_special_token_64|>",
|
| 71 |
+
"<|reserved_special_token_65|>",
|
| 72 |
+
"<|reserved_special_token_66|>",
|
| 73 |
+
"<|reserved_special_token_67|>",
|
| 74 |
+
"<|reserved_special_token_68|>",
|
| 75 |
+
"<|reserved_special_token_69|>",
|
| 76 |
+
"<|reserved_special_token_70|>",
|
| 77 |
+
"<|reserved_special_token_71|>",
|
| 78 |
+
"<|reserved_special_token_72|>",
|
| 79 |
+
"<|reserved_special_token_73|>",
|
| 80 |
+
"<|reserved_special_token_74|>",
|
| 81 |
+
"<|reserved_special_token_75|>",
|
| 82 |
+
"<|reserved_special_token_76|>",
|
| 83 |
+
"<|reserved_special_token_77|>",
|
| 84 |
+
"<|reserved_special_token_78|>",
|
| 85 |
+
"<|reserved_special_token_79|>",
|
| 86 |
+
"<|image_start|>",
|
| 87 |
+
"<|image_end|>",
|
| 88 |
+
"<|vid_start|>",
|
| 89 |
+
"<|vid_end|>",
|
| 90 |
+
"<|reserved_special_token_84|>",
|
| 91 |
+
"<|reserved_special_token_85|>",
|
| 92 |
+
"<|reserved_special_token_86|>",
|
| 93 |
+
"<|vid_frame_separator|>",
|
| 94 |
+
"<|reserved_special_token_88|>",
|
| 95 |
+
"<|reserved_special_token_89|>",
|
| 96 |
+
"<|image|>",
|
| 97 |
+
"<|video|>",
|
| 98 |
+
"<|patch|>",
|
| 99 |
+
"<|reserved_special_token_93|>",
|
| 100 |
+
"<|reserved_special_token_94|>",
|
| 101 |
+
"<|reserved_special_token_95|>",
|
| 102 |
+
"<|reserved_special_token_96|>",
|
| 103 |
+
"<|reserved_special_token_97|>",
|
| 104 |
+
"<|reserved_special_token_98|>",
|
| 105 |
+
"<|reserved_special_token_99|>",
|
| 106 |
+
"<|reserved_special_token_100|>",
|
| 107 |
+
"<|reserved_special_token_101|>",
|
| 108 |
+
"<|reserved_special_token_102|>",
|
| 109 |
+
"<|reserved_special_token_103|>",
|
| 110 |
+
"<|reserved_special_token_104|>",
|
| 111 |
+
"<|reserved_special_token_105|>",
|
| 112 |
+
"<|reserved_special_token_106|>",
|
| 113 |
+
"<|reserved_special_token_107|>",
|
| 114 |
+
"<|reserved_special_token_108|>",
|
| 115 |
+
"<|reserved_special_token_109|>",
|
| 116 |
+
"<|reserved_special_token_110|>",
|
| 117 |
+
"<|reserved_special_token_111|>",
|
| 118 |
+
"<|reserved_special_token_112|>",
|
| 119 |
+
"<|reserved_special_token_113|>",
|
| 120 |
+
"<|reserved_special_token_114|>",
|
| 121 |
+
"<|reserved_special_token_115|>",
|
| 122 |
+
"<|reserved_special_token_116|>",
|
| 123 |
+
"<|reserved_special_token_117|>",
|
| 124 |
+
"<|reserved_special_token_118|>",
|
| 125 |
+
"<|reserved_special_token_119|>",
|
| 126 |
+
"<|reserved_special_token_120|>",
|
| 127 |
+
"<|reserved_special_token_121|>",
|
| 128 |
+
"<|reserved_special_token_122|>",
|
| 129 |
+
"<|reserved_special_token_123|>",
|
| 130 |
+
"<|reserved_special_token_124|>",
|
| 131 |
+
"<|reserved_special_token_125|>",
|
| 132 |
+
"<|reserved_special_token_126|>",
|
| 133 |
+
"<|reserved_special_token_127|>",
|
| 134 |
+
"<|reserved_special_token_128|>",
|
| 135 |
+
"<|reserved_special_token_129|>",
|
| 136 |
+
"<|reserved_special_token_130|>",
|
| 137 |
+
"<|reserved_special_token_131|>",
|
| 138 |
+
"<|reserved_special_token_132|>",
|
| 139 |
+
"<|reserved_special_token_133|>",
|
| 140 |
+
"<|reserved_special_token_134|>",
|
| 141 |
+
"<|reserved_special_token_135|>",
|
| 142 |
+
"<|reserved_special_token_136|>",
|
| 143 |
+
"<|reserved_special_token_137|>",
|
| 144 |
+
"<|reserved_special_token_138|>",
|
| 145 |
+
"<|reserved_special_token_139|>",
|
| 146 |
+
"<|reserved_special_token_140|>",
|
| 147 |
+
"<|reserved_special_token_141|>",
|
| 148 |
+
"<|reserved_special_token_142|>",
|
| 149 |
+
"<|reserved_special_token_143|>",
|
| 150 |
+
"<|reserved_special_token_144|>",
|
| 151 |
+
"<|reserved_special_token_145|>",
|
| 152 |
+
"<|reserved_special_token_146|>",
|
| 153 |
+
"<|reserved_special_token_147|>",
|
| 154 |
+
"<|reserved_special_token_148|>",
|
| 155 |
+
"<|reserved_special_token_149|>",
|
| 156 |
+
"<|reserved_special_token_150|>",
|
| 157 |
+
"<|reserved_special_token_151|>",
|
| 158 |
+
"<|reserved_special_token_152|>",
|
| 159 |
+
"<|reserved_special_token_153|>",
|
| 160 |
+
"<|reserved_special_token_154|>",
|
| 161 |
+
"<|reserved_special_token_155|>",
|
| 162 |
+
"<|reserved_special_token_156|>",
|
| 163 |
+
"<|reserved_special_token_157|>",
|
| 164 |
+
"<|reserved_special_token_158|>",
|
| 165 |
+
"<|reserved_special_token_159|>",
|
| 166 |
+
"<|reserved_special_token_160|>",
|
| 167 |
+
"<|reserved_special_token_161|>",
|
| 168 |
+
"<|reserved_special_token_162|>",
|
| 169 |
+
"<|reserved_special_token_163|>",
|
| 170 |
+
"<|reserved_special_token_164|>",
|
| 171 |
+
"<|reserved_special_token_165|>",
|
| 172 |
+
"<|reserved_special_token_166|>",
|
| 173 |
+
"<|reserved_special_token_167|>",
|
| 174 |
+
"<|reserved_special_token_168|>",
|
| 175 |
+
"<|reserved_special_token_169|>",
|
| 176 |
+
"<|reserved_special_token_170|>",
|
| 177 |
+
"<|reserved_special_token_171|>",
|
| 178 |
+
"<|reserved_special_token_172|>",
|
| 179 |
+
"<|reserved_special_token_173|>",
|
| 180 |
+
"<|reserved_special_token_174|>",
|
| 181 |
+
"<|reserved_special_token_175|>",
|
| 182 |
+
"<|reserved_special_token_176|>",
|
| 183 |
+
"<|reserved_special_token_177|>",
|
| 184 |
+
"<|reserved_special_token_178|>",
|
| 185 |
+
"<|reserved_special_token_179|>",
|
| 186 |
+
"<|reserved_special_token_180|>",
|
| 187 |
+
"<|reserved_special_token_181|>",
|
| 188 |
+
"<|reserved_special_token_182|>",
|
| 189 |
+
"<|reserved_special_token_183|>",
|
| 190 |
+
"<|reserved_special_token_184|>",
|
| 191 |
+
"<|reserved_special_token_185|>",
|
| 192 |
+
"<|reserved_special_token_186|>",
|
| 193 |
+
"<|reserved_special_token_187|>",
|
| 194 |
+
"<|reserved_special_token_188|>",
|
| 195 |
+
"<|reserved_special_token_189|>",
|
| 196 |
+
"<|reserved_special_token_190|>",
|
| 197 |
+
"<|reserved_special_token_191|>",
|
| 198 |
+
"<|reserved_special_token_192|>",
|
| 199 |
+
"<|reserved_special_token_193|>",
|
| 200 |
+
"<|reserved_special_token_194|>",
|
| 201 |
+
"<|reserved_special_token_195|>",
|
| 202 |
+
"<|reserved_special_token_196|>",
|
| 203 |
+
"<|reserved_special_token_197|>",
|
| 204 |
+
"<|reserved_special_token_198|>",
|
| 205 |
+
"<|reserved_special_token_199|>",
|
| 206 |
+
"<|reserved_special_token_200|>",
|
| 207 |
+
"<|reserved_special_token_201|>",
|
| 208 |
+
"<|reserved_special_token_202|>",
|
| 209 |
+
"<|reserved_special_token_203|>",
|
| 210 |
+
"<|reserved_special_token_204|>",
|
| 211 |
+
"<|reserved_special_token_205|>",
|
| 212 |
+
"<|reserved_special_token_206|>",
|
| 213 |
+
"<|reserved_special_token_207|>",
|
| 214 |
+
"<|reserved_special_token_208|>",
|
| 215 |
+
"<|reserved_special_token_209|>",
|
| 216 |
+
"<|reserved_special_token_210|>",
|
| 217 |
+
"<|reserved_special_token_211|>",
|
| 218 |
+
"<|reserved_special_token_212|>",
|
| 219 |
+
"<|reserved_special_token_213|>",
|
| 220 |
+
"<|reserved_special_token_214|>",
|
| 221 |
+
"<|reserved_special_token_215|>",
|
| 222 |
+
"<|reserved_special_token_216|>",
|
| 223 |
+
"<|reserved_special_token_217|>",
|
| 224 |
+
"<|reserved_special_token_218|>",
|
| 225 |
+
"<|reserved_special_token_219|>",
|
| 226 |
+
"<|reserved_special_token_220|>",
|
| 227 |
+
"<|reserved_special_token_221|>",
|
| 228 |
+
"<|reserved_special_token_222|>",
|
| 229 |
+
"<|reserved_special_token_223|>",
|
| 230 |
+
"<|reserved_special_token_224|>",
|
| 231 |
+
"<|reserved_special_token_225|>",
|
| 232 |
+
"<|reserved_special_token_226|>",
|
| 233 |
+
"<|reserved_special_token_227|>",
|
| 234 |
+
"<|reserved_special_token_228|>",
|
| 235 |
+
"<|reserved_special_token_229|>",
|
| 236 |
+
"<|reserved_special_token_230|>",
|
| 237 |
+
"<|reserved_special_token_231|>",
|
| 238 |
+
"<|reserved_special_token_232|>",
|
| 239 |
+
"<|reserved_special_token_233|>",
|
| 240 |
+
"<|reserved_special_token_234|>",
|
| 241 |
+
"<|reserved_special_token_235|>",
|
| 242 |
+
"<|reserved_special_token_236|>",
|
| 243 |
+
"<|reserved_special_token_237|>",
|
| 244 |
+
"<|reserved_special_token_238|>",
|
| 245 |
+
"<|reserved_special_token_239|>",
|
| 246 |
+
"<|reserved_special_token_240|>",
|
| 247 |
+
"<|reserved_special_token_241|>",
|
| 248 |
+
"<|reserved_special_token_242|>",
|
| 249 |
+
"<|reserved_special_token_243|>",
|
| 250 |
+
"<|reserved_special_token_244|>",
|
| 251 |
+
"<|reserved_special_token_245|>",
|
| 252 |
+
"<|reserved_special_token_246|>",
|
| 253 |
+
"<|reserved_special_token_247|>",
|
| 254 |
+
"<|reserved_special_token_248|>",
|
| 255 |
+
"<|reserved_special_token_249|>",
|
| 256 |
+
"<|reserved_special_token_250|>",
|
| 257 |
+
"<|reserved_special_token_251|>",
|
| 258 |
+
"<|reserved_special_token_252|>",
|
| 259 |
+
"<|reserved_special_token_253|>",
|
| 260 |
+
"<|reserved_special_token_254|>",
|
| 261 |
+
"<|reserved_special_token_255|>",
|
| 262 |
+
"<|reserved_special_token_256|>",
|
| 263 |
+
"<|reserved_special_token_257|>",
|
| 264 |
+
"<|reserved_special_token_258|>",
|
| 265 |
+
"<|reserved_special_token_259|>",
|
| 266 |
+
"<|reserved_special_token_260|>",
|
| 267 |
+
"<|reserved_special_token_261|>",
|
| 268 |
+
"<|reserved_special_token_262|>",
|
| 269 |
+
"<|reserved_special_token_263|>",
|
| 270 |
+
"<|reserved_special_token_264|>",
|
| 271 |
+
"<|reserved_special_token_265|>",
|
| 272 |
+
"<|reserved_special_token_266|>",
|
| 273 |
+
"<|reserved_special_token_267|>",
|
| 274 |
+
"<|reserved_special_token_268|>",
|
| 275 |
+
"<|reserved_special_token_269|>",
|
| 276 |
+
"<|reserved_special_token_270|>",
|
| 277 |
+
"<|reserved_special_token_271|>",
|
| 278 |
+
"<|reserved_special_token_272|>",
|
| 279 |
+
"<|reserved_special_token_273|>",
|
| 280 |
+
"<|reserved_special_token_274|>",
|
| 281 |
+
"<|reserved_special_token_275|>",
|
| 282 |
+
"<|reserved_special_token_276|>",
|
| 283 |
+
"<|reserved_special_token_277|>",
|
| 284 |
+
"<|reserved_special_token_278|>",
|
| 285 |
+
"<|reserved_special_token_279|>",
|
| 286 |
+
"<|reserved_special_token_280|>",
|
| 287 |
+
"<|reserved_special_token_281|>",
|
| 288 |
+
"<|reserved_special_token_282|>",
|
| 289 |
+
"<|reserved_special_token_283|>",
|
| 290 |
+
"<|reserved_special_token_284|>",
|
| 291 |
+
"<|reserved_special_token_285|>",
|
| 292 |
+
"<|reserved_special_token_286|>",
|
| 293 |
+
"<|reserved_special_token_287|>",
|
| 294 |
+
"<|reserved_special_token_288|>",
|
| 295 |
+
"<|reserved_special_token_289|>",
|
| 296 |
+
"<|reserved_special_token_290|>",
|
| 297 |
+
"<|reserved_special_token_291|>",
|
| 298 |
+
"<|reserved_special_token_292|>",
|
| 299 |
+
"<|reserved_special_token_293|>",
|
| 300 |
+
"<|reserved_special_token_294|>",
|
| 301 |
+
"<|reserved_special_token_295|>",
|
| 302 |
+
"<|reserved_special_token_296|>",
|
| 303 |
+
"<|reserved_special_token_297|>",
|
| 304 |
+
"<|reserved_special_token_298|>",
|
| 305 |
+
"<|reserved_special_token_299|>",
|
| 306 |
+
"<|reserved_special_token_300|>",
|
| 307 |
+
"<|reserved_special_token_301|>",
|
| 308 |
+
"<|reserved_special_token_302|>",
|
| 309 |
+
"<|reserved_special_token_303|>",
|
| 310 |
+
"<|reserved_special_token_304|>",
|
| 311 |
+
"<|reserved_special_token_305|>",
|
| 312 |
+
"<|reserved_special_token_306|>",
|
| 313 |
+
"<|reserved_special_token_307|>",
|
| 314 |
+
"<|reserved_special_token_308|>",
|
| 315 |
+
"<|reserved_special_token_309|>",
|
| 316 |
+
"<|reserved_special_token_310|>",
|
| 317 |
+
"<|reserved_special_token_311|>",
|
| 318 |
+
"<|reserved_special_token_312|>",
|
| 319 |
+
"<|reserved_special_token_313|>",
|
| 320 |
+
"<|reserved_special_token_314|>",
|
| 321 |
+
"<|reserved_special_token_315|>",
|
| 322 |
+
"<|reserved_special_token_316|>",
|
| 323 |
+
"<|reserved_special_token_317|>",
|
| 324 |
+
"<|reserved_special_token_318|>",
|
| 325 |
+
"<|reserved_special_token_319|>",
|
| 326 |
+
"<|reserved_special_token_320|>",
|
| 327 |
+
"<|reserved_special_token_321|>",
|
| 328 |
+
"<|reserved_special_token_322|>",
|
| 329 |
+
"<|reserved_special_token_323|>",
|
| 330 |
+
"<|reserved_special_token_324|>",
|
| 331 |
+
"<|reserved_special_token_325|>",
|
| 332 |
+
"<|reserved_special_token_326|>",
|
| 333 |
+
"<|reserved_special_token_327|>",
|
| 334 |
+
"<|reserved_special_token_328|>",
|
| 335 |
+
"<|reserved_special_token_329|>",
|
| 336 |
+
"<|reserved_special_token_330|>",
|
| 337 |
+
"<|reserved_special_token_331|>",
|
| 338 |
+
"<|reserved_special_token_332|>",
|
| 339 |
+
"<|reserved_special_token_333|>",
|
| 340 |
+
"<|reserved_special_token_334|>",
|
| 341 |
+
"<|reserved_special_token_335|>",
|
| 342 |
+
"<|reserved_special_token_336|>",
|
| 343 |
+
"<|reserved_special_token_337|>",
|
| 344 |
+
"<|reserved_special_token_338|>",
|
| 345 |
+
"<|reserved_special_token_339|>",
|
| 346 |
+
"<|reserved_special_token_340|>",
|
| 347 |
+
"<|reserved_special_token_341|>",
|
| 348 |
+
"<|reserved_special_token_342|>",
|
| 349 |
+
"<|reserved_special_token_343|>",
|
| 350 |
+
"<|reserved_special_token_344|>",
|
| 351 |
+
"<|reserved_special_token_345|>",
|
| 352 |
+
"<|reserved_special_token_346|>",
|
| 353 |
+
"<|reserved_special_token_347|>",
|
| 354 |
+
"<|reserved_special_token_348|>",
|
| 355 |
+
"<|reserved_special_token_349|>",
|
| 356 |
+
"<|reserved_special_token_350|>",
|
| 357 |
+
"<|reserved_special_token_351|>",
|
| 358 |
+
"<|reserved_special_token_352|>",
|
| 359 |
+
"<|reserved_special_token_353|>",
|
| 360 |
+
"<|reserved_special_token_354|>",
|
| 361 |
+
"<|reserved_special_token_355|>",
|
| 362 |
+
"<|reserved_special_token_356|>",
|
| 363 |
+
"<|reserved_special_token_357|>",
|
| 364 |
+
"<|reserved_special_token_358|>",
|
| 365 |
+
"<|reserved_special_token_359|>",
|
| 366 |
+
"<|reserved_special_token_360|>",
|
| 367 |
+
"<|reserved_special_token_361|>",
|
| 368 |
+
"<|reserved_special_token_362|>",
|
| 369 |
+
"<|reserved_special_token_363|>",
|
| 370 |
+
"<|reserved_special_token_364|>",
|
| 371 |
+
"<|reserved_special_token_365|>",
|
| 372 |
+
"<|reserved_special_token_366|>",
|
| 373 |
+
"<|reserved_special_token_367|>",
|
| 374 |
+
"<|reserved_special_token_368|>",
|
| 375 |
+
"<|reserved_special_token_369|>",
|
| 376 |
+
"<|reserved_special_token_370|>",
|
| 377 |
+
"<|reserved_special_token_371|>",
|
| 378 |
+
"<|reserved_special_token_372|>",
|
| 379 |
+
"<|reserved_special_token_373|>",
|
| 380 |
+
"<|reserved_special_token_374|>",
|
| 381 |
+
"<|reserved_special_token_375|>",
|
| 382 |
+
"<|reserved_special_token_376|>",
|
| 383 |
+
"<|reserved_special_token_377|>",
|
| 384 |
+
"<|reserved_special_token_378|>",
|
| 385 |
+
"<|reserved_special_token_379|>",
|
| 386 |
+
"<|reserved_special_token_380|>",
|
| 387 |
+
"<|reserved_special_token_381|>",
|
| 388 |
+
"<|reserved_special_token_382|>",
|
| 389 |
+
"<|reserved_special_token_383|>",
|
| 390 |
+
"<|reserved_special_token_384|>",
|
| 391 |
+
"<|reserved_special_token_385|>",
|
| 392 |
+
"<|reserved_special_token_386|>",
|
| 393 |
+
"<|reserved_special_token_387|>",
|
| 394 |
+
"<|reserved_special_token_388|>",
|
| 395 |
+
"<|reserved_special_token_389|>",
|
| 396 |
+
"<|reserved_special_token_390|>",
|
| 397 |
+
"<|reserved_special_token_391|>",
|
| 398 |
+
"<|reserved_special_token_392|>",
|
| 399 |
+
"<|reserved_special_token_393|>",
|
| 400 |
+
"<|reserved_special_token_394|>",
|
| 401 |
+
"<|reserved_special_token_395|>",
|
| 402 |
+
"<|reserved_special_token_396|>",
|
| 403 |
+
"<|reserved_special_token_397|>",
|
| 404 |
+
"<|reserved_special_token_398|>",
|
| 405 |
+
"<|reserved_special_token_399|>",
|
| 406 |
+
"<|reserved_special_token_400|>",
|
| 407 |
+
"<|reserved_special_token_401|>",
|
| 408 |
+
"<|reserved_special_token_402|>",
|
| 409 |
+
"<|reserved_special_token_403|>",
|
| 410 |
+
"<|reserved_special_token_404|>",
|
| 411 |
+
"<|reserved_special_token_405|>",
|
| 412 |
+
"<|reserved_special_token_406|>",
|
| 413 |
+
"<|reserved_special_token_407|>",
|
| 414 |
+
"<|reserved_special_token_408|>",
|
| 415 |
+
"<|reserved_special_token_409|>",
|
| 416 |
+
"<|reserved_special_token_410|>",
|
| 417 |
+
"<|reserved_special_token_411|>",
|
| 418 |
+
"<|reserved_special_token_412|>",
|
| 419 |
+
"<|reserved_special_token_413|>",
|
| 420 |
+
"<|reserved_special_token_414|>",
|
| 421 |
+
"<|reserved_special_token_415|>",
|
| 422 |
+
"<|reserved_special_token_416|>",
|
| 423 |
+
"<|reserved_special_token_417|>",
|
| 424 |
+
"<|reserved_special_token_418|>",
|
| 425 |
+
"<|reserved_special_token_419|>",
|
| 426 |
+
"<|reserved_special_token_420|>",
|
| 427 |
+
"<|reserved_special_token_421|>",
|
| 428 |
+
"<|reserved_special_token_422|>",
|
| 429 |
+
"<|reserved_special_token_423|>",
|
| 430 |
+
"<|reserved_special_token_424|>",
|
| 431 |
+
"<|reserved_special_token_425|>",
|
| 432 |
+
"<|reserved_special_token_426|>",
|
| 433 |
+
"<|reserved_special_token_427|>",
|
| 434 |
+
"<|reserved_special_token_428|>",
|
| 435 |
+
"<|reserved_special_token_429|>",
|
| 436 |
+
"<|reserved_special_token_430|>",
|
| 437 |
+
"<|reserved_special_token_431|>",
|
| 438 |
+
"<|reserved_special_token_432|>",
|
| 439 |
+
"<|reserved_special_token_433|>",
|
| 440 |
+
"<|reserved_special_token_434|>",
|
| 441 |
+
"<|reserved_special_token_435|>",
|
| 442 |
+
"<|reserved_special_token_436|>",
|
| 443 |
+
"<|reserved_special_token_437|>",
|
| 444 |
+
"<|reserved_special_token_438|>",
|
| 445 |
+
"<|reserved_special_token_439|>",
|
| 446 |
+
"<|reserved_special_token_440|>",
|
| 447 |
+
"<|reserved_special_token_441|>",
|
| 448 |
+
"<|reserved_special_token_442|>",
|
| 449 |
+
"<|reserved_special_token_443|>",
|
| 450 |
+
"<|reserved_special_token_444|>",
|
| 451 |
+
"<|reserved_special_token_445|>",
|
| 452 |
+
"<|reserved_special_token_446|>",
|
| 453 |
+
"<|reserved_special_token_447|>",
|
| 454 |
+
"<|reserved_special_token_448|>",
|
| 455 |
+
"<|reserved_special_token_449|>",
|
| 456 |
+
"<|reserved_special_token_450|>",
|
| 457 |
+
"<|reserved_special_token_451|>",
|
| 458 |
+
"<|reserved_special_token_452|>",
|
| 459 |
+
"<|reserved_special_token_453|>",
|
| 460 |
+
"<|reserved_special_token_454|>",
|
| 461 |
+
"<|reserved_special_token_455|>",
|
| 462 |
+
"<|reserved_special_token_456|>",
|
| 463 |
+
"<|reserved_special_token_457|>",
|
| 464 |
+
"<|reserved_special_token_458|>",
|
| 465 |
+
"<|reserved_special_token_459|>",
|
| 466 |
+
"<|reserved_special_token_460|>",
|
| 467 |
+
"<|reserved_special_token_461|>",
|
| 468 |
+
"<|reserved_special_token_462|>",
|
| 469 |
+
"<|reserved_special_token_463|>",
|
| 470 |
+
"<|reserved_special_token_464|>",
|
| 471 |
+
"<|reserved_special_token_465|>",
|
| 472 |
+
"<|reserved_special_token_466|>",
|
| 473 |
+
"<|reserved_special_token_467|>",
|
| 474 |
+
"<|reserved_special_token_468|>",
|
| 475 |
+
"<|reserved_special_token_469|>",
|
| 476 |
+
"<|reserved_special_token_470|>",
|
| 477 |
+
"<|reserved_special_token_471|>",
|
| 478 |
+
"<|reserved_special_token_472|>",
|
| 479 |
+
"<|reserved_special_token_473|>",
|
| 480 |
+
"<|reserved_special_token_474|>",
|
| 481 |
+
"<|reserved_special_token_475|>",
|
| 482 |
+
"<|reserved_special_token_476|>",
|
| 483 |
+
"<|reserved_special_token_477|>",
|
| 484 |
+
"<|reserved_special_token_478|>",
|
| 485 |
+
"<|reserved_special_token_479|>",
|
| 486 |
+
"<|reserved_special_token_480|>",
|
| 487 |
+
"<|reserved_special_token_481|>",
|
| 488 |
+
"<|reserved_special_token_482|>",
|
| 489 |
+
"<|reserved_special_token_483|>",
|
| 490 |
+
"<|reserved_special_token_484|>",
|
| 491 |
+
"<|reserved_special_token_485|>",
|
| 492 |
+
"<|reserved_special_token_486|>",
|
| 493 |
+
"<|reserved_special_token_487|>",
|
| 494 |
+
"<|reserved_special_token_488|>",
|
| 495 |
+
"<|reserved_special_token_489|>",
|
| 496 |
+
"<|reserved_special_token_490|>",
|
| 497 |
+
"<|reserved_special_token_491|>",
|
| 498 |
+
"<|reserved_special_token_492|>",
|
| 499 |
+
"<|reserved_special_token_493|>",
|
| 500 |
+
"<|reserved_special_token_494|>",
|
| 501 |
+
"<|reserved_special_token_495|>",
|
| 502 |
+
"<|reserved_special_token_496|>",
|
| 503 |
+
"<|reserved_special_token_497|>",
|
| 504 |
+
"<|reserved_special_token_498|>",
|
| 505 |
+
"<|reserved_special_token_499|>",
|
| 506 |
+
"<|reserved_special_token_500|>",
|
| 507 |
+
"<|reserved_special_token_501|>",
|
| 508 |
+
"<|reserved_special_token_502|>",
|
| 509 |
+
"<|reserved_special_token_503|>",
|
| 510 |
+
"<|reserved_special_token_504|>",
|
| 511 |
+
"<|reserved_special_token_505|>",
|
| 512 |
+
"<|reserved_special_token_506|>",
|
| 513 |
+
"<|reserved_special_token_507|>",
|
| 514 |
+
"<|reserved_special_token_508|>",
|
| 515 |
+
"<|reserved_special_token_509|>",
|
| 516 |
+
"<|reserved_special_token_510|>",
|
| 517 |
+
"<|reserved_special_token_511|>",
|
| 518 |
+
"<|reserved_special_token_512|>",
|
| 519 |
+
"<|reserved_special_token_513|>",
|
| 520 |
+
"<|reserved_special_token_514|>",
|
| 521 |
+
"<|reserved_special_token_515|>",
|
| 522 |
+
"<|reserved_special_token_516|>",
|
| 523 |
+
"<|reserved_special_token_517|>",
|
| 524 |
+
"<|reserved_special_token_518|>",
|
| 525 |
+
"<|reserved_special_token_519|>",
|
| 526 |
+
"<|reserved_special_token_520|>",
|
| 527 |
+
"<|reserved_special_token_521|>",
|
| 528 |
+
"<|reserved_special_token_522|>",
|
| 529 |
+
"<|reserved_special_token_523|>",
|
| 530 |
+
"<|reserved_special_token_524|>",
|
| 531 |
+
"<|reserved_special_token_525|>",
|
| 532 |
+
"<|reserved_special_token_526|>",
|
| 533 |
+
"<|reserved_special_token_527|>",
|
| 534 |
+
"<|reserved_special_token_528|>",
|
| 535 |
+
"<|reserved_special_token_529|>",
|
| 536 |
+
"<|reserved_special_token_530|>",
|
| 537 |
+
"<|reserved_special_token_531|>",
|
| 538 |
+
"<|reserved_special_token_532|>",
|
| 539 |
+
"<|reserved_special_token_533|>",
|
| 540 |
+
"<|reserved_special_token_534|>",
|
| 541 |
+
"<|reserved_special_token_535|>",
|
| 542 |
+
"<|reserved_special_token_536|>",
|
| 543 |
+
"<|reserved_special_token_537|>",
|
| 544 |
+
"<|reserved_special_token_538|>",
|
| 545 |
+
"<|reserved_special_token_539|>",
|
| 546 |
+
"<|reserved_special_token_540|>",
|
| 547 |
+
"<|reserved_special_token_541|>",
|
| 548 |
+
"<|reserved_special_token_542|>",
|
| 549 |
+
"<|reserved_special_token_543|>",
|
| 550 |
+
"<|reserved_special_token_544|>",
|
| 551 |
+
"<|reserved_special_token_545|>",
|
| 552 |
+
"<|reserved_special_token_546|>",
|
| 553 |
+
"<|reserved_special_token_547|>",
|
| 554 |
+
"<|reserved_special_token_548|>",
|
| 555 |
+
"<|reserved_special_token_549|>",
|
| 556 |
+
"<|reserved_special_token_550|>",
|
| 557 |
+
"<|reserved_special_token_551|>",
|
| 558 |
+
"<|reserved_special_token_552|>",
|
| 559 |
+
"<|reserved_special_token_553|>",
|
| 560 |
+
"<|reserved_special_token_554|>",
|
| 561 |
+
"<|reserved_special_token_555|>",
|
| 562 |
+
"<|reserved_special_token_556|>",
|
| 563 |
+
"<|reserved_special_token_557|>",
|
| 564 |
+
"<|reserved_special_token_558|>",
|
| 565 |
+
"<|reserved_special_token_559|>",
|
| 566 |
+
"<|reserved_special_token_560|>",
|
| 567 |
+
"<|reserved_special_token_561|>",
|
| 568 |
+
"<|reserved_special_token_562|>",
|
| 569 |
+
"<|reserved_special_token_563|>",
|
| 570 |
+
"<|reserved_special_token_564|>",
|
| 571 |
+
"<|reserved_special_token_565|>",
|
| 572 |
+
"<|reserved_special_token_566|>",
|
| 573 |
+
"<|reserved_special_token_567|>",
|
| 574 |
+
"<|reserved_special_token_568|>",
|
| 575 |
+
"<|reserved_special_token_569|>",
|
| 576 |
+
"<|reserved_special_token_570|>",
|
| 577 |
+
"<|reserved_special_token_571|>",
|
| 578 |
+
"<|reserved_special_token_572|>",
|
| 579 |
+
"<|reserved_special_token_573|>",
|
| 580 |
+
"<|reserved_special_token_574|>",
|
| 581 |
+
"<|reserved_special_token_575|>",
|
| 582 |
+
"<|reserved_special_token_576|>",
|
| 583 |
+
"<|reserved_special_token_577|>",
|
| 584 |
+
"<|reserved_special_token_578|>",
|
| 585 |
+
"<|reserved_special_token_579|>",
|
| 586 |
+
"<|reserved_special_token_580|>",
|
| 587 |
+
"<|reserved_special_token_581|>",
|
| 588 |
+
"<|reserved_special_token_582|>",
|
| 589 |
+
"<|reserved_special_token_583|>",
|
| 590 |
+
"<|reserved_special_token_584|>",
|
| 591 |
+
"<|reserved_special_token_585|>",
|
| 592 |
+
"<|reserved_special_token_586|>",
|
| 593 |
+
"<|reserved_special_token_587|>",
|
| 594 |
+
"<|reserved_special_token_588|>",
|
| 595 |
+
"<|reserved_special_token_589|>",
|
| 596 |
+
"<|reserved_special_token_590|>",
|
| 597 |
+
"<|reserved_special_token_591|>",
|
| 598 |
+
"<|reserved_special_token_592|>",
|
| 599 |
+
"<|reserved_special_token_593|>",
|
| 600 |
+
"<|reserved_special_token_594|>",
|
| 601 |
+
"<|reserved_special_token_595|>",
|
| 602 |
+
"<|reserved_special_token_596|>",
|
| 603 |
+
"<|reserved_special_token_597|>",
|
| 604 |
+
"<|reserved_special_token_598|>",
|
| 605 |
+
"<|reserved_special_token_599|>",
|
| 606 |
+
"<|reserved_special_token_600|>",
|
| 607 |
+
"<|reserved_special_token_601|>",
|
| 608 |
+
"<|reserved_special_token_602|>",
|
| 609 |
+
"<|reserved_special_token_603|>",
|
| 610 |
+
"<|reserved_special_token_604|>",
|
| 611 |
+
"<|reserved_special_token_605|>",
|
| 612 |
+
"<|reserved_special_token_606|>",
|
| 613 |
+
"<|reserved_special_token_607|>",
|
| 614 |
+
"<|reserved_special_token_608|>",
|
| 615 |
+
"<|reserved_special_token_609|>",
|
| 616 |
+
"<|reserved_special_token_610|>",
|
| 617 |
+
"<|reserved_special_token_611|>",
|
| 618 |
+
"<|reserved_special_token_612|>",
|
| 619 |
+
"<|reserved_special_token_613|>",
|
| 620 |
+
"<|reserved_special_token_614|>",
|
| 621 |
+
"<|reserved_special_token_615|>",
|
| 622 |
+
"<|reserved_special_token_616|>",
|
| 623 |
+
"<|reserved_special_token_617|>",
|
| 624 |
+
"<|reserved_special_token_618|>",
|
| 625 |
+
"<|reserved_special_token_619|>",
|
| 626 |
+
"<|reserved_special_token_620|>",
|
| 627 |
+
"<|reserved_special_token_621|>",
|
| 628 |
+
"<|reserved_special_token_622|>",
|
| 629 |
+
"<|reserved_special_token_623|>",
|
| 630 |
+
"<|reserved_special_token_624|>",
|
| 631 |
+
"<|reserved_special_token_625|>",
|
| 632 |
+
"<|reserved_special_token_626|>",
|
| 633 |
+
"<|reserved_special_token_627|>",
|
| 634 |
+
"<|reserved_special_token_628|>",
|
| 635 |
+
"<|reserved_special_token_629|>",
|
| 636 |
+
"<|reserved_special_token_630|>",
|
| 637 |
+
"<|reserved_special_token_631|>",
|
| 638 |
+
"<|reserved_special_token_632|>",
|
| 639 |
+
"<|reserved_special_token_633|>",
|
| 640 |
+
"<|reserved_special_token_634|>",
|
| 641 |
+
"<|reserved_special_token_635|>",
|
| 642 |
+
"<|reserved_special_token_636|>",
|
| 643 |
+
"<|reserved_special_token_637|>",
|
| 644 |
+
"<|reserved_special_token_638|>",
|
| 645 |
+
"<|reserved_special_token_639|>",
|
| 646 |
+
"<|reserved_special_token_640|>",
|
| 647 |
+
"<|reserved_special_token_641|>",
|
| 648 |
+
"<|reserved_special_token_642|>",
|
| 649 |
+
"<|reserved_special_token_643|>",
|
| 650 |
+
"<|reserved_special_token_644|>",
|
| 651 |
+
"<|reserved_special_token_645|>",
|
| 652 |
+
"<|reserved_special_token_646|>",
|
| 653 |
+
"<|reserved_special_token_647|>",
|
| 654 |
+
"<|reserved_special_token_648|>",
|
| 655 |
+
"<|reserved_special_token_649|>",
|
| 656 |
+
"<|reserved_special_token_650|>",
|
| 657 |
+
"<|reserved_special_token_651|>",
|
| 658 |
+
"<|reserved_special_token_652|>",
|
| 659 |
+
"<|reserved_special_token_653|>",
|
| 660 |
+
"<|reserved_special_token_654|>",
|
| 661 |
+
"<|reserved_special_token_655|>",
|
| 662 |
+
"<|reserved_special_token_656|>",
|
| 663 |
+
"<|reserved_special_token_657|>",
|
| 664 |
+
"<|reserved_special_token_658|>",
|
| 665 |
+
"<|reserved_special_token_659|>",
|
| 666 |
+
"<|reserved_special_token_660|>",
|
| 667 |
+
"<|reserved_special_token_661|>",
|
| 668 |
+
"<|reserved_special_token_662|>",
|
| 669 |
+
"<|reserved_special_token_663|>",
|
| 670 |
+
"<|reserved_special_token_664|>",
|
| 671 |
+
"<|reserved_special_token_665|>",
|
| 672 |
+
"<|reserved_special_token_666|>",
|
| 673 |
+
"<|reserved_special_token_667|>",
|
| 674 |
+
"<|reserved_special_token_668|>",
|
| 675 |
+
"<|reserved_special_token_669|>",
|
| 676 |
+
"<|reserved_special_token_670|>",
|
| 677 |
+
"<|reserved_special_token_671|>",
|
| 678 |
+
"<|reserved_special_token_672|>",
|
| 679 |
+
"<|reserved_special_token_673|>",
|
| 680 |
+
"<|reserved_special_token_674|>",
|
| 681 |
+
"<|reserved_special_token_675|>",
|
| 682 |
+
"<|reserved_special_token_676|>",
|
| 683 |
+
"<|reserved_special_token_677|>",
|
| 684 |
+
"<|reserved_special_token_678|>",
|
| 685 |
+
"<|reserved_special_token_679|>",
|
| 686 |
+
"<|reserved_special_token_680|>",
|
| 687 |
+
"<|reserved_special_token_681|>",
|
| 688 |
+
"<|reserved_special_token_682|>",
|
| 689 |
+
"<|reserved_special_token_683|>",
|
| 690 |
+
"<|reserved_special_token_684|>",
|
| 691 |
+
"<|reserved_special_token_685|>",
|
| 692 |
+
"<|reserved_special_token_686|>",
|
| 693 |
+
"<|reserved_special_token_687|>",
|
| 694 |
+
"<|reserved_special_token_688|>",
|
| 695 |
+
"<|reserved_special_token_689|>",
|
| 696 |
+
"<|reserved_special_token_690|>",
|
| 697 |
+
"<|reserved_special_token_691|>",
|
| 698 |
+
"<|reserved_special_token_692|>",
|
| 699 |
+
"<|reserved_special_token_693|>",
|
| 700 |
+
"<|reserved_special_token_694|>",
|
| 701 |
+
"<|reserved_special_token_695|>",
|
| 702 |
+
"<|reserved_special_token_696|>",
|
| 703 |
+
"<|reserved_special_token_697|>",
|
| 704 |
+
"<|reserved_special_token_698|>",
|
| 705 |
+
"<|reserved_special_token_699|>",
|
| 706 |
+
"<|reserved_special_token_700|>",
|
| 707 |
+
"<|reserved_special_token_701|>",
|
| 708 |
+
"<|reserved_special_token_702|>",
|
| 709 |
+
"<|reserved_special_token_703|>",
|
| 710 |
+
"<|reserved_special_token_704|>",
|
| 711 |
+
"<|reserved_special_token_705|>",
|
| 712 |
+
"<|reserved_special_token_706|>",
|
| 713 |
+
"<|reserved_special_token_707|>",
|
| 714 |
+
"<|reserved_special_token_708|>",
|
| 715 |
+
"<|reserved_special_token_709|>",
|
| 716 |
+
"<|reserved_special_token_710|>",
|
| 717 |
+
"<|reserved_special_token_711|>",
|
| 718 |
+
"<|reserved_special_token_712|>",
|
| 719 |
+
"<|reserved_special_token_713|>",
|
| 720 |
+
"<|reserved_special_token_714|>",
|
| 721 |
+
"<|reserved_special_token_715|>",
|
| 722 |
+
"<|reserved_special_token_716|>",
|
| 723 |
+
"<|reserved_special_token_717|>",
|
| 724 |
+
"<|reserved_special_token_718|>",
|
| 725 |
+
"<|reserved_special_token_719|>",
|
| 726 |
+
"<|reserved_special_token_720|>",
|
| 727 |
+
"<|reserved_special_token_721|>",
|
| 728 |
+
"<|reserved_special_token_722|>",
|
| 729 |
+
"<|reserved_special_token_723|>",
|
| 730 |
+
"<|reserved_special_token_724|>",
|
| 731 |
+
"<|reserved_special_token_725|>",
|
| 732 |
+
"<|reserved_special_token_726|>",
|
| 733 |
+
"<|reserved_special_token_727|>",
|
| 734 |
+
"<|reserved_special_token_728|>",
|
| 735 |
+
"<|reserved_special_token_729|>",
|
| 736 |
+
"<|reserved_special_token_730|>",
|
| 737 |
+
"<|reserved_special_token_731|>",
|
| 738 |
+
"<|reserved_special_token_732|>",
|
| 739 |
+
"<|reserved_special_token_733|>",
|
| 740 |
+
"<|reserved_special_token_734|>",
|
| 741 |
+
"<|reserved_special_token_735|>",
|
| 742 |
+
"<|reserved_special_token_736|>",
|
| 743 |
+
"<|reserved_special_token_737|>",
|
| 744 |
+
"<|reserved_special_token_738|>",
|
| 745 |
+
"<|reserved_special_token_739|>",
|
| 746 |
+
"<|reserved_special_token_740|>",
|
| 747 |
+
"<|reserved_special_token_741|>",
|
| 748 |
+
"<|reserved_special_token_742|>",
|
| 749 |
+
"<|reserved_special_token_743|>",
|
| 750 |
+
"<|reserved_special_token_744|>",
|
| 751 |
+
"<|reserved_special_token_745|>",
|
| 752 |
+
"<|reserved_special_token_746|>",
|
| 753 |
+
"<|reserved_special_token_747|>",
|
| 754 |
+
"<|reserved_special_token_748|>",
|
| 755 |
+
"<|reserved_special_token_749|>",
|
| 756 |
+
"<|reserved_special_token_750|>",
|
| 757 |
+
"<|reserved_special_token_751|>",
|
| 758 |
+
"<|reserved_special_token_752|>",
|
| 759 |
+
"<|reserved_special_token_753|>",
|
| 760 |
+
"<|reserved_special_token_754|>",
|
| 761 |
+
"<|reserved_special_token_755|>",
|
| 762 |
+
"<|reserved_special_token_756|>",
|
| 763 |
+
"<|reserved_special_token_757|>",
|
| 764 |
+
"<|reserved_special_token_758|>",
|
| 765 |
+
"<|reserved_special_token_759|>",
|
| 766 |
+
"<|reserved_special_token_760|>",
|
| 767 |
+
"<|reserved_special_token_761|>",
|
| 768 |
+
"<|reserved_special_token_762|>",
|
| 769 |
+
"<|reserved_special_token_763|>",
|
| 770 |
+
"<|reserved_special_token_764|>",
|
| 771 |
+
"<|reserved_special_token_765|>",
|
| 772 |
+
"<|reserved_special_token_766|>",
|
| 773 |
+
"<|reserved_special_token_767|>",
|
| 774 |
+
"<|reserved_special_token_768|>",
|
| 775 |
+
"<|reserved_special_token_769|>",
|
| 776 |
+
"<|reserved_special_token_770|>",
|
| 777 |
+
"<|reserved_special_token_771|>",
|
| 778 |
+
"<|reserved_special_token_772|>",
|
| 779 |
+
"<|reserved_special_token_773|>",
|
| 780 |
+
"<|reserved_special_token_774|>",
|
| 781 |
+
"<|reserved_special_token_775|>",
|
| 782 |
+
"<|reserved_special_token_776|>",
|
| 783 |
+
"<|reserved_special_token_777|>",
|
| 784 |
+
"<|reserved_special_token_778|>",
|
| 785 |
+
"<|reserved_special_token_779|>",
|
| 786 |
+
"<|reserved_special_token_780|>",
|
| 787 |
+
"<|reserved_special_token_781|>",
|
| 788 |
+
"<|reserved_special_token_782|>",
|
| 789 |
+
"<|reserved_special_token_783|>",
|
| 790 |
+
"<|reserved_special_token_784|>",
|
| 791 |
+
"<|reserved_special_token_785|>",
|
| 792 |
+
"<|reserved_special_token_786|>",
|
| 793 |
+
"<|reserved_special_token_787|>",
|
| 794 |
+
"<|reserved_special_token_788|>",
|
| 795 |
+
"<|reserved_special_token_789|>",
|
| 796 |
+
"<|reserved_special_token_790|>",
|
| 797 |
+
"<|reserved_special_token_791|>",
|
| 798 |
+
"<|reserved_special_token_792|>",
|
| 799 |
+
"<|reserved_special_token_793|>",
|
| 800 |
+
"<|reserved_special_token_794|>",
|
| 801 |
+
"<|reserved_special_token_795|>",
|
| 802 |
+
"<|reserved_special_token_796|>",
|
| 803 |
+
"<|reserved_special_token_797|>",
|
| 804 |
+
"<|reserved_special_token_798|>",
|
| 805 |
+
"<|reserved_special_token_799|>",
|
| 806 |
+
"<|reserved_special_token_800|>",
|
| 807 |
+
"<|reserved_special_token_801|>",
|
| 808 |
+
"<|reserved_special_token_802|>",
|
| 809 |
+
"<|reserved_special_token_803|>",
|
| 810 |
+
"<|reserved_special_token_804|>",
|
| 811 |
+
"<|reserved_special_token_805|>",
|
| 812 |
+
"<|reserved_special_token_806|>",
|
| 813 |
+
"<|reserved_special_token_807|>",
|
| 814 |
+
"<|reserved_special_token_808|>",
|
| 815 |
+
"<|reserved_special_token_809|>",
|
| 816 |
+
"<|reserved_special_token_810|>",
|
| 817 |
+
"<|reserved_special_token_811|>",
|
| 818 |
+
"<|reserved_special_token_812|>",
|
| 819 |
+
"<|reserved_special_token_813|>",
|
| 820 |
+
"<|reserved_special_token_814|>",
|
| 821 |
+
"<|reserved_special_token_815|>",
|
| 822 |
+
"<|reserved_special_token_816|>",
|
| 823 |
+
"<|reserved_special_token_817|>",
|
| 824 |
+
"<|reserved_special_token_818|>",
|
| 825 |
+
"<|reserved_special_token_819|>",
|
| 826 |
+
"<|reserved_special_token_820|>",
|
| 827 |
+
"<|reserved_special_token_821|>",
|
| 828 |
+
"<|reserved_special_token_822|>",
|
| 829 |
+
"<|reserved_special_token_823|>",
|
| 830 |
+
"<|reserved_special_token_824|>",
|
| 831 |
+
"<|reserved_special_token_825|>",
|
| 832 |
+
"<|reserved_special_token_826|>",
|
| 833 |
+
"<|reserved_special_token_827|>",
|
| 834 |
+
"<|reserved_special_token_828|>",
|
| 835 |
+
"<|reserved_special_token_829|>",
|
| 836 |
+
"<|reserved_special_token_830|>",
|
| 837 |
+
"<|reserved_special_token_831|>",
|
| 838 |
+
"<|reserved_special_token_832|>",
|
| 839 |
+
"<|reserved_special_token_833|>",
|
| 840 |
+
"<|reserved_special_token_834|>",
|
| 841 |
+
"<|reserved_special_token_835|>",
|
| 842 |
+
"<|reserved_special_token_836|>",
|
| 843 |
+
"<|reserved_special_token_837|>",
|
| 844 |
+
"<|reserved_special_token_838|>",
|
| 845 |
+
"<|reserved_special_token_839|>",
|
| 846 |
+
"<|reserved_special_token_840|>",
|
| 847 |
+
"<|reserved_special_token_841|>",
|
| 848 |
+
"<|reserved_special_token_842|>",
|
| 849 |
+
"<|reserved_special_token_843|>",
|
| 850 |
+
"<|reserved_special_token_844|>",
|
| 851 |
+
"<|reserved_special_token_845|>",
|
| 852 |
+
"<|reserved_special_token_846|>",
|
| 853 |
+
"<|reserved_special_token_847|>",
|
| 854 |
+
"<|reserved_special_token_848|>",
|
| 855 |
+
"<|reserved_special_token_849|>",
|
| 856 |
+
"<|reserved_special_token_850|>",
|
| 857 |
+
"<|reserved_special_token_851|>",
|
| 858 |
+
"<|reserved_special_token_852|>",
|
| 859 |
+
"<|reserved_special_token_853|>",
|
| 860 |
+
"<|reserved_special_token_854|>",
|
| 861 |
+
"<|reserved_special_token_855|>",
|
| 862 |
+
"<|reserved_special_token_856|>",
|
| 863 |
+
"<|reserved_special_token_857|>",
|
| 864 |
+
"<|reserved_special_token_858|>",
|
| 865 |
+
"<|reserved_special_token_859|>",
|
| 866 |
+
"<|reserved_special_token_860|>",
|
| 867 |
+
"<|reserved_special_token_861|>",
|
| 868 |
+
"<|reserved_special_token_862|>",
|
| 869 |
+
"<|reserved_special_token_863|>",
|
| 870 |
+
"<|reserved_special_token_864|>",
|
| 871 |
+
"<|reserved_special_token_865|>",
|
| 872 |
+
"<|reserved_special_token_866|>",
|
| 873 |
+
"<|reserved_special_token_867|>",
|
| 874 |
+
"<|reserved_special_token_868|>",
|
| 875 |
+
"<|reserved_special_token_869|>",
|
| 876 |
+
"<|reserved_special_token_870|>",
|
| 877 |
+
"<|reserved_special_token_871|>",
|
| 878 |
+
"<|reserved_special_token_872|>",
|
| 879 |
+
"<|reserved_special_token_873|>",
|
| 880 |
+
"<|reserved_special_token_874|>",
|
| 881 |
+
"<|reserved_special_token_875|>",
|
| 882 |
+
"<|reserved_special_token_876|>",
|
| 883 |
+
"<|reserved_special_token_877|>",
|
| 884 |
+
"<|reserved_special_token_878|>",
|
| 885 |
+
"<|reserved_special_token_879|>",
|
| 886 |
+
"<|reserved_special_token_880|>",
|
| 887 |
+
"<|reserved_special_token_881|>",
|
| 888 |
+
"<|reserved_special_token_882|>",
|
| 889 |
+
"<|reserved_special_token_883|>",
|
| 890 |
+
"<|reserved_special_token_884|>",
|
| 891 |
+
"<|reserved_special_token_885|>",
|
| 892 |
+
"<|reserved_special_token_886|>",
|
| 893 |
+
"<|reserved_special_token_887|>",
|
| 894 |
+
"<|reserved_special_token_888|>",
|
| 895 |
+
"<|reserved_special_token_889|>",
|
| 896 |
+
"<|reserved_special_token_890|>",
|
| 897 |
+
"<|reserved_special_token_891|>",
|
| 898 |
+
"<|reserved_special_token_892|>",
|
| 899 |
+
"<|reserved_special_token_893|>",
|
| 900 |
+
"<|reserved_special_token_894|>",
|
| 901 |
+
"<|reserved_special_token_895|>",
|
| 902 |
+
"<|reserved_special_token_896|>",
|
| 903 |
+
"<|reserved_special_token_897|>",
|
| 904 |
+
"<|reserved_special_token_898|>",
|
| 905 |
+
"<|reserved_special_token_899|>",
|
| 906 |
+
"<|reserved_special_token_900|>",
|
| 907 |
+
"<|reserved_special_token_901|>",
|
| 908 |
+
"<|reserved_special_token_902|>",
|
| 909 |
+
"<|reserved_special_token_903|>",
|
| 910 |
+
"<|reserved_special_token_904|>",
|
| 911 |
+
"<|reserved_special_token_905|>",
|
| 912 |
+
"<|reserved_special_token_906|>",
|
| 913 |
+
"<|reserved_special_token_907|>",
|
| 914 |
+
"<|reserved_special_token_908|>",
|
| 915 |
+
"<|reserved_special_token_909|>",
|
| 916 |
+
"<|reserved_special_token_910|>",
|
| 917 |
+
"<|reserved_special_token_911|>",
|
| 918 |
+
"<|reserved_special_token_912|>",
|
| 919 |
+
"<|reserved_special_token_913|>",
|
| 920 |
+
"<|reserved_special_token_914|>",
|
| 921 |
+
"<|reserved_special_token_915|>",
|
| 922 |
+
"<|reserved_special_token_916|>",
|
| 923 |
+
"<|reserved_special_token_917|>",
|
| 924 |
+
"<|reserved_special_token_918|>",
|
| 925 |
+
"<|reserved_special_token_919|>",
|
| 926 |
+
"<|reserved_special_token_920|>",
|
| 927 |
+
"<|reserved_special_token_921|>",
|
| 928 |
+
"<|reserved_special_token_922|>",
|
| 929 |
+
"<|reserved_special_token_923|>",
|
| 930 |
+
"<|reserved_special_token_924|>",
|
| 931 |
+
"<|reserved_special_token_925|>",
|
| 932 |
+
"<|reserved_special_token_926|>",
|
| 933 |
+
"<|reserved_special_token_927|>",
|
| 934 |
+
"<|reserved_special_token_928|>",
|
| 935 |
+
"<|reserved_special_token_929|>",
|
| 936 |
+
"<|reserved_special_token_930|>",
|
| 937 |
+
"<|reserved_special_token_931|>",
|
| 938 |
+
"<|reserved_special_token_932|>",
|
| 939 |
+
"<|reserved_special_token_933|>",
|
| 940 |
+
"<|reserved_special_token_934|>",
|
| 941 |
+
"<|reserved_special_token_935|>",
|
| 942 |
+
"<|reserved_special_token_936|>",
|
| 943 |
+
"<|reserved_special_token_937|>",
|
| 944 |
+
"<|reserved_special_token_938|>",
|
| 945 |
+
"<|reserved_special_token_939|>",
|
| 946 |
+
"<|reserved_special_token_940|>",
|
| 947 |
+
"<|reserved_special_token_941|>",
|
| 948 |
+
"<|reserved_special_token_942|>",
|
| 949 |
+
"<|reserved_special_token_943|>",
|
| 950 |
+
"<|reserved_special_token_944|>",
|
| 951 |
+
"<|reserved_special_token_945|>",
|
| 952 |
+
"<|reserved_special_token_946|>",
|
| 953 |
+
"<|reserved_special_token_947|>",
|
| 954 |
+
"<|reserved_special_token_948|>",
|
| 955 |
+
"<|reserved_special_token_949|>",
|
| 956 |
+
"<|reserved_special_token_950|>",
|
| 957 |
+
"<|reserved_special_token_951|>",
|
| 958 |
+
"<|reserved_special_token_952|>",
|
| 959 |
+
"<|reserved_special_token_953|>",
|
| 960 |
+
"<|reserved_special_token_954|>",
|
| 961 |
+
"<|reserved_special_token_955|>",
|
| 962 |
+
"<|reserved_special_token_956|>",
|
| 963 |
+
"<|reserved_special_token_957|>",
|
| 964 |
+
"<|reserved_special_token_958|>",
|
| 965 |
+
"<|reserved_special_token_959|>",
|
| 966 |
+
"<|reserved_special_token_960|>",
|
| 967 |
+
"<|reserved_special_token_961|>",
|
| 968 |
+
"<|reserved_special_token_962|>",
|
| 969 |
+
"<|reserved_special_token_963|>",
|
| 970 |
+
"<|reserved_special_token_964|>",
|
| 971 |
+
"<|reserved_special_token_965|>",
|
| 972 |
+
"<|reserved_special_token_966|>",
|
| 973 |
+
"<|reserved_special_token_967|>",
|
| 974 |
+
"<|reserved_special_token_968|>",
|
| 975 |
+
"<|reserved_special_token_969|>",
|
| 976 |
+
"<|reserved_special_token_970|>",
|
| 977 |
+
"<|reserved_special_token_971|>",
|
| 978 |
+
"<|reserved_special_token_972|>",
|
| 979 |
+
"<|reserved_special_token_973|>",
|
| 980 |
+
"<|reserved_special_token_974|>",
|
| 981 |
+
"<|reserved_special_token_975|>",
|
| 982 |
+
"<|reserved_special_token_976|>",
|
| 983 |
+
"<|reserved_special_token_977|>",
|
| 984 |
+
"<|reserved_special_token_978|>",
|
| 985 |
+
"<|reserved_special_token_979|>",
|
| 986 |
+
"<|reserved_special_token_980|>",
|
| 987 |
+
"<|reserved_special_token_981|>",
|
| 988 |
+
"<|reserved_special_token_982|>",
|
| 989 |
+
"<|reserved_special_token_983|>",
|
| 990 |
+
"<|reserved_special_token_984|>",
|
| 991 |
+
"<|reserved_special_token_985|>",
|
| 992 |
+
"<|reserved_special_token_986|>",
|
| 993 |
+
"<|reserved_special_token_987|>",
|
| 994 |
+
"<|reserved_special_token_988|>",
|
| 995 |
+
"<|reserved_special_token_989|>",
|
| 996 |
+
"<|reserved_special_token_990|>",
|
| 997 |
+
"<|reserved_special_token_991|>",
|
| 998 |
+
"<|reserved_special_token_992|>",
|
| 999 |
+
"<|reserved_special_token_993|>",
|
| 1000 |
+
"<|reserved_special_token_994|>",
|
| 1001 |
+
"<|reserved_special_token_995|>",
|
| 1002 |
+
"<|reserved_special_token_996|>",
|
| 1003 |
+
"<|reserved_special_token_997|>",
|
| 1004 |
+
"<|reserved_special_token_998|>",
|
| 1005 |
+
"<|reserved_special_token_999|>",
|
| 1006 |
+
"<|reserved_special_token_1000|>",
|
| 1007 |
+
"<|reserved_special_token_1001|>",
|
| 1008 |
+
"<|reserved_special_token_1002|>",
|
| 1009 |
+
"<|reserved_special_token_1003|>",
|
| 1010 |
+
"<|reserved_special_token_1004|>",
|
| 1011 |
+
"<|reserved_special_token_1005|>",
|
| 1012 |
+
"<|reserved_special_token_1006|>",
|
| 1013 |
+
"<|reserved_special_token_1007|>",
|
| 1014 |
+
"<|reserved_special_token_1008|>",
|
| 1015 |
+
"<|reserved_special_token_1009|>",
|
| 1016 |
+
"<|reserved_special_token_1010|>",
|
| 1017 |
+
"<|reserved_special_token_1011|>",
|
| 1018 |
+
"<|reserved_special_token_1012|>",
|
| 1019 |
+
"<|reserved_special_token_1013|>",
|
| 1020 |
+
"<|reserved_special_token_1014|>",
|
| 1021 |
+
"<|reserved_special_token_1015|>",
|
| 1022 |
+
"<|reserved_special_token_1016|>",
|
| 1023 |
+
"<|reserved_special_token_1017|>",
|
| 1024 |
+
"<|reserved_special_token_1018|>",
|
| 1025 |
+
"<|reserved_special_token_1019|>",
|
| 1026 |
+
"<|reserved_special_token_1020|>",
|
| 1027 |
+
"<|reserved_special_token_1021|>",
|
| 1028 |
+
"<|reserved_special_token_1022|>",
|
| 1029 |
+
"<|reserved_special_token_1023|>",
|
| 1030 |
+
"<|reserved_special_token_1024|>",
|
| 1031 |
+
"<|reserved_special_token_1025|>",
|
| 1032 |
+
"<|reserved_special_token_1026|>",
|
| 1033 |
+
"<|reserved_special_token_1027|>",
|
| 1034 |
+
"<|reserved_special_token_1028|>",
|
| 1035 |
+
"<|reserved_special_token_1029|>",
|
| 1036 |
+
"<|reserved_special_token_1030|>",
|
| 1037 |
+
"<|reserved_special_token_1031|>",
|
| 1038 |
+
"<|reserved_special_token_1032|>",
|
| 1039 |
+
"<|reserved_special_token_1033|>",
|
| 1040 |
+
"<|reserved_special_token_1034|>",
|
| 1041 |
+
"<|reserved_special_token_1035|>",
|
| 1042 |
+
"<|reserved_special_token_1036|>",
|
| 1043 |
+
"<|reserved_special_token_1037|>",
|
| 1044 |
+
"<|reserved_special_token_1038|>",
|
| 1045 |
+
"<|reserved_special_token_1039|>",
|
| 1046 |
+
"<|reserved_special_token_1040|>",
|
| 1047 |
+
"<|reserved_special_token_1041|>",
|
| 1048 |
+
"<|reserved_special_token_1042|>",
|
| 1049 |
+
"<|reserved_special_token_1043|>",
|
| 1050 |
+
"<|reserved_special_token_1044|>",
|
| 1051 |
+
"<|reserved_special_token_1045|>",
|
| 1052 |
+
"<|reserved_special_token_1046|>",
|
| 1053 |
+
"<|reserved_special_token_1047|>",
|
| 1054 |
+
"<|reserved_special_token_1048|>",
|
| 1055 |
+
"<|reserved_special_token_1049|>",
|
| 1056 |
+
"<|reserved_special_token_1050|>",
|
| 1057 |
+
"<|reserved_special_token_1051|>",
|
| 1058 |
+
"<|reserved_special_token_1052|>",
|
| 1059 |
+
"<|reserved_special_token_1053|>",
|
| 1060 |
+
"<|reserved_special_token_1054|>",
|
| 1061 |
+
"<|reserved_special_token_1055|>",
|
| 1062 |
+
"<|reserved_special_token_1056|>",
|
| 1063 |
+
"<|reserved_special_token_1057|>",
|
| 1064 |
+
"<|reserved_special_token_1058|>",
|
| 1065 |
+
"<|reserved_special_token_1059|>",
|
| 1066 |
+
"<|reserved_special_token_1060|>",
|
| 1067 |
+
"<|reserved_special_token_1061|>",
|
| 1068 |
+
"<|reserved_special_token_1062|>",
|
| 1069 |
+
"<|reserved_special_token_1063|>",
|
| 1070 |
+
"<|reserved_special_token_1064|>",
|
| 1071 |
+
"<|reserved_special_token_1065|>",
|
| 1072 |
+
"<|reserved_special_token_1066|>",
|
| 1073 |
+
"<|reserved_special_token_1067|>",
|
| 1074 |
+
"<|reserved_special_token_1068|>",
|
| 1075 |
+
"<|reserved_special_token_1069|>",
|
| 1076 |
+
"<|reserved_special_token_1070|>",
|
| 1077 |
+
"<|reserved_special_token_1071|>",
|
| 1078 |
+
"<|reserved_special_token_1072|>",
|
| 1079 |
+
"<|reserved_special_token_1073|>",
|
| 1080 |
+
"<|reserved_special_token_1074|>",
|
| 1081 |
+
"<|reserved_special_token_1075|>",
|
| 1082 |
+
"<|reserved_special_token_1076|>",
|
| 1083 |
+
"<|reserved_special_token_1077|>",
|
| 1084 |
+
"<|reserved_special_token_1078|>",
|
| 1085 |
+
"<|reserved_special_token_1079|>",
|
| 1086 |
+
"<|reserved_special_token_1080|>",
|
| 1087 |
+
"<|reserved_special_token_1081|>",
|
| 1088 |
+
"<|reserved_special_token_1082|>",
|
| 1089 |
+
"<|reserved_special_token_1083|>",
|
| 1090 |
+
"<|reserved_special_token_1084|>",
|
| 1091 |
+
"<|reserved_special_token_1085|>",
|
| 1092 |
+
"<|reserved_special_token_1086|>",
|
| 1093 |
+
"<|reserved_special_token_1087|>",
|
| 1094 |
+
"<|reserved_special_token_1088|>",
|
| 1095 |
+
"<|reserved_special_token_1089|>",
|
| 1096 |
+
"<|reserved_special_token_1090|>",
|
| 1097 |
+
"<|reserved_special_token_1091|>",
|
| 1098 |
+
"<|reserved_special_token_1092|>",
|
| 1099 |
+
"<|reserved_special_token_1093|>",
|
| 1100 |
+
"<|reserved_special_token_1094|>",
|
| 1101 |
+
"<|reserved_special_token_1095|>",
|
| 1102 |
+
"<|reserved_special_token_1096|>",
|
| 1103 |
+
"<|reserved_special_token_1097|>",
|
| 1104 |
+
"<|reserved_special_token_1098|>",
|
| 1105 |
+
"<|reserved_special_token_1099|>",
|
| 1106 |
+
"<|reserved_special_token_1100|>",
|
| 1107 |
+
"<|reserved_special_token_1101|>",
|
| 1108 |
+
"<|reserved_special_token_1102|>",
|
| 1109 |
+
"<|reserved_special_token_1103|>",
|
| 1110 |
+
"<|reserved_special_token_1104|>",
|
| 1111 |
+
"<|reserved_special_token_1105|>",
|
| 1112 |
+
"<|reserved_special_token_1106|>",
|
| 1113 |
+
"<|reserved_special_token_1107|>",
|
| 1114 |
+
"<|reserved_special_token_1108|>",
|
| 1115 |
+
"<|reserved_special_token_1109|>",
|
| 1116 |
+
"<|reserved_special_token_1110|>",
|
| 1117 |
+
"<|reserved_special_token_1111|>",
|
| 1118 |
+
"<|reserved_special_token_1112|>",
|
| 1119 |
+
"<|reserved_special_token_1113|>",
|
| 1120 |
+
"<|reserved_special_token_1114|>",
|
| 1121 |
+
"<|reserved_special_token_1115|>",
|
| 1122 |
+
"<|reserved_special_token_1116|>",
|
| 1123 |
+
"<|reserved_special_token_1117|>",
|
| 1124 |
+
"<|reserved_special_token_1118|>",
|
| 1125 |
+
"<|reserved_special_token_1119|>",
|
| 1126 |
+
"<|reserved_special_token_1120|>",
|
| 1127 |
+
"<|reserved_special_token_1121|>",
|
| 1128 |
+
"<|reserved_special_token_1122|>",
|
| 1129 |
+
"<|reserved_special_token_1123|>",
|
| 1130 |
+
"<|reserved_special_token_1124|>",
|
| 1131 |
+
"<|reserved_special_token_1125|>",
|
| 1132 |
+
"<|reserved_special_token_1126|>",
|
| 1133 |
+
"<|reserved_special_token_1127|>",
|
| 1134 |
+
"<|reserved_special_token_1128|>",
|
| 1135 |
+
"<|reserved_special_token_1129|>",
|
| 1136 |
+
"<|reserved_special_token_1130|>",
|
| 1137 |
+
"<|reserved_special_token_1131|>",
|
| 1138 |
+
"<|reserved_special_token_1132|>",
|
| 1139 |
+
"<|reserved_special_token_1133|>",
|
| 1140 |
+
"<|reserved_special_token_1134|>",
|
| 1141 |
+
"<|reserved_special_token_1135|>",
|
| 1142 |
+
"<|reserved_special_token_1136|>",
|
| 1143 |
+
"<|reserved_special_token_1137|>",
|
| 1144 |
+
"<|reserved_special_token_1138|>",
|
| 1145 |
+
"<|reserved_special_token_1139|>",
|
| 1146 |
+
"<|reserved_special_token_1140|>",
|
| 1147 |
+
"<|reserved_special_token_1141|>",
|
| 1148 |
+
"<|reserved_special_token_1142|>",
|
| 1149 |
+
"<|reserved_special_token_1143|>",
|
| 1150 |
+
"<|reserved_special_token_1144|>",
|
| 1151 |
+
"<|reserved_special_token_1145|>",
|
| 1152 |
+
"<|reserved_special_token_1146|>",
|
| 1153 |
+
"<|reserved_special_token_1147|>",
|
| 1154 |
+
"<|reserved_special_token_1148|>",
|
| 1155 |
+
"<|reserved_special_token_1149|>",
|
| 1156 |
+
"<|reserved_special_token_1150|>",
|
| 1157 |
+
"<|reserved_special_token_1151|>",
|
| 1158 |
+
"<|reserved_special_token_1152|>",
|
| 1159 |
+
"<|reserved_special_token_1153|>",
|
| 1160 |
+
"<|reserved_special_token_1154|>",
|
| 1161 |
+
"<|reserved_special_token_1155|>",
|
| 1162 |
+
"<|reserved_special_token_1156|>",
|
| 1163 |
+
"<|reserved_special_token_1157|>",
|
| 1164 |
+
"<|reserved_special_token_1158|>",
|
| 1165 |
+
"<|reserved_special_token_1159|>",
|
| 1166 |
+
"<|reserved_special_token_1160|>",
|
| 1167 |
+
"<|reserved_special_token_1161|>",
|
| 1168 |
+
"<|reserved_special_token_1162|>",
|
| 1169 |
+
"<|reserved_special_token_1163|>",
|
| 1170 |
+
"<|reserved_special_token_1164|>",
|
| 1171 |
+
"<|reserved_special_token_1165|>",
|
| 1172 |
+
"<|reserved_special_token_1166|>",
|
| 1173 |
+
"<|reserved_special_token_1167|>",
|
| 1174 |
+
"<|reserved_special_token_1168|>",
|
| 1175 |
+
"<|reserved_special_token_1169|>",
|
| 1176 |
+
"<|reserved_special_token_1170|>",
|
| 1177 |
+
"<|reserved_special_token_1171|>",
|
| 1178 |
+
"<|reserved_special_token_1172|>",
|
| 1179 |
+
"<|reserved_special_token_1173|>",
|
| 1180 |
+
"<|reserved_special_token_1174|>",
|
| 1181 |
+
"<|reserved_special_token_1175|>",
|
| 1182 |
+
"<|reserved_special_token_1176|>",
|
| 1183 |
+
"<|reserved_special_token_1177|>",
|
| 1184 |
+
"<|reserved_special_token_1178|>",
|
| 1185 |
+
"<|reserved_special_token_1179|>",
|
| 1186 |
+
"<|reserved_special_token_1180|>",
|
| 1187 |
+
"<|reserved_special_token_1181|>",
|
| 1188 |
+
"<|reserved_special_token_1182|>",
|
| 1189 |
+
"<|reserved_special_token_1183|>",
|
| 1190 |
+
"<|reserved_special_token_1184|>",
|
| 1191 |
+
"<|reserved_special_token_1185|>",
|
| 1192 |
+
"<|reserved_special_token_1186|>",
|
| 1193 |
+
"<|reserved_special_token_1187|>",
|
| 1194 |
+
"<|reserved_special_token_1188|>",
|
| 1195 |
+
"<|reserved_special_token_1189|>",
|
| 1196 |
+
"<|reserved_special_token_1190|>",
|
| 1197 |
+
"<|reserved_special_token_1191|>",
|
| 1198 |
+
"<|reserved_special_token_1192|>",
|
| 1199 |
+
"<|reserved_special_token_1193|>",
|
| 1200 |
+
"<|reserved_special_token_1194|>",
|
| 1201 |
+
"<|reserved_special_token_1195|>",
|
| 1202 |
+
"<|reserved_special_token_1196|>",
|
| 1203 |
+
"<|reserved_special_token_1197|>",
|
| 1204 |
+
"<|reserved_special_token_1198|>",
|
| 1205 |
+
"<|reserved_special_token_1199|>",
|
| 1206 |
+
"<|reserved_special_token_1200|>",
|
| 1207 |
+
"<|reserved_special_token_1201|>",
|
| 1208 |
+
"<|reserved_special_token_1202|>",
|
| 1209 |
+
"<|reserved_special_token_1203|>",
|
| 1210 |
+
"<|reserved_special_token_1204|>",
|
| 1211 |
+
"<|reserved_special_token_1205|>",
|
| 1212 |
+
"<|reserved_special_token_1206|>",
|
| 1213 |
+
"<|reserved_special_token_1207|>",
|
| 1214 |
+
"<|reserved_special_token_1208|>",
|
| 1215 |
+
"<|reserved_special_token_1209|>",
|
| 1216 |
+
"<|reserved_special_token_1210|>",
|
| 1217 |
+
"<|reserved_special_token_1211|>",
|
| 1218 |
+
"<|reserved_special_token_1212|>",
|
| 1219 |
+
"<|reserved_special_token_1213|>",
|
| 1220 |
+
"<|reserved_special_token_1214|>",
|
| 1221 |
+
"<|reserved_special_token_1215|>",
|
| 1222 |
+
"<|reserved_special_token_1216|>",
|
| 1223 |
+
"<|reserved_special_token_1217|>",
|
| 1224 |
+
"<|reserved_special_token_1218|>",
|
| 1225 |
+
"<|reserved_special_token_1219|>",
|
| 1226 |
+
"<|reserved_special_token_1220|>",
|
| 1227 |
+
"<|reserved_special_token_1221|>",
|
| 1228 |
+
"<|reserved_special_token_1222|>",
|
| 1229 |
+
"<|reserved_special_token_1223|>",
|
| 1230 |
+
"<|reserved_special_token_1224|>",
|
| 1231 |
+
"<|reserved_special_token_1225|>",
|
| 1232 |
+
"<|reserved_special_token_1226|>",
|
| 1233 |
+
"<|reserved_special_token_1227|>",
|
| 1234 |
+
"<|reserved_special_token_1228|>",
|
| 1235 |
+
"<|reserved_special_token_1229|>",
|
| 1236 |
+
"<|reserved_special_token_1230|>",
|
| 1237 |
+
"<|reserved_special_token_1231|>",
|
| 1238 |
+
"<|reserved_special_token_1232|>",
|
| 1239 |
+
"<|reserved_special_token_1233|>",
|
| 1240 |
+
"<|reserved_special_token_1234|>",
|
| 1241 |
+
"<|reserved_special_token_1235|>",
|
| 1242 |
+
"<|reserved_special_token_1236|>",
|
| 1243 |
+
"<|reserved_special_token_1237|>",
|
| 1244 |
+
"<|reserved_special_token_1238|>",
|
| 1245 |
+
"<|reserved_special_token_1239|>",
|
| 1246 |
+
"<|reserved_special_token_1240|>",
|
| 1247 |
+
"<|reserved_special_token_1241|>",
|
| 1248 |
+
"<|reserved_special_token_1242|>",
|
| 1249 |
+
"<|reserved_special_token_1243|>",
|
| 1250 |
+
"<|reserved_special_token_1244|>",
|
| 1251 |
+
"<|reserved_special_token_1245|>",
|
| 1252 |
+
"<|reserved_special_token_1246|>",
|
| 1253 |
+
"<|reserved_special_token_1247|>",
|
| 1254 |
+
"<|reserved_special_token_1248|>",
|
| 1255 |
+
"<|reserved_special_token_1249|>",
|
| 1256 |
+
"<|reserved_special_token_1250|>",
|
| 1257 |
+
"<|reserved_special_token_1251|>",
|
| 1258 |
+
"<|reserved_special_token_1252|>",
|
| 1259 |
+
"<|reserved_special_token_1253|>",
|
| 1260 |
+
"<|reserved_special_token_1254|>",
|
| 1261 |
+
"<|reserved_special_token_1255|>",
|
| 1262 |
+
"<|reserved_special_token_1256|>",
|
| 1263 |
+
"<|reserved_special_token_1257|>",
|
| 1264 |
+
"<|reserved_special_token_1258|>",
|
| 1265 |
+
"<|reserved_special_token_1259|>",
|
| 1266 |
+
"<|reserved_special_token_1260|>",
|
| 1267 |
+
"<|reserved_special_token_1261|>",
|
| 1268 |
+
"<|reserved_special_token_1262|>",
|
| 1269 |
+
"<|reserved_special_token_1263|>",
|
| 1270 |
+
"<|reserved_special_token_1264|>",
|
| 1271 |
+
"<|reserved_special_token_1265|>",
|
| 1272 |
+
"<|reserved_special_token_1266|>",
|
| 1273 |
+
"<|reserved_special_token_1267|>",
|
| 1274 |
+
"<|reserved_special_token_1268|>",
|
| 1275 |
+
"<|reserved_special_token_1269|>",
|
| 1276 |
+
"<|reserved_special_token_1270|>",
|
| 1277 |
+
"<|reserved_special_token_1271|>",
|
| 1278 |
+
"<|reserved_special_token_1272|>",
|
| 1279 |
+
"<|reserved_special_token_1273|>",
|
| 1280 |
+
"<|reserved_special_token_1274|>",
|
| 1281 |
+
"<|reserved_special_token_1275|>",
|
| 1282 |
+
"<|reserved_special_token_1276|>",
|
| 1283 |
+
"<|reserved_special_token_1277|>",
|
| 1284 |
+
"<|reserved_special_token_1278|>",
|
| 1285 |
+
"<|reserved_special_token_1279|>",
|
| 1286 |
+
"<|reserved_special_token_1280|>",
|
| 1287 |
+
"<|reserved_special_token_1281|>",
|
| 1288 |
+
"<|reserved_special_token_1282|>",
|
| 1289 |
+
"<|reserved_special_token_1283|>",
|
| 1290 |
+
"<|reserved_special_token_1284|>",
|
| 1291 |
+
"<|reserved_special_token_1285|>",
|
| 1292 |
+
"<|reserved_special_token_1286|>",
|
| 1293 |
+
"<|reserved_special_token_1287|>",
|
| 1294 |
+
"<|reserved_special_token_1288|>",
|
| 1295 |
+
"<|reserved_special_token_1289|>",
|
| 1296 |
+
"<|reserved_special_token_1290|>",
|
| 1297 |
+
"<|reserved_special_token_1291|>",
|
| 1298 |
+
"<|reserved_special_token_1292|>",
|
| 1299 |
+
"<|reserved_special_token_1293|>",
|
| 1300 |
+
"<|reserved_special_token_1294|>",
|
| 1301 |
+
"<|reserved_special_token_1295|>",
|
| 1302 |
+
"<|reserved_special_token_1296|>",
|
| 1303 |
+
"<|reserved_special_token_1297|>",
|
| 1304 |
+
"<|reserved_special_token_1298|>",
|
| 1305 |
+
"<|reserved_special_token_1299|>",
|
| 1306 |
+
"<|reserved_special_token_1300|>",
|
| 1307 |
+
"<|reserved_special_token_1301|>",
|
| 1308 |
+
"<|reserved_special_token_1302|>",
|
| 1309 |
+
"<|reserved_special_token_1303|>",
|
| 1310 |
+
"<|reserved_special_token_1304|>",
|
| 1311 |
+
"<|reserved_special_token_1305|>",
|
| 1312 |
+
"<|reserved_special_token_1306|>",
|
| 1313 |
+
"<|reserved_special_token_1307|>",
|
| 1314 |
+
"<|reserved_special_token_1308|>",
|
| 1315 |
+
"<|reserved_special_token_1309|>",
|
| 1316 |
+
"<|reserved_special_token_1310|>",
|
| 1317 |
+
"<|reserved_special_token_1311|>",
|
| 1318 |
+
"<|reserved_special_token_1312|>",
|
| 1319 |
+
"<|reserved_special_token_1313|>",
|
| 1320 |
+
"<|reserved_special_token_1314|>",
|
| 1321 |
+
"<|reserved_special_token_1315|>",
|
| 1322 |
+
"<|reserved_special_token_1316|>",
|
| 1323 |
+
"<|reserved_special_token_1317|>",
|
| 1324 |
+
"<|reserved_special_token_1318|>",
|
| 1325 |
+
"<|reserved_special_token_1319|>",
|
| 1326 |
+
"<|reserved_special_token_1320|>",
|
| 1327 |
+
"<|reserved_special_token_1321|>",
|
| 1328 |
+
"<|reserved_special_token_1322|>",
|
| 1329 |
+
"<|reserved_special_token_1323|>",
|
| 1330 |
+
"<|reserved_special_token_1324|>",
|
| 1331 |
+
"<|reserved_special_token_1325|>",
|
| 1332 |
+
"<|reserved_special_token_1326|>",
|
| 1333 |
+
"<|reserved_special_token_1327|>",
|
| 1334 |
+
"<|reserved_special_token_1328|>",
|
| 1335 |
+
"<|reserved_special_token_1329|>",
|
| 1336 |
+
"<|reserved_special_token_1330|>",
|
| 1337 |
+
"<|reserved_special_token_1331|>",
|
| 1338 |
+
"<|reserved_special_token_1332|>",
|
| 1339 |
+
"<|reserved_special_token_1333|>",
|
| 1340 |
+
"<|reserved_special_token_1334|>",
|
| 1341 |
+
"<|reserved_special_token_1335|>",
|
| 1342 |
+
"<|reserved_special_token_1336|>",
|
| 1343 |
+
"<|reserved_special_token_1337|>",
|
| 1344 |
+
"<|reserved_special_token_1338|>",
|
| 1345 |
+
"<|reserved_special_token_1339|>",
|
| 1346 |
+
"<|reserved_special_token_1340|>",
|
| 1347 |
+
"<|reserved_special_token_1341|>",
|
| 1348 |
+
"<|reserved_special_token_1342|>",
|
| 1349 |
+
"<|reserved_special_token_1343|>",
|
| 1350 |
+
"<|reserved_special_token_1344|>",
|
| 1351 |
+
"<|reserved_special_token_1345|>",
|
| 1352 |
+
"<|reserved_special_token_1346|>",
|
| 1353 |
+
"<|reserved_special_token_1347|>",
|
| 1354 |
+
"<|reserved_special_token_1348|>",
|
| 1355 |
+
"<|reserved_special_token_1349|>",
|
| 1356 |
+
"<|reserved_special_token_1350|>",
|
| 1357 |
+
"<|reserved_special_token_1351|>",
|
| 1358 |
+
"<|reserved_special_token_1352|>",
|
| 1359 |
+
"<|reserved_special_token_1353|>",
|
| 1360 |
+
"<|reserved_special_token_1354|>",
|
| 1361 |
+
"<|reserved_special_token_1355|>",
|
| 1362 |
+
"<|reserved_special_token_1356|>",
|
| 1363 |
+
"<|reserved_special_token_1357|>",
|
| 1364 |
+
"<|reserved_special_token_1358|>",
|
| 1365 |
+
"<|reserved_special_token_1359|>",
|
| 1366 |
+
"<|reserved_special_token_1360|>",
|
| 1367 |
+
"<|reserved_special_token_1361|>",
|
| 1368 |
+
"<|reserved_special_token_1362|>",
|
| 1369 |
+
"<|reserved_special_token_1363|>",
|
| 1370 |
+
"<|reserved_special_token_1364|>",
|
| 1371 |
+
"<|reserved_special_token_1365|>",
|
| 1372 |
+
"<|reserved_special_token_1366|>",
|
| 1373 |
+
"<|reserved_special_token_1367|>",
|
| 1374 |
+
"<|reserved_special_token_1368|>",
|
| 1375 |
+
"<|reserved_special_token_1369|>",
|
| 1376 |
+
"<|reserved_special_token_1370|>",
|
| 1377 |
+
"<|reserved_special_token_1371|>",
|
| 1378 |
+
"<|reserved_special_token_1372|>",
|
| 1379 |
+
"<|reserved_special_token_1373|>",
|
| 1380 |
+
"<|reserved_special_token_1374|>",
|
| 1381 |
+
"<|reserved_special_token_1375|>",
|
| 1382 |
+
"<|reserved_special_token_1376|>",
|
| 1383 |
+
"<|reserved_special_token_1377|>",
|
| 1384 |
+
"<|reserved_special_token_1378|>",
|
| 1385 |
+
"<|reserved_special_token_1379|>",
|
| 1386 |
+
"<|reserved_special_token_1380|>",
|
| 1387 |
+
"<|reserved_special_token_1381|>",
|
| 1388 |
+
"<|reserved_special_token_1382|>",
|
| 1389 |
+
"<|reserved_special_token_1383|>",
|
| 1390 |
+
"<|reserved_special_token_1384|>",
|
| 1391 |
+
"<|reserved_special_token_1385|>",
|
| 1392 |
+
"<|reserved_special_token_1386|>",
|
| 1393 |
+
"<|reserved_special_token_1387|>",
|
| 1394 |
+
"<|reserved_special_token_1388|>",
|
| 1395 |
+
"<|reserved_special_token_1389|>",
|
| 1396 |
+
"<|reserved_special_token_1390|>",
|
| 1397 |
+
"<|reserved_special_token_1391|>",
|
| 1398 |
+
"<|reserved_special_token_1392|>",
|
| 1399 |
+
"<|reserved_special_token_1393|>",
|
| 1400 |
+
"<|reserved_special_token_1394|>",
|
| 1401 |
+
"<|reserved_special_token_1395|>",
|
| 1402 |
+
"<|reserved_special_token_1396|>",
|
| 1403 |
+
"<|reserved_special_token_1397|>",
|
| 1404 |
+
"<|reserved_special_token_1398|>",
|
| 1405 |
+
"<|reserved_special_token_1399|>",
|
| 1406 |
+
"<|reserved_special_token_1400|>",
|
| 1407 |
+
"<|reserved_special_token_1401|>",
|
| 1408 |
+
"<|reserved_special_token_1402|>",
|
| 1409 |
+
"<|reserved_special_token_1403|>",
|
| 1410 |
+
"<|reserved_special_token_1404|>",
|
| 1411 |
+
"<|reserved_special_token_1405|>",
|
| 1412 |
+
"<|reserved_special_token_1406|>",
|
| 1413 |
+
"<|reserved_special_token_1407|>",
|
| 1414 |
+
"<|reserved_special_token_1408|>",
|
| 1415 |
+
"<|reserved_special_token_1409|>",
|
| 1416 |
+
"<|reserved_special_token_1410|>",
|
| 1417 |
+
"<|reserved_special_token_1411|>",
|
| 1418 |
+
"<|reserved_special_token_1412|>",
|
| 1419 |
+
"<|reserved_special_token_1413|>",
|
| 1420 |
+
"<|reserved_special_token_1414|>",
|
| 1421 |
+
"<|reserved_special_token_1415|>",
|
| 1422 |
+
"<|reserved_special_token_1416|>",
|
| 1423 |
+
"<|reserved_special_token_1417|>",
|
| 1424 |
+
"<|reserved_special_token_1418|>",
|
| 1425 |
+
"<|reserved_special_token_1419|>",
|
| 1426 |
+
"<|reserved_special_token_1420|>",
|
| 1427 |
+
"<|reserved_special_token_1421|>",
|
| 1428 |
+
"<|reserved_special_token_1422|>",
|
| 1429 |
+
"<|reserved_special_token_1423|>",
|
| 1430 |
+
"<|reserved_special_token_1424|>",
|
| 1431 |
+
"<|reserved_special_token_1425|>",
|
| 1432 |
+
"<|reserved_special_token_1426|>",
|
| 1433 |
+
"<|reserved_special_token_1427|>",
|
| 1434 |
+
"<|reserved_special_token_1428|>",
|
| 1435 |
+
"<|reserved_special_token_1429|>",
|
| 1436 |
+
"<|reserved_special_token_1430|>",
|
| 1437 |
+
"<|reserved_special_token_1431|>",
|
| 1438 |
+
"<|reserved_special_token_1432|>",
|
| 1439 |
+
"<|reserved_special_token_1433|>",
|
| 1440 |
+
"<|reserved_special_token_1434|>",
|
| 1441 |
+
"<|reserved_special_token_1435|>",
|
| 1442 |
+
"<|reserved_special_token_1436|>",
|
| 1443 |
+
"<|reserved_special_token_1437|>",
|
| 1444 |
+
"<|reserved_special_token_1438|>",
|
| 1445 |
+
"<|reserved_special_token_1439|>",
|
| 1446 |
+
"<|reserved_special_token_1440|>",
|
| 1447 |
+
"<|reserved_special_token_1441|>",
|
| 1448 |
+
"<|reserved_special_token_1442|>",
|
| 1449 |
+
"<|reserved_special_token_1443|>",
|
| 1450 |
+
"<|reserved_special_token_1444|>",
|
| 1451 |
+
"<|reserved_special_token_1445|>",
|
| 1452 |
+
"<|reserved_special_token_1446|>",
|
| 1453 |
+
"<|reserved_special_token_1447|>",
|
| 1454 |
+
"<|reserved_special_token_1448|>",
|
| 1455 |
+
"<|reserved_special_token_1449|>",
|
| 1456 |
+
"<|reserved_special_token_1450|>",
|
| 1457 |
+
"<|reserved_special_token_1451|>",
|
| 1458 |
+
"<|reserved_special_token_1452|>",
|
| 1459 |
+
"<|reserved_special_token_1453|>",
|
| 1460 |
+
"<|reserved_special_token_1454|>",
|
| 1461 |
+
"<|reserved_special_token_1455|>",
|
| 1462 |
+
"<|reserved_special_token_1456|>",
|
| 1463 |
+
"<|reserved_special_token_1457|>",
|
| 1464 |
+
"<|reserved_special_token_1458|>",
|
| 1465 |
+
"<|reserved_special_token_1459|>",
|
| 1466 |
+
"<|reserved_special_token_1460|>",
|
| 1467 |
+
"<|reserved_special_token_1461|>",
|
| 1468 |
+
"<|reserved_special_token_1462|>",
|
| 1469 |
+
"<|reserved_special_token_1463|>",
|
| 1470 |
+
"<|reserved_special_token_1464|>",
|
| 1471 |
+
"<|reserved_special_token_1465|>",
|
| 1472 |
+
"<|reserved_special_token_1466|>",
|
| 1473 |
+
"<|reserved_special_token_1467|>",
|
| 1474 |
+
"<|reserved_special_token_1468|>",
|
| 1475 |
+
"<|reserved_special_token_1469|>",
|
| 1476 |
+
"<|reserved_special_token_1470|>",
|
| 1477 |
+
"<|reserved_special_token_1471|>",
|
| 1478 |
+
"<|reserved_special_token_1472|>",
|
| 1479 |
+
"<|reserved_special_token_1473|>",
|
| 1480 |
+
"<|reserved_special_token_1474|>",
|
| 1481 |
+
"<|reserved_special_token_1475|>",
|
| 1482 |
+
"<|reserved_special_token_1476|>",
|
| 1483 |
+
"<|reserved_special_token_1477|>",
|
| 1484 |
+
"<|reserved_special_token_1478|>",
|
| 1485 |
+
"<|reserved_special_token_1479|>",
|
| 1486 |
+
"<|reserved_special_token_1480|>",
|
| 1487 |
+
"<|reserved_special_token_1481|>",
|
| 1488 |
+
"<|reserved_special_token_1482|>",
|
| 1489 |
+
"<|reserved_special_token_1483|>",
|
| 1490 |
+
"<|reserved_special_token_1484|>",
|
| 1491 |
+
"<|reserved_special_token_1485|>",
|
| 1492 |
+
"<|reserved_special_token_1486|>",
|
| 1493 |
+
"<|reserved_special_token_1487|>",
|
| 1494 |
+
"<|reserved_special_token_1488|>",
|
| 1495 |
+
"<|reserved_special_token_1489|>",
|
| 1496 |
+
"<|reserved_special_token_1490|>",
|
| 1497 |
+
"<|reserved_special_token_1491|>",
|
| 1498 |
+
"<|reserved_special_token_1492|>",
|
| 1499 |
+
"<|reserved_special_token_1493|>",
|
| 1500 |
+
"<|reserved_special_token_1494|>",
|
| 1501 |
+
"<|reserved_special_token_1495|>",
|
| 1502 |
+
"<|reserved_special_token_1496|>",
|
| 1503 |
+
"<|reserved_special_token_1497|>",
|
| 1504 |
+
"<|reserved_special_token_1498|>",
|
| 1505 |
+
"<|reserved_special_token_1499|>",
|
| 1506 |
+
"<|reserved_special_token_1500|>",
|
| 1507 |
+
"<|reserved_special_token_1501|>",
|
| 1508 |
+
"<|reserved_special_token_1502|>",
|
| 1509 |
+
"<|reserved_special_token_1503|>",
|
| 1510 |
+
"<|reserved_special_token_1504|>",
|
| 1511 |
+
"<|reserved_special_token_1505|>",
|
| 1512 |
+
"<|reserved_special_token_1506|>",
|
| 1513 |
+
"<|reserved_special_token_1507|>",
|
| 1514 |
+
"<|reserved_special_token_1508|>",
|
| 1515 |
+
"<|reserved_special_token_1509|>",
|
| 1516 |
+
"<|reserved_special_token_1510|>",
|
| 1517 |
+
"<|reserved_special_token_1511|>",
|
| 1518 |
+
"<|reserved_special_token_1512|>",
|
| 1519 |
+
"<|reserved_special_token_1513|>",
|
| 1520 |
+
"<|reserved_special_token_1514|>",
|
| 1521 |
+
"<|reserved_special_token_1515|>",
|
| 1522 |
+
"<|reserved_special_token_1516|>",
|
| 1523 |
+
"<|reserved_special_token_1517|>",
|
| 1524 |
+
"<|reserved_special_token_1518|>",
|
| 1525 |
+
"<|reserved_special_token_1519|>",
|
| 1526 |
+
"<|reserved_special_token_1520|>",
|
| 1527 |
+
"<|reserved_special_token_1521|>",
|
| 1528 |
+
"<|reserved_special_token_1522|>",
|
| 1529 |
+
"<|reserved_special_token_1523|>",
|
| 1530 |
+
"<|reserved_special_token_1524|>",
|
| 1531 |
+
"<|reserved_special_token_1525|>",
|
| 1532 |
+
"<|reserved_special_token_1526|>",
|
| 1533 |
+
"<|reserved_special_token_1527|>",
|
| 1534 |
+
"<|reserved_special_token_1528|>",
|
| 1535 |
+
"<|reserved_special_token_1529|>",
|
| 1536 |
+
"<|reserved_special_token_1530|>",
|
| 1537 |
+
"<|reserved_special_token_1531|>",
|
| 1538 |
+
"<|reserved_special_token_1532|>",
|
| 1539 |
+
"<|reserved_special_token_1533|>",
|
| 1540 |
+
"<|reserved_special_token_1534|>",
|
| 1541 |
+
"<|reserved_special_token_1535|>",
|
| 1542 |
+
"<|reserved_special_token_1536|>",
|
| 1543 |
+
"<|reserved_special_token_1537|>",
|
| 1544 |
+
"<|reserved_special_token_1538|>",
|
| 1545 |
+
"<|reserved_special_token_1539|>",
|
| 1546 |
+
"<|reserved_special_token_1540|>",
|
| 1547 |
+
"<|reserved_special_token_1541|>",
|
| 1548 |
+
"<|reserved_special_token_1542|>",
|
| 1549 |
+
"<|reserved_special_token_1543|>",
|
| 1550 |
+
"<|reserved_special_token_1544|>",
|
| 1551 |
+
"<|reserved_special_token_1545|>",
|
| 1552 |
+
"<|reserved_special_token_1546|>",
|
| 1553 |
+
"<|reserved_special_token_1547|>",
|
| 1554 |
+
"<|reserved_special_token_1548|>",
|
| 1555 |
+
"<|reserved_special_token_1549|>",
|
| 1556 |
+
"<|reserved_special_token_1550|>",
|
| 1557 |
+
"<|reserved_special_token_1551|>",
|
| 1558 |
+
"<|reserved_special_token_1552|>",
|
| 1559 |
+
"<|reserved_special_token_1553|>",
|
| 1560 |
+
"<|reserved_special_token_1554|>",
|
| 1561 |
+
"<|reserved_special_token_1555|>",
|
| 1562 |
+
"<|reserved_special_token_1556|>",
|
| 1563 |
+
"<|reserved_special_token_1557|>",
|
| 1564 |
+
"<|reserved_special_token_1558|>",
|
| 1565 |
+
"<|reserved_special_token_1559|>",
|
| 1566 |
+
"<|reserved_special_token_1560|>",
|
| 1567 |
+
"<|reserved_special_token_1561|>",
|
| 1568 |
+
"<|reserved_special_token_1562|>",
|
| 1569 |
+
"<|reserved_special_token_1563|>",
|
| 1570 |
+
"<|reserved_special_token_1564|>",
|
| 1571 |
+
"<|reserved_special_token_1565|>",
|
| 1572 |
+
"<|reserved_special_token_1566|>",
|
| 1573 |
+
"<|reserved_special_token_1567|>",
|
| 1574 |
+
"<|reserved_special_token_1568|>",
|
| 1575 |
+
"<|reserved_special_token_1569|>",
|
| 1576 |
+
"<|reserved_special_token_1570|>",
|
| 1577 |
+
"<|reserved_special_token_1571|>",
|
| 1578 |
+
"<|reserved_special_token_1572|>",
|
| 1579 |
+
"<|reserved_special_token_1573|>",
|
| 1580 |
+
"<|reserved_special_token_1574|>",
|
| 1581 |
+
"<|reserved_special_token_1575|>",
|
| 1582 |
+
"<|reserved_special_token_1576|>",
|
| 1583 |
+
"<|reserved_special_token_1577|>",
|
| 1584 |
+
"<|reserved_special_token_1578|>",
|
| 1585 |
+
"<|reserved_special_token_1579|>",
|
| 1586 |
+
"<|reserved_special_token_1580|>",
|
| 1587 |
+
"<|reserved_special_token_1581|>",
|
| 1588 |
+
"<|reserved_special_token_1582|>",
|
| 1589 |
+
"<|reserved_special_token_1583|>",
|
| 1590 |
+
"<|reserved_special_token_1584|>",
|
| 1591 |
+
"<|reserved_special_token_1585|>",
|
| 1592 |
+
"<|reserved_special_token_1586|>",
|
| 1593 |
+
"<|reserved_special_token_1587|>",
|
| 1594 |
+
"<|reserved_special_token_1588|>",
|
| 1595 |
+
"<|reserved_special_token_1589|>",
|
| 1596 |
+
"<|reserved_special_token_1590|>",
|
| 1597 |
+
"<|reserved_special_token_1591|>",
|
| 1598 |
+
"<|reserved_special_token_1592|>",
|
| 1599 |
+
"<|reserved_special_token_1593|>",
|
| 1600 |
+
"<|reserved_special_token_1594|>",
|
| 1601 |
+
"<|reserved_special_token_1595|>",
|
| 1602 |
+
"<|reserved_special_token_1596|>",
|
| 1603 |
+
"<|reserved_special_token_1597|>",
|
| 1604 |
+
"<|reserved_special_token_1598|>",
|
| 1605 |
+
"<|reserved_special_token_1599|>",
|
| 1606 |
+
"<|reserved_special_token_1600|>",
|
| 1607 |
+
"<|reserved_special_token_1601|>",
|
| 1608 |
+
"<|reserved_special_token_1602|>",
|
| 1609 |
+
"<|reserved_special_token_1603|>",
|
| 1610 |
+
"<|reserved_special_token_1604|>",
|
| 1611 |
+
"<|reserved_special_token_1605|>",
|
| 1612 |
+
"<|reserved_special_token_1606|>",
|
| 1613 |
+
"<|reserved_special_token_1607|>",
|
| 1614 |
+
"<|reserved_special_token_1608|>",
|
| 1615 |
+
"<|reserved_special_token_1609|>",
|
| 1616 |
+
"<|reserved_special_token_1610|>",
|
| 1617 |
+
"<|reserved_special_token_1611|>",
|
| 1618 |
+
"<|reserved_special_token_1612|>",
|
| 1619 |
+
"<|reserved_special_token_1613|>",
|
| 1620 |
+
"<|reserved_special_token_1614|>",
|
| 1621 |
+
"<|reserved_special_token_1615|>",
|
| 1622 |
+
"<|reserved_special_token_1616|>",
|
| 1623 |
+
"<|reserved_special_token_1617|>",
|
| 1624 |
+
"<|reserved_special_token_1618|>",
|
| 1625 |
+
"<|reserved_special_token_1619|>",
|
| 1626 |
+
"<|reserved_special_token_1620|>",
|
| 1627 |
+
"<|reserved_special_token_1621|>",
|
| 1628 |
+
"<|reserved_special_token_1622|>",
|
| 1629 |
+
"<|reserved_special_token_1623|>",
|
| 1630 |
+
"<|reserved_special_token_1624|>",
|
| 1631 |
+
"<|reserved_special_token_1625|>",
|
| 1632 |
+
"<|reserved_special_token_1626|>",
|
| 1633 |
+
"<|reserved_special_token_1627|>",
|
| 1634 |
+
"<|reserved_special_token_1628|>",
|
| 1635 |
+
"<|reserved_special_token_1629|>",
|
| 1636 |
+
"<|reserved_special_token_1630|>",
|
| 1637 |
+
"<|reserved_special_token_1631|>",
|
| 1638 |
+
"<|reserved_special_token_1632|>",
|
| 1639 |
+
"<|reserved_special_token_1633|>",
|
| 1640 |
+
"<|reserved_special_token_1634|>",
|
| 1641 |
+
"<|reserved_special_token_1635|>",
|
| 1642 |
+
"<|reserved_special_token_1636|>",
|
| 1643 |
+
"<|reserved_special_token_1637|>",
|
| 1644 |
+
"<|reserved_special_token_1638|>",
|
| 1645 |
+
"<|reserved_special_token_1639|>",
|
| 1646 |
+
"<|reserved_special_token_1640|>",
|
| 1647 |
+
"<|reserved_special_token_1641|>",
|
| 1648 |
+
"<|reserved_special_token_1642|>",
|
| 1649 |
+
"<|reserved_special_token_1643|>",
|
| 1650 |
+
"<|reserved_special_token_1644|>",
|
| 1651 |
+
"<|reserved_special_token_1645|>",
|
| 1652 |
+
"<|reserved_special_token_1646|>",
|
| 1653 |
+
"<|reserved_special_token_1647|>",
|
| 1654 |
+
"<|reserved_special_token_1648|>",
|
| 1655 |
+
"<|reserved_special_token_1649|>",
|
| 1656 |
+
"<|reserved_special_token_1650|>",
|
| 1657 |
+
"<|reserved_special_token_1651|>",
|
| 1658 |
+
"<|reserved_special_token_1652|>",
|
| 1659 |
+
"<|reserved_special_token_1653|>",
|
| 1660 |
+
"<|reserved_special_token_1654|>",
|
| 1661 |
+
"<|reserved_special_token_1655|>",
|
| 1662 |
+
"<|reserved_special_token_1656|>",
|
| 1663 |
+
"<|reserved_special_token_1657|>",
|
| 1664 |
+
"<|reserved_special_token_1658|>",
|
| 1665 |
+
"<|reserved_special_token_1659|>",
|
| 1666 |
+
"<|reserved_special_token_1660|>",
|
| 1667 |
+
"<|reserved_special_token_1661|>",
|
| 1668 |
+
"<|reserved_special_token_1662|>",
|
| 1669 |
+
"<|reserved_special_token_1663|>",
|
| 1670 |
+
"<|reserved_special_token_1664|>",
|
| 1671 |
+
"<|reserved_special_token_1665|>",
|
| 1672 |
+
"<|reserved_special_token_1666|>",
|
| 1673 |
+
"<|reserved_special_token_1667|>",
|
| 1674 |
+
"<|reserved_special_token_1668|>",
|
| 1675 |
+
"<|reserved_special_token_1669|>",
|
| 1676 |
+
"<|reserved_special_token_1670|>",
|
| 1677 |
+
"<|reserved_special_token_1671|>",
|
| 1678 |
+
"<|reserved_special_token_1672|>",
|
| 1679 |
+
"<|reserved_special_token_1673|>",
|
| 1680 |
+
"<|reserved_special_token_1674|>",
|
| 1681 |
+
"<|reserved_special_token_1675|>",
|
| 1682 |
+
"<|reserved_special_token_1676|>",
|
| 1683 |
+
"<|reserved_special_token_1677|>",
|
| 1684 |
+
"<|reserved_special_token_1678|>",
|
| 1685 |
+
"<|reserved_special_token_1679|>",
|
| 1686 |
+
"<|reserved_special_token_1680|>",
|
| 1687 |
+
"<|reserved_special_token_1681|>",
|
| 1688 |
+
"<|reserved_special_token_1682|>",
|
| 1689 |
+
"<|reserved_special_token_1683|>",
|
| 1690 |
+
"<|reserved_special_token_1684|>",
|
| 1691 |
+
"<|reserved_special_token_1685|>",
|
| 1692 |
+
"<|reserved_special_token_1686|>",
|
| 1693 |
+
"<|reserved_special_token_1687|>",
|
| 1694 |
+
"<|reserved_special_token_1688|>",
|
| 1695 |
+
"<|reserved_special_token_1689|>",
|
| 1696 |
+
"<|reserved_special_token_1690|>",
|
| 1697 |
+
"<|reserved_special_token_1691|>",
|
| 1698 |
+
"<|reserved_special_token_1692|>",
|
| 1699 |
+
"<|reserved_special_token_1693|>",
|
| 1700 |
+
"<|reserved_special_token_1694|>",
|
| 1701 |
+
"<|reserved_special_token_1695|>",
|
| 1702 |
+
"<|reserved_special_token_1696|>",
|
| 1703 |
+
"<|reserved_special_token_1697|>",
|
| 1704 |
+
"<|reserved_special_token_1698|>",
|
| 1705 |
+
"<|reserved_special_token_1699|>",
|
| 1706 |
+
"<|reserved_special_token_1700|>",
|
| 1707 |
+
"<|reserved_special_token_1701|>",
|
| 1708 |
+
"<|reserved_special_token_1702|>",
|
| 1709 |
+
"<|reserved_special_token_1703|>",
|
| 1710 |
+
"<|reserved_special_token_1704|>",
|
| 1711 |
+
"<|reserved_special_token_1705|>",
|
| 1712 |
+
"<|reserved_special_token_1706|>",
|
| 1713 |
+
"<|reserved_special_token_1707|>",
|
| 1714 |
+
"<|reserved_special_token_1708|>",
|
| 1715 |
+
"<|reserved_special_token_1709|>",
|
| 1716 |
+
"<|reserved_special_token_1710|>",
|
| 1717 |
+
"<|reserved_special_token_1711|>",
|
| 1718 |
+
"<|reserved_special_token_1712|>",
|
| 1719 |
+
"<|reserved_special_token_1713|>",
|
| 1720 |
+
"<|reserved_special_token_1714|>",
|
| 1721 |
+
"<|reserved_special_token_1715|>",
|
| 1722 |
+
"<|reserved_special_token_1716|>",
|
| 1723 |
+
"<|reserved_special_token_1717|>",
|
| 1724 |
+
"<|reserved_special_token_1718|>",
|
| 1725 |
+
"<|reserved_special_token_1719|>",
|
| 1726 |
+
"<|reserved_special_token_1720|>",
|
| 1727 |
+
"<|reserved_special_token_1721|>",
|
| 1728 |
+
"<|reserved_special_token_1722|>",
|
| 1729 |
+
"<|reserved_special_token_1723|>",
|
| 1730 |
+
"<|reserved_special_token_1724|>",
|
| 1731 |
+
"<|reserved_special_token_1725|>",
|
| 1732 |
+
"<|reserved_special_token_1726|>",
|
| 1733 |
+
"<|reserved_special_token_1727|>",
|
| 1734 |
+
"<|reserved_special_token_1728|>",
|
| 1735 |
+
"<|reserved_special_token_1729|>",
|
| 1736 |
+
"<|reserved_special_token_1730|>",
|
| 1737 |
+
"<|reserved_special_token_1731|>",
|
| 1738 |
+
"<|reserved_special_token_1732|>",
|
| 1739 |
+
"<|reserved_special_token_1733|>",
|
| 1740 |
+
"<|reserved_special_token_1734|>",
|
| 1741 |
+
"<|reserved_special_token_1735|>",
|
| 1742 |
+
"<|reserved_special_token_1736|>",
|
| 1743 |
+
"<|reserved_special_token_1737|>",
|
| 1744 |
+
"<|reserved_special_token_1738|>",
|
| 1745 |
+
"<|reserved_special_token_1739|>",
|
| 1746 |
+
"<|reserved_special_token_1740|>",
|
| 1747 |
+
"<|reserved_special_token_1741|>",
|
| 1748 |
+
"<|reserved_special_token_1742|>",
|
| 1749 |
+
"<|reserved_special_token_1743|>",
|
| 1750 |
+
"<|reserved_special_token_1744|>",
|
| 1751 |
+
"<|reserved_special_token_1745|>",
|
| 1752 |
+
"<|reserved_special_token_1746|>",
|
| 1753 |
+
"<|reserved_special_token_1747|>",
|
| 1754 |
+
"<|reserved_special_token_1748|>",
|
| 1755 |
+
"<|reserved_special_token_1749|>",
|
| 1756 |
+
"<|reserved_special_token_1750|>",
|
| 1757 |
+
"<|reserved_special_token_1751|>",
|
| 1758 |
+
"<|reserved_special_token_1752|>",
|
| 1759 |
+
"<|reserved_special_token_1753|>",
|
| 1760 |
+
"<|reserved_special_token_1754|>",
|
| 1761 |
+
"<|reserved_special_token_1755|>",
|
| 1762 |
+
"<|reserved_special_token_1756|>",
|
| 1763 |
+
"<|reserved_special_token_1757|>",
|
| 1764 |
+
"<|reserved_special_token_1758|>",
|
| 1765 |
+
"<|reserved_special_token_1759|>",
|
| 1766 |
+
"<|reserved_special_token_1760|>",
|
| 1767 |
+
"<|reserved_special_token_1761|>",
|
| 1768 |
+
"<|reserved_special_token_1762|>",
|
| 1769 |
+
"<|reserved_special_token_1763|>",
|
| 1770 |
+
"<|reserved_special_token_1764|>",
|
| 1771 |
+
"<|reserved_special_token_1765|>",
|
| 1772 |
+
"<|reserved_special_token_1766|>",
|
| 1773 |
+
"<|reserved_special_token_1767|>",
|
| 1774 |
+
"<|reserved_special_token_1768|>",
|
| 1775 |
+
"<|reserved_special_token_1769|>",
|
| 1776 |
+
"<|reserved_special_token_1770|>",
|
| 1777 |
+
"<|reserved_special_token_1771|>",
|
| 1778 |
+
"<|reserved_special_token_1772|>",
|
| 1779 |
+
"<|reserved_special_token_1773|>",
|
| 1780 |
+
"<|reserved_special_token_1774|>",
|
| 1781 |
+
"<|reserved_special_token_1775|>",
|
| 1782 |
+
"<|reserved_special_token_1776|>",
|
| 1783 |
+
"<|reserved_special_token_1777|>",
|
| 1784 |
+
"<|reserved_special_token_1778|>",
|
| 1785 |
+
"<|reserved_special_token_1779|>",
|
| 1786 |
+
"<|reserved_special_token_1780|>",
|
| 1787 |
+
"<|reserved_special_token_1781|>",
|
| 1788 |
+
"<|reserved_special_token_1782|>",
|
| 1789 |
+
"<|reserved_special_token_1783|>",
|
| 1790 |
+
"<|reserved_special_token_1784|>",
|
| 1791 |
+
"<|reserved_special_token_1785|>",
|
| 1792 |
+
"<|reserved_special_token_1786|>",
|
| 1793 |
+
"<|reserved_special_token_1787|>",
|
| 1794 |
+
"<|reserved_special_token_1788|>",
|
| 1795 |
+
"<|reserved_special_token_1789|>",
|
| 1796 |
+
"<|reserved_special_token_1790|>",
|
| 1797 |
+
"<|reserved_special_token_1791|>",
|
| 1798 |
+
"<|reserved_special_token_1792|>",
|
| 1799 |
+
"<|reserved_special_token_1793|>",
|
| 1800 |
+
"<|reserved_special_token_1794|>",
|
| 1801 |
+
"<|reserved_special_token_1795|>",
|
| 1802 |
+
"<|reserved_special_token_1796|>",
|
| 1803 |
+
"<|reserved_special_token_1797|>",
|
| 1804 |
+
"<|reserved_special_token_1798|>",
|
| 1805 |
+
"<|reserved_special_token_1799|>",
|
| 1806 |
+
"<|reserved_special_token_1800|>",
|
| 1807 |
+
"<|reserved_special_token_1801|>",
|
| 1808 |
+
"<|reserved_special_token_1802|>",
|
| 1809 |
+
"<|reserved_special_token_1803|>",
|
| 1810 |
+
"<|reserved_special_token_1804|>",
|
| 1811 |
+
"<|reserved_special_token_1805|>",
|
| 1812 |
+
"<|reserved_special_token_1806|>",
|
| 1813 |
+
"<|reserved_special_token_1807|>",
|
| 1814 |
+
"<|reserved_special_token_1808|>",
|
| 1815 |
+
"<|reserved_special_token_1809|>",
|
| 1816 |
+
"<|reserved_special_token_1810|>",
|
| 1817 |
+
"<|reserved_special_token_1811|>",
|
| 1818 |
+
"<|reserved_special_token_1812|>",
|
| 1819 |
+
"<|reserved_special_token_1813|>",
|
| 1820 |
+
"<|reserved_special_token_1814|>",
|
| 1821 |
+
"<|reserved_special_token_1815|>",
|
| 1822 |
+
"<|reserved_special_token_1816|>",
|
| 1823 |
+
"<|reserved_special_token_1817|>",
|
| 1824 |
+
"<|reserved_special_token_1818|>",
|
| 1825 |
+
"<|reserved_special_token_1819|>",
|
| 1826 |
+
"<|reserved_special_token_1820|>",
|
| 1827 |
+
"<|reserved_special_token_1821|>",
|
| 1828 |
+
"<|reserved_special_token_1822|>",
|
| 1829 |
+
"<|reserved_special_token_1823|>",
|
| 1830 |
+
"<|reserved_special_token_1824|>",
|
| 1831 |
+
"<|reserved_special_token_1825|>",
|
| 1832 |
+
"<|reserved_special_token_1826|>",
|
| 1833 |
+
"<|reserved_special_token_1827|>",
|
| 1834 |
+
"<|reserved_special_token_1828|>",
|
| 1835 |
+
"<|reserved_special_token_1829|>",
|
| 1836 |
+
"<|reserved_special_token_1830|>",
|
| 1837 |
+
"<|reserved_special_token_1831|>",
|
| 1838 |
+
"<|reserved_special_token_1832|>",
|
| 1839 |
+
"<|reserved_special_token_1833|>",
|
| 1840 |
+
"<|reserved_special_token_1834|>",
|
| 1841 |
+
"<|reserved_special_token_1835|>",
|
| 1842 |
+
"<|reserved_special_token_1836|>",
|
| 1843 |
+
"<|reserved_special_token_1837|>",
|
| 1844 |
+
"<|reserved_special_token_1838|>",
|
| 1845 |
+
"<|reserved_special_token_1839|>",
|
| 1846 |
+
"<|reserved_special_token_1840|>",
|
| 1847 |
+
"<|reserved_special_token_1841|>",
|
| 1848 |
+
"<|reserved_special_token_1842|>",
|
| 1849 |
+
"<|reserved_special_token_1843|>",
|
| 1850 |
+
"<|reserved_special_token_1844|>",
|
| 1851 |
+
"<|reserved_special_token_1845|>",
|
| 1852 |
+
"<|reserved_special_token_1846|>",
|
| 1853 |
+
"<|reserved_special_token_1847|>",
|
| 1854 |
+
"<|reserved_special_token_1848|>",
|
| 1855 |
+
"<|reserved_special_token_1849|>",
|
| 1856 |
+
"<|reserved_special_token_1850|>",
|
| 1857 |
+
"<|reserved_special_token_1851|>",
|
| 1858 |
+
"<|reserved_special_token_1852|>",
|
| 1859 |
+
"<|reserved_special_token_1853|>",
|
| 1860 |
+
"<|reserved_special_token_1854|>",
|
| 1861 |
+
"<|reserved_special_token_1855|>",
|
| 1862 |
+
"<|reserved_special_token_1856|>",
|
| 1863 |
+
"<|reserved_special_token_1857|>",
|
| 1864 |
+
"<|reserved_special_token_1858|>",
|
| 1865 |
+
"<|reserved_special_token_1859|>",
|
| 1866 |
+
"<|reserved_special_token_1860|>",
|
| 1867 |
+
"<|reserved_special_token_1861|>",
|
| 1868 |
+
"<|reserved_special_token_1862|>",
|
| 1869 |
+
"<|reserved_special_token_1863|>",
|
| 1870 |
+
"<|reserved_special_token_1864|>",
|
| 1871 |
+
"<|reserved_special_token_1865|>",
|
| 1872 |
+
"<|reserved_special_token_1866|>",
|
| 1873 |
+
"<|reserved_special_token_1867|>",
|
| 1874 |
+
"<|reserved_special_token_1868|>",
|
| 1875 |
+
"<|reserved_special_token_1869|>",
|
| 1876 |
+
"<|reserved_special_token_1870|>",
|
| 1877 |
+
"<|reserved_special_token_1871|>",
|
| 1878 |
+
"<|reserved_special_token_1872|>",
|
| 1879 |
+
"<|reserved_special_token_1873|>",
|
| 1880 |
+
"<|reserved_special_token_1874|>",
|
| 1881 |
+
"<|reserved_special_token_1875|>",
|
| 1882 |
+
"<|reserved_special_token_1876|>",
|
| 1883 |
+
"<|reserved_special_token_1877|>",
|
| 1884 |
+
"<|reserved_special_token_1878|>",
|
| 1885 |
+
"<|reserved_special_token_1879|>",
|
| 1886 |
+
"<|reserved_special_token_1880|>",
|
| 1887 |
+
"<|reserved_special_token_1881|>",
|
| 1888 |
+
"<|reserved_special_token_1882|>",
|
| 1889 |
+
"<|reserved_special_token_1883|>",
|
| 1890 |
+
"<|reserved_special_token_1884|>",
|
| 1891 |
+
"<|reserved_special_token_1885|>",
|
| 1892 |
+
"<|reserved_special_token_1886|>",
|
| 1893 |
+
"<|reserved_special_token_1887|>",
|
| 1894 |
+
"<|reserved_special_token_1888|>",
|
| 1895 |
+
"<|reserved_special_token_1889|>",
|
| 1896 |
+
"<|reserved_special_token_1890|>",
|
| 1897 |
+
"<|reserved_special_token_1891|>",
|
| 1898 |
+
"<|reserved_special_token_1892|>",
|
| 1899 |
+
"<|reserved_special_token_1893|>",
|
| 1900 |
+
"<|reserved_special_token_1894|>",
|
| 1901 |
+
"<|reserved_special_token_1895|>",
|
| 1902 |
+
"<|reserved_special_token_1896|>",
|
| 1903 |
+
"<|reserved_special_token_1897|>",
|
| 1904 |
+
"<|reserved_special_token_1898|>",
|
| 1905 |
+
"<|reserved_special_token_1899|>",
|
| 1906 |
+
"<|reserved_special_token_1900|>",
|
| 1907 |
+
"<|reserved_special_token_1901|>",
|
| 1908 |
+
"<|reserved_special_token_1902|>",
|
| 1909 |
+
"<|reserved_special_token_1903|>",
|
| 1910 |
+
"<|reserved_special_token_1904|>",
|
| 1911 |
+
"<|reserved_special_token_1905|>",
|
| 1912 |
+
"<|reserved_special_token_1906|>",
|
| 1913 |
+
"<|reserved_special_token_1907|>",
|
| 1914 |
+
"<|reserved_special_token_1908|>",
|
| 1915 |
+
"<|reserved_special_token_1909|>",
|
| 1916 |
+
"<|reserved_special_token_1910|>",
|
| 1917 |
+
"<|reserved_special_token_1911|>",
|
| 1918 |
+
"<|reserved_special_token_1912|>",
|
| 1919 |
+
"<|reserved_special_token_1913|>",
|
| 1920 |
+
"<|reserved_special_token_1914|>",
|
| 1921 |
+
"<|reserved_special_token_1915|>",
|
| 1922 |
+
"<|reserved_special_token_1916|>",
|
| 1923 |
+
"<|reserved_special_token_1917|>",
|
| 1924 |
+
"<|reserved_special_token_1918|>",
|
| 1925 |
+
"<|reserved_special_token_1919|>",
|
| 1926 |
+
"<|reserved_special_token_1920|>",
|
| 1927 |
+
"<|reserved_special_token_1921|>",
|
| 1928 |
+
"<|reserved_special_token_1922|>",
|
| 1929 |
+
"<|reserved_special_token_1923|>",
|
| 1930 |
+
"<|reserved_special_token_1924|>",
|
| 1931 |
+
"<|reserved_special_token_1925|>",
|
| 1932 |
+
"<|reserved_special_token_1926|>",
|
| 1933 |
+
"<|reserved_special_token_1927|>",
|
| 1934 |
+
"<|reserved_special_token_1928|>",
|
| 1935 |
+
"<|reserved_special_token_1929|>",
|
| 1936 |
+
"<|reserved_special_token_1930|>",
|
| 1937 |
+
"<|reserved_special_token_1931|>",
|
| 1938 |
+
"<|reserved_special_token_1932|>",
|
| 1939 |
+
"<|reserved_special_token_1933|>",
|
| 1940 |
+
"<|reserved_special_token_1934|>",
|
| 1941 |
+
"<|reserved_special_token_1935|>",
|
| 1942 |
+
"<|reserved_special_token_1936|>",
|
| 1943 |
+
"<|reserved_special_token_1937|>",
|
| 1944 |
+
"<|reserved_special_token_1938|>",
|
| 1945 |
+
"<|reserved_special_token_1939|>",
|
| 1946 |
+
"<|reserved_special_token_1940|>",
|
| 1947 |
+
"<|reserved_special_token_1941|>",
|
| 1948 |
+
"<|reserved_special_token_1942|>",
|
| 1949 |
+
"<|reserved_special_token_1943|>",
|
| 1950 |
+
"<|reserved_special_token_1944|>",
|
| 1951 |
+
"<|reserved_special_token_1945|>",
|
| 1952 |
+
"<|reserved_special_token_1946|>",
|
| 1953 |
+
"<|reserved_special_token_1947|>",
|
| 1954 |
+
"<|reserved_special_token_1948|>",
|
| 1955 |
+
"<|reserved_special_token_1949|>",
|
| 1956 |
+
"<|reserved_special_token_1950|>",
|
| 1957 |
+
"<|reserved_special_token_1951|>",
|
| 1958 |
+
"<|reserved_special_token_1952|>",
|
| 1959 |
+
"<|reserved_special_token_1953|>",
|
| 1960 |
+
"<|reserved_special_token_1954|>",
|
| 1961 |
+
"<|reserved_special_token_1955|>",
|
| 1962 |
+
"<|reserved_special_token_1956|>",
|
| 1963 |
+
"<|reserved_special_token_1957|>",
|
| 1964 |
+
"<|reserved_special_token_1958|>",
|
| 1965 |
+
"<|reserved_special_token_1959|>",
|
| 1966 |
+
"<|reserved_special_token_1960|>",
|
| 1967 |
+
"<|reserved_special_token_1961|>",
|
| 1968 |
+
"<|reserved_special_token_1962|>",
|
| 1969 |
+
"<|reserved_special_token_1963|>",
|
| 1970 |
+
"<|reserved_special_token_1964|>",
|
| 1971 |
+
"<|reserved_special_token_1965|>",
|
| 1972 |
+
"<|reserved_special_token_1966|>",
|
| 1973 |
+
"<|reserved_special_token_1967|>",
|
| 1974 |
+
"<|reserved_special_token_1968|>",
|
| 1975 |
+
"<|reserved_special_token_1969|>",
|
| 1976 |
+
"<|reserved_special_token_1970|>",
|
| 1977 |
+
"<|reserved_special_token_1971|>",
|
| 1978 |
+
"<|reserved_special_token_1972|>",
|
| 1979 |
+
"<|reserved_special_token_1973|>",
|
| 1980 |
+
"<|reserved_special_token_1974|>",
|
| 1981 |
+
"<|reserved_special_token_1975|>",
|
| 1982 |
+
"<|reserved_special_token_1976|>",
|
| 1983 |
+
"<|reserved_special_token_1977|>",
|
| 1984 |
+
"<|reserved_special_token_1978|>",
|
| 1985 |
+
"<|reserved_special_token_1979|>",
|
| 1986 |
+
"<|reserved_special_token_1980|>",
|
| 1987 |
+
"<|reserved_special_token_1981|>",
|
| 1988 |
+
"<|reserved_special_token_1982|>",
|
| 1989 |
+
"<|reserved_special_token_1983|>",
|
| 1990 |
+
"<|reserved_special_token_1984|>",
|
| 1991 |
+
"<|reserved_special_token_1985|>",
|
| 1992 |
+
"<|reserved_special_token_1986|>",
|
| 1993 |
+
"<|reserved_special_token_1987|>",
|
| 1994 |
+
"<|reserved_special_token_1988|>",
|
| 1995 |
+
"<|reserved_special_token_1989|>",
|
| 1996 |
+
"<|reserved_special_token_1990|>",
|
| 1997 |
+
"<|reserved_special_token_1991|>",
|
| 1998 |
+
"<|reserved_special_token_1992|>",
|
| 1999 |
+
"<|reserved_special_token_1993|>",
|
| 2000 |
+
"<|reserved_special_token_1994|>",
|
| 2001 |
+
"<|reserved_special_token_1995|>",
|
| 2002 |
+
"<|reserved_special_token_1996|>",
|
| 2003 |
+
"<|reserved_special_token_1997|>",
|
| 2004 |
+
"<|reserved_special_token_1998|>",
|
| 2005 |
+
"<|reserved_special_token_1999|>",
|
| 2006 |
+
"<|reserved_special_token_2000|>",
|
| 2007 |
+
"<|reserved_special_token_2001|>",
|
| 2008 |
+
"<|reserved_special_token_2002|>",
|
| 2009 |
+
"<|reserved_special_token_2003|>",
|
| 2010 |
+
"<|reserved_special_token_2004|>",
|
| 2011 |
+
"<|reserved_special_token_2005|>",
|
| 2012 |
+
"<|reserved_special_token_2006|>",
|
| 2013 |
+
"<|reserved_special_token_2007|>",
|
| 2014 |
+
"<|reserved_special_token_2008|>",
|
| 2015 |
+
"<|reserved_special_token_2009|>",
|
| 2016 |
+
"<|reserved_special_token_2010|>",
|
| 2017 |
+
"<|reserved_special_token_2011|>",
|
| 2018 |
+
"<|reserved_special_token_2012|>",
|
| 2019 |
+
"<|reserved_special_token_2013|>",
|
| 2020 |
+
"<|reserved_special_token_2014|>",
|
| 2021 |
+
"<|reserved_special_token_2015|>",
|
| 2022 |
+
"<|reserved_special_token_2016|>",
|
| 2023 |
+
"<|reserved_special_token_2017|>",
|
| 2024 |
+
"<|reserved_special_token_2018|>",
|
| 2025 |
+
"<|reserved_special_token_2019|>",
|
| 2026 |
+
"<|reserved_special_token_2020|>",
|
| 2027 |
+
"<|reserved_special_token_2021|>",
|
| 2028 |
+
"<|reserved_special_token_2022|>",
|
| 2029 |
+
"<|reserved_special_token_2023|>",
|
| 2030 |
+
"<|reserved_special_token_2024|>",
|
| 2031 |
+
"<|reserved_special_token_2025|>",
|
| 2032 |
+
"<|reserved_special_token_2026|>",
|
| 2033 |
+
"<|reserved_special_token_2027|>",
|
| 2034 |
+
"<|reserved_special_token_2028|>",
|
| 2035 |
+
"<|reserved_special_token_2029|>",
|
| 2036 |
+
"<|reserved_special_token_2030|>",
|
| 2037 |
+
"<|reserved_special_token_2031|>",
|
| 2038 |
+
"<|reserved_special_token_2032|>",
|
| 2039 |
+
"<|reserved_special_token_2033|>",
|
| 2040 |
+
"<|reserved_special_token_2034|>",
|
| 2041 |
+
"<|reserved_special_token_2035|>",
|
| 2042 |
+
"<|reserved_special_token_2036|>",
|
| 2043 |
+
"<|reserved_special_token_2037|>",
|
| 2044 |
+
"<|reserved_special_token_2038|>",
|
| 2045 |
+
"<|reserved_special_token_2039|>",
|
| 2046 |
+
"<|reserved_special_token_2040|>",
|
| 2047 |
+
"<|reserved_special_token_2041|>",
|
| 2048 |
+
"<|reserved_special_token_2042|>",
|
| 2049 |
+
"<|reserved_special_token_2043|>",
|
| 2050 |
+
"<|reserved_special_token_2044|>",
|
| 2051 |
+
"<|reserved_special_token_2045|>",
|
| 2052 |
+
"<|reserved_special_token_2046|>",
|
| 2053 |
+
"<|reserved_special_token_2047|>"
|
| 2054 |
+
],
|
| 2055 |
+
"is_local": true,
|
| 2056 |
+
"local_files_only": true,
|
| 2057 |
+
"model_max_length": 131072,
|
| 2058 |
+
"pad_token": "<|finetune_right_pad|>",
|
| 2059 |
+
"processor_class": "MuseGlimmerProcessor",
|
| 2060 |
+
"response_template": {
|
| 2061 |
+
"defaults": {
|
| 2062 |
+
"role": "assistant"
|
| 2063 |
+
},
|
| 2064 |
+
"fields": {
|
| 2065 |
+
"content": {
|
| 2066 |
+
"close": [
|
| 2067 |
+
"<|eot|>",
|
| 2068 |
+
"<|eom|>"
|
| 2069 |
+
],
|
| 2070 |
+
"content": "text",
|
| 2071 |
+
"open_pattern": "to=user<\\|message\\|>"
|
| 2072 |
+
},
|
| 2073 |
+
"reasoning_content": {
|
| 2074 |
+
"close": "<|eom|>",
|
| 2075 |
+
"content": "text",
|
| 2076 |
+
"open_pattern": "to=self<\\|message\\|>"
|
| 2077 |
+
},
|
| 2078 |
+
"tool_calls": {
|
| 2079 |
+
"close": "</atem:invoke>",
|
| 2080 |
+
"content": "xml-inline",
|
| 2081 |
+
"content_args": {
|
| 2082 |
+
"tag_pattern": "<atem:parameter\\b[^>]*?\\bname=\"(?P<key>[^\"]+)\"[^>]*?>(?P<value>.*?)</atem:parameter>",
|
| 2083 |
+
"value_parser": {
|
| 2084 |
+
"args": {
|
| 2085 |
+
"allow_non_json": true
|
| 2086 |
+
},
|
| 2087 |
+
"name": "json"
|
| 2088 |
+
}
|
| 2089 |
+
},
|
| 2090 |
+
"open_pattern": "<atem:invoke\\b[^>]*?\\bname=\"(?P<name>[^\"]+)\">",
|
| 2091 |
+
"repeats": true,
|
| 2092 |
+
"transform": {
|
| 2093 |
+
"function": {
|
| 2094 |
+
"arguments": "{content}",
|
| 2095 |
+
"name": "{name}"
|
| 2096 |
+
},
|
| 2097 |
+
"type": "function"
|
| 2098 |
+
}
|
| 2099 |
+
}
|
| 2100 |
+
},
|
| 2101 |
+
"start_anchor": "<|start|>assistant"
|
| 2102 |
+
},
|
| 2103 |
+
"tokenizer_class": "TokenizersBackend"
|
| 2104 |
+
}
|
upstream_metaencoder.py
ADDED
|
@@ -0,0 +1,369 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MetaEncoder-30B: match a task against candidates, over text, images, and video.
|
| 2 |
+
|
| 3 |
+
from modeling_metaencoder import MetaEncoder
|
| 4 |
+
|
| 5 |
+
model = MetaEncoder.from_pretrained("<this repo>")
|
| 6 |
+
|
| 7 |
+
results = model.match(
|
| 8 |
+
task={"text": "Which photo shows a bicycle?", "image": context_img},
|
| 9 |
+
candidates=[
|
| 10 |
+
{"image": img_a},
|
| 11 |
+
{"image": img_b, "instruction": "Represent the photo"},
|
| 12 |
+
{"text": "a red bicycle leaning on a wall"},
|
| 13 |
+
{"video": frames},
|
| 14 |
+
],
|
| 15 |
+
)
|
| 16 |
+
results[0].index, results[0].score, results[0].candidate
|
| 17 |
+
|
| 18 |
+
A *task* is what you are asking: text, images, and video in one object. Put the question,
|
| 19 |
+
query or instruction into its `text` — that text is what grounds the task to the candidates.
|
| 20 |
+
A *candidate* is the same shape, plus an optional `instruction` describing how that candidate
|
| 21 |
+
should be encoded.
|
| 22 |
+
|
| 23 |
+
`match` returns candidates ranked by similarity. For a corpus you query repeatedly, encode it
|
| 24 |
+
once with `encode` and pass the resulting tensor as `candidates`.
|
| 25 |
+
|
| 26 |
+
Prompt construction, media routing, padding and pooling are handled internally and must match
|
| 27 |
+
training exactly; see `_encode_batch` for the contract.
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
from __future__ import annotations
|
| 31 |
+
|
| 32 |
+
import unicodedata
|
| 33 |
+
from dataclasses import dataclass
|
| 34 |
+
from typing import Any, Mapping, Sequence, Union
|
| 35 |
+
|
| 36 |
+
import torch
|
| 37 |
+
import torch.nn.functional as F
|
| 38 |
+
from PIL import Image
|
| 39 |
+
from transformers import AutoProcessor
|
| 40 |
+
from transformers.models.muse_glimmer.modeling_muse_glimmer import (
|
| 41 |
+
MuseGlimmerForConditionalGeneration,
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
# Held constant across every input. A system turn that varies per task is a task-ID channel the
|
| 45 |
+
# model keys on instead of reading the instruction, which makes the embeddings brittle to
|
| 46 |
+
# unseen phrasings. It is emitted rather than omitted because the chat template's no-system
|
| 47 |
+
# branch interpolates the current date, which would make a corpus encoded yesterday not match a
|
| 48 |
+
# query encoded today.
|
| 49 |
+
DEFAULT_SYSTEM_INSTRUCTION = "Represent the user's input."
|
| 50 |
+
|
| 51 |
+
IMAGE_TOKEN = "<|patch|>"
|
| 52 |
+
VIDEO_TOKEN = "<|video|>"
|
| 53 |
+
|
| 54 |
+
DEFAULT_MAX_LENGTH = 8192
|
| 55 |
+
DEFAULT_BATCH_SIZE = 8
|
| 56 |
+
|
| 57 |
+
# Frames arrive already sampled, so the video processor must be told not to sample again. It
|
| 58 |
+
# defaults to do_sample_frames=True against an inferred fps (24 when it cannot tell), which both
|
| 59 |
+
# drops frames and writes wrong wall-clock timestamps into the prompt.
|
| 60 |
+
VIDEO_FPS = 2.0
|
| 61 |
+
|
| 62 |
+
Item = Union[str, Image.Image, Mapping[str, Any]]
|
| 63 |
+
|
| 64 |
+
_ITEM_KEYS = {"text", "instruction", "image", "images", "video"}
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
@dataclass
|
| 68 |
+
class Match:
|
| 69 |
+
"""One ranked candidate."""
|
| 70 |
+
|
| 71 |
+
index: int # position in the candidates you passed in
|
| 72 |
+
score: float # cosine similarity to the task, in [-1, 1]
|
| 73 |
+
candidate: Any # the candidate object itself, or None if you passed embeddings
|
| 74 |
+
|
| 75 |
+
def __repr__(self):
|
| 76 |
+
return f"Match(index={self.index}, score={self.score:.4f})"
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _ensure_punct(s: str) -> str:
|
| 80 |
+
s = (s or "").strip()
|
| 81 |
+
if s and not unicodedata.category(s[-1]).startswith("P"):
|
| 82 |
+
s += "."
|
| 83 |
+
return s
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _user_text(instruction: str | None, text: str | None) -> str:
|
| 87 |
+
# Media markers are placeholders, not content: drop them and collapse whitespace, matching
|
| 88 |
+
# the training-time strip exactly (replace with a space, then re-join on whitespace) so an
|
| 89 |
+
# embedded marker cannot fuse two words together.
|
| 90 |
+
body = (text or "").replace(IMAGE_TOKEN, " ").replace(VIDEO_TOKEN, " ")
|
| 91 |
+
body = " ".join(body.split())
|
| 92 |
+
instr = _ensure_punct(instruction)
|
| 93 |
+
return " ".join(p for p in (instr, body) if p)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _as_items(inputs: Item | Sequence[Item], instruction: str | None) -> list[dict]:
|
| 97 |
+
"""Normalise any accepted input shape into one dict per embedding."""
|
| 98 |
+
if inputs is None:
|
| 99 |
+
raise ValueError("encode() got no inputs")
|
| 100 |
+
if isinstance(inputs, (str, Image.Image, Mapping)):
|
| 101 |
+
inputs = [inputs]
|
| 102 |
+
else:
|
| 103 |
+
try:
|
| 104 |
+
inputs = list(inputs)
|
| 105 |
+
except TypeError:
|
| 106 |
+
# Not a sequence either; wrap it so the per-item check reports what was expected
|
| 107 |
+
# instead of leaking "object is not iterable".
|
| 108 |
+
inputs = [inputs]
|
| 109 |
+
|
| 110 |
+
items = []
|
| 111 |
+
for i, raw in enumerate(inputs):
|
| 112 |
+
if isinstance(raw, str):
|
| 113 |
+
raw = {"text": raw}
|
| 114 |
+
elif isinstance(raw, Image.Image):
|
| 115 |
+
raw = {"image": raw}
|
| 116 |
+
elif not isinstance(raw, Mapping):
|
| 117 |
+
raise TypeError(
|
| 118 |
+
f"item {i} is {type(raw).__name__}; expected str, PIL.Image, or a dict with "
|
| 119 |
+
f"keys {sorted(_ITEM_KEYS)}"
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
unknown = set(raw) - _ITEM_KEYS
|
| 123 |
+
if unknown:
|
| 124 |
+
hint = ""
|
| 125 |
+
if "videos" in unknown:
|
| 126 |
+
hint = " (use 'video' with a list of frames for a single clip)"
|
| 127 |
+
raise ValueError(f"item {i} has unknown key(s) {sorted(unknown)}{hint}")
|
| 128 |
+
|
| 129 |
+
images = []
|
| 130 |
+
if raw.get("image") is not None:
|
| 131 |
+
images.append(raw["image"])
|
| 132 |
+
if raw.get("images") is not None:
|
| 133 |
+
got = raw["images"]
|
| 134 |
+
images.extend(got if isinstance(got, (list, tuple)) else [got])
|
| 135 |
+
images = [im for im in images if im is not None]
|
| 136 |
+
|
| 137 |
+
video = raw.get("video")
|
| 138 |
+
if video is not None:
|
| 139 |
+
video = [f for f in (video if isinstance(video, (list, tuple)) else [video])
|
| 140 |
+
if f is not None]
|
| 141 |
+
if not video:
|
| 142 |
+
video = None
|
| 143 |
+
|
| 144 |
+
items.append({
|
| 145 |
+
"text": raw.get("text"),
|
| 146 |
+
"instruction": raw.get("instruction", instruction),
|
| 147 |
+
"images": images or None,
|
| 148 |
+
"video": video,
|
| 149 |
+
})
|
| 150 |
+
return items
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
class MetaEncoder:
|
| 154 |
+
"""Embedding model over text, images, and video."""
|
| 155 |
+
|
| 156 |
+
def __init__(self, model, processor):
|
| 157 |
+
self.model = model
|
| 158 |
+
self.processor = processor
|
| 159 |
+
# Last-token pooling reads the final column, which is only the last real token under
|
| 160 |
+
# left padding. The processor defaults to right.
|
| 161 |
+
self.processor.tokenizer.padding_side = "left"
|
| 162 |
+
|
| 163 |
+
@classmethod
|
| 164 |
+
def from_pretrained(cls, path, device_map="auto", dtype=torch.bfloat16,
|
| 165 |
+
attn_implementation=None):
|
| 166 |
+
if attn_implementation is None:
|
| 167 |
+
attn_implementation = "sdpa"
|
| 168 |
+
# flash-attn can be installed and still be unusable: it is CUDA-only, so probing
|
| 169 |
+
# the import alone picks a backend that fails at init on a CPU load.
|
| 170 |
+
if torch.cuda.is_available():
|
| 171 |
+
try:
|
| 172 |
+
import flash_attn # noqa: F401
|
| 173 |
+
|
| 174 |
+
attn_implementation = "flash_attention_2"
|
| 175 |
+
except ImportError:
|
| 176 |
+
pass
|
| 177 |
+
model = MuseGlimmerForConditionalGeneration.from_pretrained(
|
| 178 |
+
path,
|
| 179 |
+
dtype=dtype,
|
| 180 |
+
device_map=device_map,
|
| 181 |
+
attn_implementation=attn_implementation,
|
| 182 |
+
low_cpu_mem_usage=True,
|
| 183 |
+
)
|
| 184 |
+
model.config.use_cache = False
|
| 185 |
+
model.eval()
|
| 186 |
+
# transformers keeps the rotary inv_freq buffers in fp32 whatever `dtype` is, but every
|
| 187 |
+
# reported number was measured after a `.to(bf16)`, which rounds them too. Rounding them
|
| 188 |
+
# here reproduces those numbers exactly; with fp32 inv_freq, image and video embeddings
|
| 189 |
+
# drift by up to ~1.5% cosine.
|
| 190 |
+
for module in model.modules():
|
| 191 |
+
for name, buf in list(module.named_buffers(recurse=False)):
|
| 192 |
+
if "inv_freq" in name and buf.dtype != dtype:
|
| 193 |
+
setattr(module, name, buf.to(dtype))
|
| 194 |
+
return cls(model, AutoProcessor.from_pretrained(path))
|
| 195 |
+
|
| 196 |
+
@property
|
| 197 |
+
def embedding_dim(self) -> int:
|
| 198 |
+
return self.model.config.text_config.hidden_size
|
| 199 |
+
|
| 200 |
+
def encode(
|
| 201 |
+
self,
|
| 202 |
+
inputs: Item | Sequence[Item],
|
| 203 |
+
instruction: str | None = None,
|
| 204 |
+
batch_size: int = DEFAULT_BATCH_SIZE,
|
| 205 |
+
max_length: int = DEFAULT_MAX_LENGTH,
|
| 206 |
+
) -> torch.Tensor:
|
| 207 |
+
"""Embed items directly. Returns L2-normalised float32 (n_items, embedding_dim).
|
| 208 |
+
|
| 209 |
+
Most uses want `match` instead. Reach for `encode` to embed a corpus once and pass the
|
| 210 |
+
tensor to `match` as `candidates`, or when you need the vectors themselves.
|
| 211 |
+
|
| 212 |
+
`inputs` is one item or a list of items. An item is a string, a PIL image, or a dict:
|
| 213 |
+
|
| 214 |
+
{"text": str, "image": img, "images": [img, ...], "video": [frame, ...],
|
| 215 |
+
"instruction": str}
|
| 216 |
+
|
| 217 |
+
All keys optional. Use `video` for pre-sampled clips (temporal patching); use `images`
|
| 218 |
+
only for genuinely independent stills.
|
| 219 |
+
"""
|
| 220 |
+
items = _as_items(inputs, instruction)
|
| 221 |
+
if not items:
|
| 222 |
+
return torch.empty(0, self.embedding_dim, device=self.model.device)
|
| 223 |
+
out = [self._encode_batch(items[i:i + batch_size], max_length)
|
| 224 |
+
for i in range(0, len(items), batch_size)]
|
| 225 |
+
return torch.cat(out, dim=0)
|
| 226 |
+
|
| 227 |
+
def match(
|
| 228 |
+
self,
|
| 229 |
+
task: Item,
|
| 230 |
+
candidates: Sequence[Item] | torch.Tensor,
|
| 231 |
+
top_k: int | None = None,
|
| 232 |
+
batch_size: int = DEFAULT_BATCH_SIZE,
|
| 233 |
+
) -> list[Match]:
|
| 234 |
+
"""Rank `candidates` against `task`, best first.
|
| 235 |
+
|
| 236 |
+
`task` is a single item — text, images, video. Put the question, query or instruction
|
| 237 |
+
in its `text`; that is what grounds the task to the candidates.
|
| 238 |
+
|
| 239 |
+
`candidates` is a list of items, each optionally carrying its own `instruction` for
|
| 240 |
+
how it should be encoded. It may instead be a tensor of embeddings from a previous
|
| 241 |
+
`encode` call, which is how you avoid re-encoding a corpus on every query.
|
| 242 |
+
|
| 243 |
+
Returns a list of `Match(index, score, candidate)` sorted by descending score.
|
| 244 |
+
"""
|
| 245 |
+
scores = self.score(task, candidates, batch_size=batch_size)
|
| 246 |
+
order = torch.argsort(scores, descending=True)
|
| 247 |
+
if top_k is not None:
|
| 248 |
+
order = order[:top_k]
|
| 249 |
+
originals = None if isinstance(candidates, torch.Tensor) else list(candidates)
|
| 250 |
+
return [
|
| 251 |
+
Match(index=int(i),
|
| 252 |
+
score=float(scores[i]),
|
| 253 |
+
candidate=None if originals is None else originals[int(i)])
|
| 254 |
+
for i in order
|
| 255 |
+
]
|
| 256 |
+
|
| 257 |
+
def score(
|
| 258 |
+
self,
|
| 259 |
+
task: Item,
|
| 260 |
+
candidates: Sequence[Item] | torch.Tensor,
|
| 261 |
+
batch_size: int = DEFAULT_BATCH_SIZE,
|
| 262 |
+
) -> torch.Tensor:
|
| 263 |
+
"""Cosine similarity of `task` to each candidate, in input order. Shape (n_candidates,)."""
|
| 264 |
+
if isinstance(task, (list, tuple)):
|
| 265 |
+
raise TypeError(
|
| 266 |
+
"task must be a single item, not a list. To score several tasks, call match() "
|
| 267 |
+
"per task (encode the candidates once and pass the tensor back in)."
|
| 268 |
+
)
|
| 269 |
+
task_emb = self.encode(task, batch_size=1)
|
| 270 |
+
if isinstance(candidates, torch.Tensor):
|
| 271 |
+
cand_emb = candidates
|
| 272 |
+
if cand_emb.ndim != 2 or cand_emb.shape[1] != task_emb.shape[1]:
|
| 273 |
+
raise ValueError(
|
| 274 |
+
f"candidate embeddings must be (n, {task_emb.shape[1]}), "
|
| 275 |
+
f"got {tuple(cand_emb.shape)}"
|
| 276 |
+
)
|
| 277 |
+
cand_emb = cand_emb.to(task_emb.device)
|
| 278 |
+
else:
|
| 279 |
+
cand_emb = self.encode(candidates, batch_size=batch_size)
|
| 280 |
+
return (task_emb @ cand_emb.T)[0]
|
| 281 |
+
|
| 282 |
+
@staticmethod
|
| 283 |
+
def similarity(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
| 284 |
+
"""Cosine similarity matrix (n_a, n_b). Inputs are already unit-norm."""
|
| 285 |
+
return a @ b.T
|
| 286 |
+
|
| 287 |
+
# ---------------------------------------------------------------- internals
|
| 288 |
+
|
| 289 |
+
def _conversations(self, items):
|
| 290 |
+
conversations, flat_images, flat_videos = [], [], []
|
| 291 |
+
for it in items:
|
| 292 |
+
content = []
|
| 293 |
+
for im in (it["images"] or []):
|
| 294 |
+
content.append({"type": "image", "image": im})
|
| 295 |
+
flat_images.append(im)
|
| 296 |
+
if it["video"] is not None:
|
| 297 |
+
content.append({"type": "video", "video": it["video"]})
|
| 298 |
+
flat_videos.append(it["video"])
|
| 299 |
+
|
| 300 |
+
ut = _user_text(it["instruction"], it["text"])
|
| 301 |
+
if ut:
|
| 302 |
+
content.append({"type": "text", "text": ut})
|
| 303 |
+
elif not content:
|
| 304 |
+
content.append({"type": "text", "text": " "})
|
| 305 |
+
|
| 306 |
+
conversations.append([
|
| 307 |
+
{"role": "system",
|
| 308 |
+
"content": [{"type": "text", "text": DEFAULT_SYSTEM_INSTRUCTION}]},
|
| 309 |
+
{"role": "user", "content": content},
|
| 310 |
+
])
|
| 311 |
+
return conversations, flat_images, flat_videos
|
| 312 |
+
|
| 313 |
+
@torch.no_grad()
|
| 314 |
+
def _encode_batch(self, items, max_length):
|
| 315 |
+
conversations, flat_images, flat_videos = self._conversations(items)
|
| 316 |
+
chat_texts = [
|
| 317 |
+
self.processor.apply_chat_template(c, add_generation_prompt=False, tokenize=False)
|
| 318 |
+
for c in conversations
|
| 319 |
+
]
|
| 320 |
+
|
| 321 |
+
kwargs = dict(
|
| 322 |
+
text=chat_texts,
|
| 323 |
+
images=flat_images or None,
|
| 324 |
+
videos=flat_videos or None,
|
| 325 |
+
padding=True,
|
| 326 |
+
return_tensors="pt",
|
| 327 |
+
# The chat template already emits bos_token and the tokenizer has
|
| 328 |
+
# add_bos_token=True; leaving this on yields a doubled <|begin_of_text|>.
|
| 329 |
+
add_special_tokens=False,
|
| 330 |
+
)
|
| 331 |
+
if flat_videos:
|
| 332 |
+
kwargs["do_sample_frames"] = False
|
| 333 |
+
from transformers.video_utils import VideoMetadata
|
| 334 |
+
|
| 335 |
+
kwargs["video_metadata"] = [
|
| 336 |
+
VideoMetadata(
|
| 337 |
+
total_num_frames=len(v),
|
| 338 |
+
fps=VIDEO_FPS,
|
| 339 |
+
duration=len(v) / VIDEO_FPS,
|
| 340 |
+
frames_indices=list(range(len(v))),
|
| 341 |
+
)
|
| 342 |
+
for v in flat_videos
|
| 343 |
+
]
|
| 344 |
+
if flat_images or flat_videos:
|
| 345 |
+
# Truncation would desynchronise the expanded <|patch|>/<|video|> runs from the pixels.
|
| 346 |
+
kwargs["truncation"] = False
|
| 347 |
+
else:
|
| 348 |
+
kwargs["truncation"] = True
|
| 349 |
+
kwargs["max_length"] = max_length
|
| 350 |
+
|
| 351 |
+
inputs = self.processor(**kwargs)
|
| 352 |
+
inputs["input_ids"] = inputs["input_ids"].long()
|
| 353 |
+
inputs["attention_mask"] = inputs["attention_mask"].long()
|
| 354 |
+
inputs = {k: (v.to(self.model.device) if hasattr(v, "to") else v)
|
| 355 |
+
for k, v in inputs.items()}
|
| 356 |
+
|
| 357 |
+
am = inputs["attention_mask"]
|
| 358 |
+
assert bool((am[:, -1] == 1).all()), "expected left padding"
|
| 359 |
+
|
| 360 |
+
out = self.model(
|
| 361 |
+
**inputs,
|
| 362 |
+
return_dict=True,
|
| 363 |
+
output_hidden_states=True,
|
| 364 |
+
# ForConditionalGeneration otherwise materialises full-sequence logits over a
|
| 365 |
+
# 202k vocab, which dominates memory; only hidden states are needed here.
|
| 366 |
+
logits_to_keep=1,
|
| 367 |
+
)
|
| 368 |
+
pooled = out.hidden_states[-1][:, -1, :]
|
| 369 |
+
return F.normalize(pooled.float(), p=2, dim=-1)
|
validate_metaencoder.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Run local text, image and video inference checks for one quantization."""
|
| 2 |
+
import argparse
|
| 3 |
+
import json
|
| 4 |
+
import time
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
import mlx.core as mx
|
| 8 |
+
import numpy as np
|
| 9 |
+
from PIL import Image
|
| 10 |
+
|
| 11 |
+
from metaencoder_mlx import MetaEncoderMLX
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def main():
|
| 15 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 16 |
+
parser.add_argument("--model", type=Path, required=True)
|
| 17 |
+
parser.add_argument("--report", type=Path, required=True)
|
| 18 |
+
args = parser.parse_args()
|
| 19 |
+
start = time.monotonic()
|
| 20 |
+
model = MetaEncoderMLX(args.model)
|
| 21 |
+
report = {"model": args.model.name, "load_seconds": time.monotonic()-start,
|
| 22 |
+
"embedding_dim": model.embedding_dim, "checks": []}
|
| 23 |
+
red = Image.new("RGB", (56, 56), (255, 0, 0))
|
| 24 |
+
cases = [
|
| 25 |
+
("text_animal", "Which animal says meow?", ["cat", "dog", "horse"]),
|
| 26 |
+
("text_action", "Select the action to cancel a pending order.",
|
| 27 |
+
["Click Cancel order", "Open profile settings", "Search products"]),
|
| 28 |
+
("image_color", {"image": red, "text": "What is the main color of this image?"},
|
| 29 |
+
["red", "blue", "green"]),
|
| 30 |
+
("video_color", {"video": [red, red], "text": "What is the main color of this video?"},
|
| 31 |
+
["red", "blue", "green"]),
|
| 32 |
+
]
|
| 33 |
+
vectors = []
|
| 34 |
+
for name, task, candidates in cases:
|
| 35 |
+
started = time.monotonic()
|
| 36 |
+
embeddings = model.encode([task, *candidates])
|
| 37 |
+
array = np.asarray(embeddings)
|
| 38 |
+
assert array.shape == (4, model.embedding_dim), array.shape
|
| 39 |
+
assert np.isfinite(array).all(), name
|
| 40 |
+
norms = np.linalg.norm(array, axis=-1)
|
| 41 |
+
assert np.allclose(norms, 1.0, atol=1e-5), norms
|
| 42 |
+
scores = array[1:] @ array[0]
|
| 43 |
+
predicted = int(scores.argmax())
|
| 44 |
+
report["checks"].append({"name": name, "seconds": time.monotonic()-started,
|
| 45 |
+
"scores": scores.tolist(), "candidates": candidates,
|
| 46 |
+
"predicted": candidates[predicted], "expected": candidates[0],
|
| 47 |
+
"correct": predicted == 0, "norms": norms.tolist()})
|
| 48 |
+
vectors.append(array)
|
| 49 |
+
print(json.dumps(report["checks"][-1]), flush=True)
|
| 50 |
+
report["total_seconds"] = time.monotonic()-start
|
| 51 |
+
report["peak_mlx_memory_gb"] = mx.get_peak_memory()/1e9
|
| 52 |
+
args.report.write_text(json.dumps(report, indent=2) + "\n")
|
| 53 |
+
np.savez(args.report.with_suffix(".npz"), **{case[0]: vec for case, vec in zip(cases, vectors)})
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
if __name__ == "__main__":
|
| 57 |
+
main()
|
validation.json
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "meta-encoder-mlx-4bit",
|
| 3 |
+
"load_seconds": 4.304674542014254,
|
| 4 |
+
"embedding_dim": 6656,
|
| 5 |
+
"checks": [
|
| 6 |
+
{
|
| 7 |
+
"name": "text_animal",
|
| 8 |
+
"seconds": 3.765016416000435,
|
| 9 |
+
"scores": [
|
| 10 |
+
0.3861164450645447,
|
| 11 |
+
0.2507832944393158,
|
| 12 |
+
0.19388027489185333
|
| 13 |
+
],
|
| 14 |
+
"candidates": [
|
| 15 |
+
"cat",
|
| 16 |
+
"dog",
|
| 17 |
+
"horse"
|
| 18 |
+
],
|
| 19 |
+
"predicted": "cat",
|
| 20 |
+
"expected": "cat",
|
| 21 |
+
"correct": true,
|
| 22 |
+
"norms": [
|
| 23 |
+
1.0,
|
| 24 |
+
1.0,
|
| 25 |
+
1.0,
|
| 26 |
+
0.9999999403953552
|
| 27 |
+
]
|
| 28 |
+
},
|
| 29 |
+
{
|
| 30 |
+
"name": "text_action",
|
| 31 |
+
"seconds": 1.8200458750070538,
|
| 32 |
+
"scores": [
|
| 33 |
+
0.6188708543777466,
|
| 34 |
+
0.17132438719272614,
|
| 35 |
+
0.19522921741008759
|
| 36 |
+
],
|
| 37 |
+
"candidates": [
|
| 38 |
+
"Click Cancel order",
|
| 39 |
+
"Open profile settings",
|
| 40 |
+
"Search products"
|
| 41 |
+
],
|
| 42 |
+
"predicted": "Click Cancel order",
|
| 43 |
+
"expected": "Click Cancel order",
|
| 44 |
+
"correct": true,
|
| 45 |
+
"norms": [
|
| 46 |
+
1.0,
|
| 47 |
+
1.0,
|
| 48 |
+
1.0,
|
| 49 |
+
1.0
|
| 50 |
+
]
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"name": "image_color",
|
| 54 |
+
"seconds": 1.9545344169891905,
|
| 55 |
+
"scores": [
|
| 56 |
+
0.4928099513053894,
|
| 57 |
+
0.4084464907646179,
|
| 58 |
+
0.40224990248680115
|
| 59 |
+
],
|
| 60 |
+
"candidates": [
|
| 61 |
+
"red",
|
| 62 |
+
"blue",
|
| 63 |
+
"green"
|
| 64 |
+
],
|
| 65 |
+
"predicted": "red",
|
| 66 |
+
"expected": "red",
|
| 67 |
+
"correct": true,
|
| 68 |
+
"norms": [
|
| 69 |
+
1.0,
|
| 70 |
+
1.0,
|
| 71 |
+
1.0,
|
| 72 |
+
1.0
|
| 73 |
+
]
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"name": "video_color",
|
| 77 |
+
"seconds": 1.4857492909941357,
|
| 78 |
+
"scores": [
|
| 79 |
+
0.5262743830680847,
|
| 80 |
+
0.4340401291847229,
|
| 81 |
+
0.418895423412323
|
| 82 |
+
],
|
| 83 |
+
"candidates": [
|
| 84 |
+
"red",
|
| 85 |
+
"blue",
|
| 86 |
+
"green"
|
| 87 |
+
],
|
| 88 |
+
"predicted": "red",
|
| 89 |
+
"expected": "red",
|
| 90 |
+
"correct": true,
|
| 91 |
+
"norms": [
|
| 92 |
+
1.0,
|
| 93 |
+
1.0,
|
| 94 |
+
1.0,
|
| 95 |
+
1.0
|
| 96 |
+
]
|
| 97 |
+
}
|
| 98 |
+
],
|
| 99 |
+
"total_seconds": 13.330337374994997,
|
| 100 |
+
"peak_mlx_memory_gb": 19.696706988,
|
| 101 |
+
"bf16_mlx_reference_task_cosine": 0.97298264503479,
|
| 102 |
+
"bf16_mlx_reference_video_task_cosine": 0.9559712409973145,
|
| 103 |
+
"reference_scope": "One text task and one two-frame video task compared with streamed BF16 MLX inference from the original checkpoint. Full-model Transformers parity and benchmark accuracy were not measured."
|
| 104 |
+
}
|