Maxwell commited on
Commit
5a278bd
·
0 Parent(s):

Initial commit

Browse files
.gitattributes ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ *.png filter=lfs diff=lfs merge=lfs -text
LICENSE ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
README.md ADDED
The diff for this file is too large to render. See raw diff
 
assets/logo-en.png ADDED

Git LFS Details

  • SHA256: be953bfb303870903e4da659cdb3ac7c78fc9dd1be15cd2c94d4d301aa2db6d2
  • Pointer size: 130 Bytes
  • Size of remote file: 25.1 kB
assets/performance-overview.svg ADDED
chat_template.jinja ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- set image_count = namespace(value=0) %}
2
+ {%- set video_count = namespace(value=0) %}
3
+ {%- macro render_content(content, do_vision_count, is_system_content=false) %}
4
+ {%- if content is string %}
5
+ {{- content }}
6
+ {%- elif content is iterable and content is not mapping %}
7
+ {%- for item in content %}
8
+ {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
9
+ {%- if is_system_content %}
10
+ {{- raise_exception('System message cannot contain images.') }}
11
+ {%- endif %}
12
+ {%- if do_vision_count %}
13
+ {%- set image_count.value = image_count.value + 1 %}
14
+ {%- endif %}
15
+ {%- if add_vision_id %}
16
+ {{- 'Picture ' ~ image_count.value ~ ': ' }}
17
+ {%- endif %}
18
+ {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
19
+ {%- elif 'video' in item or item.type == 'video' %}
20
+ {%- if is_system_content %}
21
+ {{- raise_exception('System message cannot contain videos.') }}
22
+ {%- endif %}
23
+ {%- if do_vision_count %}
24
+ {%- set video_count.value = video_count.value + 1 %}
25
+ {%- endif %}
26
+ {%- if add_vision_id %}
27
+ {{- 'Video ' ~ video_count.value ~ ': ' }}
28
+ {%- endif %}
29
+ {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
30
+ {%- elif 'text' in item %}
31
+ {{- item.text }}
32
+ {%- else %}
33
+ {{- raise_exception('Unexpected item type in content.') }}
34
+ {%- endif %}
35
+ {%- endfor %}
36
+ {%- elif content is none or content is undefined %}
37
+ {{- '' }}
38
+ {%- else %}
39
+ {{- raise_exception('Unexpected content type.') }}
40
+ {%- endif %}
41
+ {%- endmacro %}
42
+ {%- if not messages %}
43
+ {{- raise_exception('No messages provided.') }}
44
+ {%- endif %}
45
+ {%- if tools and tools is iterable and tools is not mapping %}
46
+ {{- '<|maxwell_start|>system\n' }}
47
+ {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
48
+ {%- for tool in tools %}
49
+ {{- "\n" }}
50
+ {{- tool | tojson }}
51
+ {%- endfor %}
52
+ {{- "\n</tools>" }}
53
+ {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
54
+ {%- if messages[0].role == 'system' %}
55
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
56
+ {%- if content %}
57
+ {{- '\n\n' + content }}
58
+ {%- endif %}
59
+ {%- endif %}
60
+ {{- '<|maxwell_end|>\n' }}
61
+ {%- else %}
62
+ {%- if messages[0].role == 'system' %}
63
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
64
+ {{- '<|maxwell_start|>system\n' + content + '<|maxwell_end|>\n' }}
65
+ {%- endif %}
66
+ {%- endif %}
67
+ {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
68
+ {%- for message in messages[::-1] %}
69
+ {%- set index = (messages|length - 1) - loop.index0 %}
70
+ {%- if ns.multi_step_tool and message.role == "user" %}
71
+ {%- set content = render_content(message.content, false)|trim %}
72
+ {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
73
+ {%- set ns.multi_step_tool = false %}
74
+ {%- set ns.last_query_index = index %}
75
+ {%- endif %}
76
+ {%- endif %}
77
+ {%- endfor %}
78
+ {%- if ns.multi_step_tool %}
79
+ {{- raise_exception('No user query found in messages.') }}
80
+ {%- endif %}
81
+ {%- for message in messages %}
82
+ {%- set content = render_content(message.content, true)|trim %}
83
+ {%- if message.role == "system" %}
84
+ {%- if not loop.first %}
85
+ {{- raise_exception('System message must be at the beginning.') }}
86
+ {%- endif %}
87
+ {%- elif message.role == "user" %}
88
+ {{- '<|maxwell_start|>' + message.role + '\n' + content + '<|maxwell_end|>' + '\n' }}
89
+ {%- elif message.role == "assistant" %}
90
+ {%- set reasoning_content = '' %}
91
+ {%- if message.reasoning_content is string %}
92
+ {%- set reasoning_content = message.reasoning_content %}
93
+ {%- else %}
94
+ {%- if '</think>' in content %}
95
+ {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
96
+ {%- set content = content.split('</think>')[-1].lstrip('\n') %}
97
+ {%- endif %}
98
+ {%- endif %}
99
+ {%- set reasoning_content = reasoning_content|trim %}
100
+ {#- keep <think> for ALL assistant turns (multi-turn + tool turns), not only the last one #}
101
+ {{- '<|maxwell_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
102
+ {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
103
+ {%- for tool_call in message.tool_calls %}
104
+ {%- if tool_call.function is defined %}
105
+ {%- set tool_call = tool_call.function %}
106
+ {%- endif %}
107
+ {%- if loop.first %}
108
+ {%- if content|trim %}
109
+ {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
110
+ {%- else %}
111
+ {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
112
+ {%- endif %}
113
+ {%- else %}
114
+ {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
115
+ {%- endif %}
116
+ {%- if tool_call.arguments is defined %}
117
+ {%- for args_name, args_value in tool_call.arguments|items %}
118
+ {{- '<parameter=' + args_name + '>\n' }}
119
+ {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
120
+ {{- args_value }}
121
+ {{- '\n</parameter>\n' }}
122
+ {%- endfor %}
123
+ {%- endif %}
124
+ {{- '</function>\n</tool_call>' }}
125
+ {%- endfor %}
126
+ {%- endif %}
127
+ {{- '<|maxwell_end|>\n' }}
128
+ {%- elif message.role == "tool" %}
129
+ {%- if loop.previtem and loop.previtem.role != "tool" %}
130
+ {{- '<|maxwell_start|>user' }}
131
+ {%- endif %}
132
+ {{- '\n<tool_response>\n' }}
133
+ {{- content }}
134
+ {{- '\n</tool_response>' }}
135
+ {%- if not loop.last and loop.nextitem.role != "tool" %}
136
+ {{- '<|maxwell_end|>\n' }}
137
+ {%- elif loop.last %}
138
+ {{- '<|maxwell_end|>\n' }}
139
+ {%- endif %}
140
+ {%- else %}
141
+ {{- raise_exception('Unexpected message role.') }}
142
+ {%- endif %}
143
+ {%- endfor %}
144
+ {%- if add_generation_prompt %}
145
+ {{- '<|maxwell_start|>assistant\n' }}
146
+ {%- if enable_thinking is defined and enable_thinking is false %}
147
+ {{- '<think>\n\n</think>\n\n' }}
148
+ {%- else %}
149
+ {{- '<think>\n' }}
150
+ {%- endif %}
151
+ {%- endif %}
config.json ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "MaxwellMoeForCausalLM"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_maxwell_moe.MaxwellMoeConfig",
7
+ "AutoModelForCausalLM": "modeling_maxwell_moe.MaxwellMoeForCausalLM"
8
+ },
9
+ "attention_variant": "block_sparse_gqa",
10
+ "attention_dropout": 0.0,
11
+ "bos_token_id": 1,
12
+ "eos_token_id": 2,
13
+ "pad_token_id": 3,
14
+ "hidden_act": "silu",
15
+ "hidden_size": 2048,
16
+ "initializer_range": 0.008,
17
+ "intermediate_size": 6144,
18
+ "max_position_embeddings": 524288,
19
+ "model_type": "maxwell_moe",
20
+ "moe_intermediate_size": 512,
21
+ "shared_expert_intermediate_size": 1024,
22
+ "num_attention_heads": 32,
23
+ "num_experts": 256,
24
+ "num_experts_per_tok": 12,
25
+ "num_hidden_layers": 45,
26
+ "num_key_value_heads": 4,
27
+ "attention_head_size": 128,
28
+ "output_router_logits": false,
29
+ "rms_norm_eps": 1e-06,
30
+ "rope_theta": 20000000,
31
+ "router_aux_loss_coef": 0.0,
32
+ "tie_word_embeddings": false,
33
+ "torch_dtype": "bfloat16",
34
+ "transformers_version": "4.46.0",
35
+ "use_cache": true,
36
+ "block_sparse_gqa_indexer_n_heads": 4,
37
+ "block_sparse_gqa_indexer_head_dim": 128,
38
+ "block_sparse_gqa_block_size": 128,
39
+ "block_sparse_gqa_topk_blocks": 16,
40
+ "block_sparse_gqa_total_support_blocks": 32,
41
+ "block_sparse_gqa_group_reduction": "per_group_max",
42
+ "block_sparse_gqa_prefix_window_size": 1024,
43
+ "block_sparse_gqa_local_window_size": 1024,
44
+ "block_sparse_gqa_num_remote_blocks": 16,
45
+ "block_sparse_gqa_layer_freq": [
46
+ 1,
47
+ 1,
48
+ 1,
49
+ 1,
50
+ 1,
51
+ 1,
52
+ 1,
53
+ 1,
54
+ 1,
55
+ 1,
56
+ 1,
57
+ 1,
58
+ 1,
59
+ 1,
60
+ 1,
61
+ 1,
62
+ 1,
63
+ 1,
64
+ 1,
65
+ 1,
66
+ 1,
67
+ 1,
68
+ 1,
69
+ 1,
70
+ 1,
71
+ 1,
72
+ 1,
73
+ 1,
74
+ 1,
75
+ 1,
76
+ 1,
77
+ 1,
78
+ 1,
79
+ 1,
80
+ 1,
81
+ 1,
82
+ 1,
83
+ 1,
84
+ 1,
85
+ 1,
86
+ 1,
87
+ 1,
88
+ 1,
89
+ 1,
90
+ 1
91
+ ],
92
+ "block_sparse_gqa_query_chunk_size": 64,
93
+ "vocab_size": 200704,
94
+ "mlp_only_layers": [
95
+ 0,
96
+ 1,
97
+ 2
98
+ ],
99
+ "qk_layernorm": true,
100
+ "enable_hyper_connections": true,
101
+ "num_residual_streams": 4,
102
+ "mhc_sinkhorn_iterations": 20,
103
+ "mhc_init_gating_factor": 0.01,
104
+ "moe_router_enable_expert_bias": true,
105
+ "router_score_function": "sigmoid",
106
+ "mtp_num_layers": 1
107
+ }
configuration_maxwell_moe.py ADDED
@@ -0,0 +1,235 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2026 the Maxwell team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ """Maxwell MoE model configuration with mHC, QK LayerNorm, expert bias support."""
17
+
18
+ from transformers.configuration_utils import PretrainedConfig
19
+ from transformers.modeling_rope_utils import rope_config_validation
20
+ from transformers.utils import logging
21
+
22
+ logger = logging.get_logger(__name__)
23
+
24
+
25
+ class MaxwellMoeConfig(PretrainedConfig):
26
+ model_type = "maxwell_moe"
27
+ keys_to_ignore_at_inference = ["past_key_values"]
28
+
29
+ def __init__(
30
+ self,
31
+ vocab_size=200704,
32
+ hidden_size=2048,
33
+ intermediate_size=6144,
34
+ num_hidden_layers=45,
35
+ num_attention_heads=32,
36
+ num_key_value_heads=4,
37
+ attention_head_size=128,
38
+ hidden_act="silu",
39
+ max_position_embeddings=524288,
40
+ initializer_range=0.02,
41
+ rms_norm_eps=1e-6,
42
+ use_cache=True,
43
+ tie_word_embeddings=False,
44
+ rope_theta=20000000,
45
+ rope_scaling=None,
46
+ attention_variant="block_sparse_gqa",
47
+ block_sparse_gqa_indexer_n_heads=4,
48
+ block_sparse_gqa_indexer_head_dim=128,
49
+ block_sparse_gqa_block_size=128,
50
+ # Remote blocks are selected independently per GQA group using the
51
+ # maximum token score within each block (fixed architecture semantics).
52
+ # Prefix/local block counts are derived from window_size / block_size.
53
+ block_sparse_gqa_prefix_window_size=1024,
54
+ block_sparse_gqa_local_window_size=1024,
55
+ block_sparse_gqa_num_remote_blocks=16,
56
+ block_sparse_gqa_layer_freq=None,
57
+ # Inference execution tuning: query tokens processed per chunk by
58
+ # indexer block selection and eager sparse attention. Must be positive.
59
+ # Smaller chunks reduce chunk-local temporary memory; larger chunks
60
+ # may reduce loop overhead. This is not the sparse block size or budget
61
+ # and does not change the intended attention pattern.
62
+ block_sparse_gqa_query_chunk_size=64,
63
+ attention_dropout=0.0,
64
+ moe_intermediate_size=512,
65
+ shared_expert_intermediate_size=1024,
66
+ num_experts_per_tok=12,
67
+ num_experts=256,
68
+ output_router_logits=False,
69
+ router_aux_loss_coef=0.0,
70
+ mlp_only_layers=None,
71
+ use_shared_expert_gate=False,
72
+ # QK LayerNorm
73
+ qk_layernorm=True,
74
+ # mHC parameters
75
+ enable_hyper_connections=True,
76
+ num_residual_streams=4,
77
+ mhc_sinkhorn_iterations=20,
78
+ mhc_init_gating_factor=0.01,
79
+ # Router
80
+ moe_router_enable_expert_bias=True,
81
+ router_score_function="sigmoid",
82
+ # MTP
83
+ mtp_num_layers=0,
84
+ **kwargs,
85
+ ):
86
+ # Sliding-window attention (SWA) is not supported.
87
+ if kwargs.pop("use_sliding_window", False):
88
+ raise ValueError("Sliding-window attention is not supported.")
89
+ kwargs.pop("sliding_window", None)
90
+ kwargs.pop("max_window_layers", None)
91
+ compression_fields = sorted(
92
+ key
93
+ for key in kwargs
94
+ if key.startswith("block_sparse_gqa_compression_")
95
+ )
96
+ if compression_fields:
97
+ raise ValueError(
98
+ "This template is raw-token-only and rejects compression "
99
+ f"fields: {compression_fields}"
100
+ )
101
+ self.vocab_size = vocab_size
102
+ self.hidden_size = hidden_size
103
+ self.intermediate_size = intermediate_size
104
+ self.num_hidden_layers = num_hidden_layers
105
+ self.num_attention_heads = num_attention_heads
106
+ self.num_key_value_heads = num_key_value_heads
107
+ self.attention_head_size = attention_head_size
108
+ self.max_position_embeddings = max_position_embeddings
109
+
110
+ if attention_variant != "block_sparse_gqa":
111
+ raise ValueError(
112
+ "This template requires `attention_variant='block_sparse_gqa'`."
113
+ )
114
+ self.attention_variant = attention_variant
115
+
116
+ self.block_sparse_gqa_indexer_n_heads = int(
117
+ block_sparse_gqa_indexer_n_heads
118
+ )
119
+ self.block_sparse_gqa_indexer_head_dim = int(
120
+ block_sparse_gqa_indexer_head_dim
121
+ )
122
+ self.block_sparse_gqa_block_size = int(block_sparse_gqa_block_size)
123
+ # The indexer reuses the main attention's RoPE cos/sin tensors;
124
+ # rope_theta is the single source of truth for their rotary base.
125
+ self.block_sparse_gqa_prefix_window_size = int(
126
+ block_sparse_gqa_prefix_window_size
127
+ )
128
+ self.block_sparse_gqa_local_window_size = int(
129
+ block_sparse_gqa_local_window_size
130
+ )
131
+ self.block_sparse_gqa_num_remote_blocks = int(
132
+ block_sparse_gqa_num_remote_blocks
133
+ )
134
+ if block_sparse_gqa_layer_freq is None:
135
+ block_sparse_gqa_layer_freq = [1] * num_hidden_layers
136
+ self.block_sparse_gqa_layer_freq = list(
137
+ block_sparse_gqa_layer_freq
138
+ )
139
+ self.block_sparse_gqa_query_chunk_size = int(
140
+ block_sparse_gqa_query_chunk_size
141
+ )
142
+
143
+ if self.block_sparse_gqa_indexer_n_heads != self.num_key_value_heads:
144
+ raise ValueError(
145
+ "block_sparse_gqa requires one index query head per GQA group"
146
+ )
147
+ if self.block_sparse_gqa_indexer_head_dim != self.attention_head_size:
148
+ raise ValueError(
149
+ "block_sparse_gqa indexer_head_dim must equal attention_head_size"
150
+ )
151
+ if self.block_sparse_gqa_block_size <= 0:
152
+ raise ValueError(
153
+ "block_sparse_gqa_block_size must be positive"
154
+ )
155
+ if self.block_sparse_gqa_query_chunk_size <= 0:
156
+ raise ValueError(
157
+ "block_sparse_gqa_query_chunk_size must be positive"
158
+ )
159
+ if self.block_sparse_gqa_prefix_window_size <= 0:
160
+ raise ValueError(
161
+ "block_sparse_gqa_prefix_window_size must be positive"
162
+ )
163
+ if self.block_sparse_gqa_local_window_size <= 0:
164
+ raise ValueError(
165
+ "block_sparse_gqa_local_window_size must be positive"
166
+ )
167
+ if (
168
+ self.block_sparse_gqa_prefix_window_size
169
+ % self.block_sparse_gqa_block_size
170
+ ):
171
+ raise ValueError(
172
+ "block_sparse_gqa_prefix_window_size must be block aligned"
173
+ )
174
+ if (
175
+ self.block_sparse_gqa_local_window_size
176
+ % self.block_sparse_gqa_block_size
177
+ ):
178
+ raise ValueError(
179
+ "block_sparse_gqa_local_window_size must be block aligned"
180
+ )
181
+ if self.block_sparse_gqa_num_remote_blocks <= 0:
182
+ raise ValueError(
183
+ "block_sparse_gqa_num_remote_blocks must be positive"
184
+ )
185
+ expected_layer_freq = [1] * self.num_hidden_layers
186
+ if self.block_sparse_gqa_layer_freq != expected_layer_freq:
187
+ raise ValueError(
188
+ "This template requires every decoder layer to use "
189
+ "Block Sparse GQA"
190
+ )
191
+ if attention_dropout != 0.0:
192
+ raise ValueError(
193
+ "block_sparse_gqa requires attention_dropout=0"
194
+ )
195
+ if self.num_attention_heads % self.num_key_value_heads:
196
+ raise ValueError(
197
+ "num_attention_heads must be divisible by num_key_value_heads"
198
+ )
199
+
200
+ self.hidden_act = hidden_act
201
+ self.initializer_range = initializer_range
202
+ self.rms_norm_eps = rms_norm_eps
203
+ self.use_cache = use_cache
204
+ self.rope_theta = rope_theta
205
+ self.rope_scaling = rope_scaling
206
+ self.attention_dropout = attention_dropout
207
+ if self.rope_scaling is not None and "type" in self.rope_scaling:
208
+ self.rope_scaling["rope_type"] = self.rope_scaling["type"]
209
+ rope_config_validation(self)
210
+
211
+ self.moe_intermediate_size = moe_intermediate_size
212
+ self.shared_expert_intermediate_size = shared_expert_intermediate_size
213
+ self.num_experts_per_tok = num_experts_per_tok
214
+ self.num_experts = num_experts
215
+ self.output_router_logits = output_router_logits
216
+ self.router_aux_loss_coef = router_aux_loss_coef
217
+ self.mlp_only_layers = [] if mlp_only_layers is None else mlp_only_layers
218
+ self.use_shared_expert_gate = use_shared_expert_gate
219
+
220
+ self.qk_layernorm = qk_layernorm
221
+
222
+ self.enable_hyper_connections = enable_hyper_connections
223
+ self.num_residual_streams = num_residual_streams
224
+ self.mhc_sinkhorn_iterations = mhc_sinkhorn_iterations
225
+ self.mhc_init_gating_factor = mhc_init_gating_factor
226
+
227
+ self.moe_router_enable_expert_bias = moe_router_enable_expert_bias
228
+ self.router_score_function = router_score_function
229
+
230
+ self.mtp_num_layers = mtp_num_layers
231
+
232
+ super().__init__(
233
+ tie_word_embeddings=tie_word_embeddings,
234
+ **kwargs,
235
+ )
generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": [
5
+ 2,
6
+ 200001
7
+ ],
8
+ "transformers_version": "4.46.0"
9
+ }
model-00001.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c0d39552161e7fcef5a1d39d5f93f2230ceeb2c5a6cada0bf890702d0dfd31b6
3
+ size 2148967438
model-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13c9e3642183190759e55ae84f23a065bfaaef63c447fa970e2d511e104a09a7
3
+ size 2147888692
model-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cadbe0c83ab9b2766ecba587edf235fa88f9b3a45c4f4e6e2422ede29f96c122
3
+ size 2148161064
model-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d14e2004b72e64d63dd076c11526382bf2b74e8002d7586b22b6a0b371c1894e
3
+ size 2147888716
model-00005.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70dda32e3327f9bfe3b196df95401cf6851d808f638b078fcd0edfbc70b2e95c
3
+ size 2147888700
model-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:772c908a83818903ffc21e87d995c7725380170c19eece1ddd5b0f3d9d963571
3
+ size 2148161128
model-00007.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a8476e20efbbb72e97b032c9cca7784880af621da57bfaaccfa67a05223e6541
3
+ size 2147889724
model-00008.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:26687516c63e26781ae8e332e335c0c8bc733e052ccf2216e936ec956ac661dc
3
+ size 2147889748
model-00009.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92438f545fe5827775114224eeb8f4dbeb03c020b57cd397d6209f9c1c9da787
3
+ size 2147889724
model-00010.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1787603dabe6b99e1d1fc4285be08938a64c5136d613fac2bd5a079837c06dba
3
+ size 2148162080
model-00011.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8d1dfa1539893dba6ea91fe963540ab0eb06203d83f3c52d33e56ec79b5feb8
3
+ size 2147889748
model-00012.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3689df3f038542eb9e5fcada2650ddcca81160eb70f7f002b7d5e469f7ed5a3a
3
+ size 2147889724
model-00013.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e4debf67f02f1f9e082ab86b048f307274c462a466147cee3a176a1581f0f232
3
+ size 2148162104
model-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0e85955f4622ad4385575b4801b961ae993eb578757714fdcd5a443b89764b3
3
+ size 2147889724
model-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:38518ee5d3f537f18ffb07bcfd46116ef10d5981e5a41a9784327a76fc95c3ce
3
+ size 2147889748
model-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9ddd4e13d42e98dafff008bcd2e1f90848bd79acb1451ebe73c661d36f319354
3
+ size 2147889724
model-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:edcef1aa9d4fd81f151410db596b408bbd6368aef5433c4c815b5fa00417c2f1
3
+ size 2148162080
model-00018.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:29410b51e877afa64dafcb2099d4b4123c80d5ba076cd99fd4dd9c6d3ad96839
3
+ size 2147889748
model-00019.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06af10367e053d533eb4673f55caeba184a0d85986f3db9841c51e843f81a2db
3
+ size 2147889724
model-00020.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b79455e73cd5cdec7c44b4c8e079a4b43cc81e61e3f813bc9a610d72a56ce58
3
+ size 2148162096
model-00021.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c30019103852b970b6ece3428095a8d183274944f861fe1e7662c3040ab66bf8
3
+ size 2147889732
model-00022.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4898112417ff4a07b2b8447898ccecc2fbb20776e3733102be61c11649209421
3
+ size 2147889748
model-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0c5abacc1394ce6ff9e09ce0aa2baa0b39fff05252240d2b034e7344d56d863
3
+ size 2147889716
model-00024.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:743c24de4063977f442497fc9bab23936865d50c34f767e4887fc75cdd62c111
3
+ size 2148162072
model-00025.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06de71b0fd5992efa928b26effc6ea7e6040ba244c69ccf919d77ecdcb960627
3
+ size 2147889748
model-00026.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7fcee0131e53e17e2cd51db335210248b62ac98fbc6fadda583bacb932c73c3d
3
+ size 2147889724
model-00027.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:31ea19bc31c16ae6abb26cea2641517ed6ff167e2c894913b9068b26d75d40da
3
+ size 2148162096
model-00028.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:78536567d114224bfe03566abf53eb7472b45d0b969bd759d49834cbd42b68af
3
+ size 2147889740
model-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e90853ac2852e5ab914a26983893c84ed89cf52830675b4c3b4db6e85e20d9ee
3
+ size 2147889724
model-00030.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e8f69b46743d1769b7a650b18031fc5c7eaa6db4fe63d03c6943853556357f64
3
+ size 2147889724
model-00031.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:107e76defe0d25beee21004c8bd626d9243c3ff9cb2f750dd4574c627c131923
3
+ size 2148162072
model-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0ecaf531d0de039c05877d0276ab49b94e8ebc16e9ed1abe84b0c2d0767fc07
3
+ size 2147889748
model-00033.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5343672e9b93ea20f94b51137f8a6e4505e52c96bb33a7f01369840687742e81
3
+ size 2147889716
model-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:96fb5d3f591d1e4fd0e1729b51d39d79cf676626bc0e484fd39a97affc5538a6
3
+ size 2148365332
model-00035.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cfea0bf381feb9f8c5e2a0cb1e45d123b7dfa2fc83b13a5eeafc73f08836d2ae
3
+ size 593357142
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
modeling_maxwell_moe.py ADDED
@@ -0,0 +1,2310 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2026 the Maxwell team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ """PyTorch MaxwellMoE model — w35a3 variant.
17
+
18
+ Supports: mHC, QK LayerNorm, expert bias, sigmoid routing, learned HC head,
19
+ mixed dense/MoE layers, shared experts.
20
+ """
21
+
22
+ import math
23
+ from dataclasses import dataclass
24
+ from typing import List, Optional, Tuple, Union
25
+
26
+ import torch
27
+ import torch.nn.functional as F
28
+ import torch.utils.checkpoint
29
+ from torch import nn
30
+
31
+ from transformers.activations import ACT2FN
32
+ from transformers.cache_utils import Cache, DynamicCache, StaticCache
33
+
34
+ from transformers.generation import GenerationMixin
35
+ from transformers.modeling_attn_mask_utils import AttentionMaskConverter
36
+ from transformers.modeling_outputs import (
37
+ MoeModelOutputWithPast,
38
+ ModelOutput,
39
+ )
40
+ from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
41
+ from transformers.modeling_utils import PreTrainedModel
42
+ from transformers.utils import (
43
+ add_start_docstrings,
44
+ add_start_docstrings_to_model_forward,
45
+ is_flash_attn_2_available,
46
+ logging,
47
+ replace_return_docstrings,
48
+ )
49
+
50
+ try:
51
+ from transformers.utils import is_flash_attn_greater_or_equal_2_10
52
+ except ImportError:
53
+ try:
54
+ from transformers.utils import is_flash_attn_greater_or_equal
55
+ def is_flash_attn_greater_or_equal_2_10():
56
+ return is_flash_attn_greater_or_equal("2.1.0")
57
+ except ImportError:
58
+ def is_flash_attn_greater_or_equal_2_10():
59
+ return True
60
+ from .configuration_maxwell_moe import MaxwellMoeConfig
61
+
62
+ if is_flash_attn_2_available():
63
+ from transformers.modeling_flash_attention_utils import _flash_attention_forward
64
+
65
+ logger = logging.get_logger(__name__)
66
+
67
+
68
+ # ---------------------------------------------------------------------------
69
+ # Helpers
70
+ # ---------------------------------------------------------------------------
71
+
72
+ class MaxwellMoeRMSNorm(nn.Module):
73
+ def __init__(self, hidden_size, eps=1e-6):
74
+ super().__init__()
75
+ self.weight = nn.Parameter(torch.ones(hidden_size))
76
+ self.variance_epsilon = eps
77
+
78
+ def forward(self, hidden_states):
79
+ input_dtype = hidden_states.dtype
80
+ hidden_states = hidden_states.to(torch.float32)
81
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
82
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
83
+ # Apply the weight in fp32 and round once, as TE RMSNorm does. Rounding
84
+ # before the multiply is visible at large norm weights (q/k norms of the
85
+ # last layer reach ~14) where attention logits run into the thousands.
86
+ return (self.weight.float() * hidden_states).to(input_dtype)
87
+
88
+
89
+ class MaxwellMoeRotaryEmbedding(nn.Module):
90
+ def __init__(self, dim=None, max_position_embeddings=2048, base=10000,
91
+ device=None, scaling_factor=1.0, rope_type="default",
92
+ config: Optional[MaxwellMoeConfig] = None):
93
+ super().__init__()
94
+ self.rope_kwargs = {}
95
+ if config is None:
96
+ self.rope_kwargs = {
97
+ "rope_type": rope_type, "factor": scaling_factor,
98
+ "dim": dim, "base": base,
99
+ "max_position_embeddings": max_position_embeddings,
100
+ }
101
+ self.rope_type = rope_type
102
+ self.max_seq_len_cached = max_position_embeddings
103
+ self.original_max_seq_len = max_position_embeddings
104
+ else:
105
+ if config.rope_scaling is not None:
106
+ self.rope_type = config.rope_scaling.get(
107
+ "rope_type", config.rope_scaling.get("type"))
108
+ else:
109
+ self.rope_type = "default"
110
+ self.max_seq_len_cached = config.max_position_embeddings
111
+ self.original_max_seq_len = config.max_position_embeddings
112
+
113
+ self.config = config
114
+ self._rope_base = base
115
+ self._rope_dim = dim
116
+ if self.rope_type == "default":
117
+ if config is not None:
118
+ self._rope_base = config.rope_theta
119
+ self._rope_dim = (config.attention_head_size
120
+ if config.attention_head_size
121
+ else config.hidden_size // config.num_attention_heads)
122
+ else:
123
+ self._rope_base = self.rope_kwargs.get("base", 10000)
124
+ self._rope_dim = self.rope_kwargs.get("dim", 64)
125
+ inv_freq = self._compute_default_inv_freq(device)
126
+ self.attention_scaling = 1.0
127
+ else:
128
+ self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
129
+ inv_freq, self.attention_scaling = self.rope_init_fn(
130
+ self.config, device, **self.rope_kwargs)
131
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
132
+ self.original_inv_freq = self.inv_freq
133
+
134
+ def _compute_default_inv_freq(self, device=None):
135
+ return 1.0 / (self._rope_base ** (
136
+ torch.arange(0, self._rope_dim, 2, dtype=torch.int64).float().to(device)
137
+ / self._rope_dim))
138
+
139
+ def _inv_freq_is_valid(self):
140
+ if self.inv_freq.numel() == 0:
141
+ return False
142
+ expected = self._compute_default_inv_freq(self.inv_freq.device)
143
+ if self.inv_freq.shape != expected.shape:
144
+ return False
145
+ if not torch.isfinite(self.inv_freq).all():
146
+ return False
147
+ if (self.inv_freq <= 0).any():
148
+ return False
149
+ return torch.allclose(self.inv_freq.float(), expected.float(), rtol=1e-5, atol=1e-8)
150
+
151
+ def _reset_inv_freq(self, device):
152
+ inv_freq = self._compute_default_inv_freq(device)
153
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
154
+ self.original_inv_freq = self.inv_freq
155
+
156
+ def _dynamic_frequency_update(self, position_ids, device):
157
+ seq_len = torch.max(position_ids) + 1
158
+ if seq_len > self.max_seq_len_cached:
159
+ if hasattr(self, 'rope_init_fn'):
160
+ inv_freq, self.attention_scaling = self.rope_init_fn(
161
+ self.config, device, seq_len=seq_len, **self.rope_kwargs)
162
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
163
+ self.max_seq_len_cached = seq_len
164
+ if (seq_len < self.original_max_seq_len
165
+ and self.max_seq_len_cached > self.original_max_seq_len):
166
+ self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)
167
+ self.max_seq_len_cached = self.original_max_seq_len
168
+
169
+ @torch.no_grad()
170
+ def forward(self, x, position_ids):
171
+ if "dynamic" in self.rope_type:
172
+ self._dynamic_frequency_update(position_ids, device=x.device)
173
+ if self.rope_type == "default" and not self._inv_freq_is_valid():
174
+ self._reset_inv_freq(x.device)
175
+ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(
176
+ position_ids.shape[0], -1, 1)
177
+ position_ids_expanded = position_ids[:, None, :].float()
178
+ device_type = x.device.type
179
+ device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
180
+ with torch.autocast(device_type=device_type, enabled=False):
181
+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
182
+ emb = torch.cat((freqs, freqs), dim=-1)
183
+ cos = emb.cos() * self.attention_scaling
184
+ sin = emb.sin() * self.attention_scaling
185
+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
186
+
187
+
188
+ def rotate_half(x):
189
+ x1 = x[..., : x.shape[-1] // 2]
190
+ x2 = x[..., x.shape[-1] // 2 :]
191
+ return torch.cat((-x2, x1), dim=-1)
192
+
193
+
194
+ def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
195
+ cos = cos.unsqueeze(unsqueeze_dim)
196
+ sin = sin.unsqueeze(unsqueeze_dim)
197
+ q_embed = (q * cos) + (rotate_half(q) * sin)
198
+ k_embed = (k * cos) + (rotate_half(k) * sin)
199
+ return q_embed, k_embed
200
+
201
+
202
+ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
203
+ batch, num_key_value_heads, slen, head_dim = hidden_states.shape
204
+ if n_rep == 1:
205
+ return hidden_states
206
+ hidden_states = hidden_states[:, :, None, :, :].expand(
207
+ batch, num_key_value_heads, n_rep, slen, head_dim)
208
+ return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
209
+
210
+
211
+ # ---------------------------------------------------------------------------
212
+ # MLP
213
+ # ---------------------------------------------------------------------------
214
+
215
+ class MaxwellMoeMLP(nn.Module):
216
+ def __init__(self, config, intermediate_size=None):
217
+ super().__init__()
218
+ self.hidden_size = config.hidden_size
219
+ self.intermediate_size = intermediate_size
220
+ self.w1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
221
+ self.w3 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
222
+ self.w2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
223
+ self.act_fn = ACT2FN[config.hidden_act]
224
+
225
+ def forward(self, x):
226
+ return self.w2(self.act_fn(self.w1(x)) * self.w3(x))
227
+
228
+
229
+ # ---------------------------------------------------------------------------
230
+ # Attention with QK LayerNorm
231
+ # ---------------------------------------------------------------------------
232
+
233
+ class MaxwellMoeAttention(nn.Module):
234
+ def __init__(self, config: MaxwellMoeConfig, layer_idx: Optional[int] = None):
235
+ super().__init__()
236
+ self.config = config
237
+ self.layer_idx = layer_idx
238
+ self.hidden_size = config.hidden_size
239
+ self.num_heads = config.num_attention_heads
240
+ self.head_dim = (config.attention_head_size
241
+ if config.attention_head_size
242
+ else self.hidden_size // self.num_heads)
243
+ self.num_key_value_heads = config.num_key_value_heads
244
+ self.num_key_value_groups = self.num_heads // self.num_key_value_heads
245
+ self.max_position_embeddings = config.max_position_embeddings
246
+ self.rope_theta = config.rope_theta
247
+ self.is_causal = True
248
+ self.attention_dropout = config.attention_dropout
249
+ self.qk_layernorm = getattr(config, 'qk_layernorm', False)
250
+
251
+ self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
252
+ self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
253
+ self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
254
+ self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
255
+
256
+ if self.qk_layernorm:
257
+ self.q_layernorm = MaxwellMoeRMSNorm(self.head_dim, eps=config.rms_norm_eps)
258
+ self.k_layernorm = MaxwellMoeRMSNorm(self.head_dim, eps=config.rms_norm_eps)
259
+
260
+ self.rotary_emb = MaxwellMoeRotaryEmbedding(config=self.config)
261
+
262
+ def forward(
263
+ self, hidden_states, attention_mask=None, position_ids=None,
264
+ past_key_value=None, output_attentions=False, use_cache=False,
265
+ cache_position=None, position_embeddings=None,
266
+ ):
267
+ bsz, q_len, _ = hidden_states.size()
268
+
269
+ query_states = self.q_proj(hidden_states)
270
+ key_states = self.k_proj(hidden_states)
271
+ value_states = self.v_proj(hidden_states)
272
+
273
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim)
274
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim)
275
+
276
+ if self.qk_layernorm:
277
+ query_states = self.q_layernorm(query_states)
278
+ key_states = self.k_layernorm(key_states)
279
+
280
+ query_states = query_states.transpose(1, 2)
281
+ key_states = key_states.transpose(1, 2)
282
+ value_states = value_states.view(
283
+ bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
284
+
285
+ if position_embeddings is None:
286
+ cos, sin = self.rotary_emb(value_states, position_ids)
287
+ else:
288
+ cos, sin = position_embeddings
289
+ query_states, key_states = apply_rotary_pos_emb(
290
+ query_states, key_states, cos, sin, position_ids)
291
+
292
+ if past_key_value is not None:
293
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
294
+ key_states, value_states = past_key_value.update(
295
+ key_states, value_states, self.layer_idx, cache_kwargs)
296
+
297
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
298
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
299
+
300
+ attn_weights = torch.matmul(
301
+ query_states.float(), key_states.float().transpose(2, 3)) / math.sqrt(self.head_dim)
302
+
303
+ if attention_mask is not None:
304
+ causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
305
+ attn_weights = attn_weights + causal_mask
306
+
307
+ attn_weights = nn.functional.softmax(
308
+ attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
309
+ attn_weights = nn.functional.dropout(
310
+ attn_weights, p=self.attention_dropout, training=self.training)
311
+ attn_output = torch.matmul(attn_weights, value_states)
312
+
313
+ attn_output = attn_output.transpose(1, 2).contiguous()
314
+ attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.head_dim)
315
+ attn_output = self.o_proj(attn_output)
316
+
317
+ if not output_attentions:
318
+ attn_weights = None
319
+ return attn_output, attn_weights, past_key_value
320
+
321
+
322
+ class MaxwellMoeBlockSparseGQAIndexer(nn.Module):
323
+ """Per-group index branch used by the Block Sparse GQA model."""
324
+
325
+ def __init__(self, config: MaxwellMoeConfig):
326
+ super().__init__()
327
+ self.num_groups = config.block_sparse_gqa_indexer_n_heads
328
+ self.index_head_dim = config.block_sparse_gqa_indexer_head_dim
329
+ self.index_softmax_scale = self.index_head_dim ** -0.5
330
+
331
+ self.linear_q_proj = nn.Linear(
332
+ config.hidden_size,
333
+ self.num_groups * self.index_head_dim,
334
+ bias=False,
335
+ )
336
+ self.linear_k_proj = nn.Linear(
337
+ config.hidden_size,
338
+ self.index_head_dim,
339
+ bias=False,
340
+ )
341
+ self.q_layernorm = (
342
+ MaxwellMoeRMSNorm(
343
+ self.index_head_dim,
344
+ eps=config.rms_norm_eps,
345
+ )
346
+ if config.qk_layernorm
347
+ else None
348
+ )
349
+ self.k_layernorm = (
350
+ MaxwellMoeRMSNorm(
351
+ self.index_head_dim,
352
+ eps=config.rms_norm_eps,
353
+ )
354
+ if config.qk_layernorm
355
+ else None
356
+ )
357
+
358
+ def project_qk(self, hidden_states, cos, sin):
359
+ """Project detached hidden states and reuse the main branch RoPE."""
360
+ hidden_states = hidden_states.detach()
361
+ batch_size, query_len, _ = hidden_states.shape
362
+ query = self.linear_q_proj(hidden_states).view(
363
+ batch_size,
364
+ query_len,
365
+ self.num_groups,
366
+ self.index_head_dim,
367
+ )
368
+ key = self.linear_k_proj(hidden_states).view(
369
+ batch_size,
370
+ query_len,
371
+ 1,
372
+ self.index_head_dim,
373
+ )
374
+ if self.q_layernorm is not None:
375
+ query = self.q_layernorm(query)
376
+ if self.k_layernorm is not None:
377
+ key = self.k_layernorm(key)
378
+ query = query.transpose(1, 2)
379
+ key = key.transpose(1, 2)
380
+ query, key = apply_rotary_pos_emb(query, key, cos, sin)
381
+ return query, key.squeeze(1)
382
+
383
+
384
+ class MaxwellMoeBlockSparseGQACache(DynamicCache):
385
+ """Dynamic main K/V cache plus the post-RoPE shared indexer K."""
386
+
387
+ def __init__(self, *args, **kwargs):
388
+ super().__init__(*args, **kwargs)
389
+ self.indexer_key_states = []
390
+
391
+ def _ensure_indexer_layer(self, layer_idx):
392
+ while len(self.indexer_key_states) <= layer_idx:
393
+ self.indexer_key_states.append(None)
394
+
395
+ def update_indexer_key_state(self, key_states, layer_idx):
396
+ self._ensure_indexer_layer(layer_idx)
397
+ previous = self.indexer_key_states[layer_idx]
398
+ if previous is None:
399
+ updated = key_states
400
+ else:
401
+ updated = torch.cat(
402
+ (
403
+ previous.to(
404
+ device=key_states.device,
405
+ dtype=key_states.dtype,
406
+ ),
407
+ key_states,
408
+ ),
409
+ dim=1,
410
+ )
411
+ self.indexer_key_states[layer_idx] = updated
412
+ return updated
413
+
414
+ def get_indexer_seq_length(self, layer_idx=0):
415
+ if (
416
+ layer_idx >= len(self.indexer_key_states)
417
+ or self.indexer_key_states[layer_idx] is None
418
+ ):
419
+ return 0
420
+ return int(self.indexer_key_states[layer_idx].shape[1])
421
+
422
+ def reorder_cache(self, beam_idx):
423
+ result = super().reorder_cache(beam_idx)
424
+ for layer_idx, state in enumerate(self.indexer_key_states):
425
+ if state is not None:
426
+ self.indexer_key_states[layer_idx] = state.index_select(
427
+ 0, beam_idx.to(state.device)
428
+ )
429
+ return result
430
+
431
+ def batch_repeat_interleave(self, repeats):
432
+ result = super().batch_repeat_interleave(repeats)
433
+ for layer_idx, state in enumerate(self.indexer_key_states):
434
+ if state is not None:
435
+ self.indexer_key_states[layer_idx] = state.repeat_interleave(
436
+ repeats, dim=0
437
+ )
438
+ return result
439
+
440
+ def batch_select_indices(self, indices):
441
+ result = super().batch_select_indices(indices)
442
+ for layer_idx, state in enumerate(self.indexer_key_states):
443
+ if state is not None:
444
+ self.indexer_key_states[layer_idx] = state.index_select(
445
+ 0, indices.to(state.device)
446
+ )
447
+ return result
448
+
449
+ def crop(self, max_length):
450
+ result = super().crop(max_length)
451
+ retained_length = self.get_seq_length()
452
+ for layer_idx, state in enumerate(self.indexer_key_states):
453
+ if state is not None:
454
+ self.indexer_key_states[layer_idx] = state[:, :retained_length]
455
+ return result
456
+
457
+ def to_legacy_cache(self):
458
+ raise ValueError(
459
+ "block_sparse_gqa cannot export a legacy cache because that format "
460
+ "cannot retain indexer K"
461
+ )
462
+
463
+ @classmethod
464
+ def from_legacy_cache(cls, past_key_values=None):
465
+ if past_key_values:
466
+ raise ValueError(
467
+ "block_sparse_gqa cannot restore a non-empty legacy cache "
468
+ "because historical indexer K is absent"
469
+ )
470
+ return cls()
471
+
472
+
473
+ class MaxwellMoeBlockSparseGQAAttention(MaxwellMoeAttention):
474
+ """Eager block-sparse attention used by every backbone decoder layer.
475
+
476
+ Combines block-aligned prefix/local support with per-group block-max
477
+ remote selection. The MTP decoder uses its separate attention class.
478
+ """
479
+
480
+ def __init__(self, config: MaxwellMoeConfig, layer_idx: Optional[int] = None):
481
+ super().__init__(config, layer_idx)
482
+ if layer_idx is None:
483
+ raise ValueError("block_sparse_gqa attention requires layer_idx")
484
+ self.indexer = MaxwellMoeBlockSparseGQAIndexer(config)
485
+ self.block_size = config.block_sparse_gqa_block_size
486
+ self.prefix_window_size = (
487
+ config.block_sparse_gqa_prefix_window_size
488
+ )
489
+ self.local_window_size = config.block_sparse_gqa_local_window_size
490
+ self.num_remote_blocks = config.block_sparse_gqa_num_remote_blocks
491
+ self.num_prefix_blocks = self.prefix_window_size // self.block_size
492
+ self.num_local_blocks = self.local_window_size // self.block_size
493
+ self.query_chunk_size = config.block_sparse_gqa_query_chunk_size
494
+ self.softmax_scale = self.head_dim ** -0.5
495
+
496
+ @staticmethod
497
+ def _query_positions(
498
+ *, batch_size, query_len, key_len, device, cache_position
499
+ ):
500
+ if cache_position is None:
501
+ positions = torch.arange(
502
+ key_len - query_len,
503
+ key_len,
504
+ device=device,
505
+ dtype=torch.long,
506
+ )
507
+ else:
508
+ positions = cache_position.to(
509
+ device=device, dtype=torch.long
510
+ ).reshape(-1)
511
+ if positions.numel() != query_len:
512
+ raise ValueError(
513
+ f"cache_position has {positions.numel()} entries for "
514
+ f"query_len={query_len}"
515
+ )
516
+ return positions.unsqueeze(0).expand(batch_size, -1)
517
+
518
+ @staticmethod
519
+ def _allowed_keys(*, query_positions, key_len, attention_mask):
520
+ key_positions = torch.arange(key_len, device=query_positions.device)
521
+ allowed = (
522
+ key_positions.view(1, 1, key_len)
523
+ <= query_positions.unsqueeze(-1)
524
+ )
525
+ if attention_mask is None:
526
+ return allowed
527
+ if attention_mask.dim() != 4 or attention_mask.dtype == torch.bool:
528
+ raise ValueError(
529
+ "block_sparse_gqa attention expects the model-level 4D "
530
+ "additive causal mask"
531
+ )
532
+ mask = attention_mask[
533
+ :, 0, -query_positions.shape[-1]:, :key_len
534
+ ]
535
+ return allowed & (mask >= 0)
536
+
537
+ def _select_blocks(self, index_query, index_key, query_positions, allowed):
538
+ """Select an independent block-max remote route for every GQA group."""
539
+ batch_size, num_groups, query_len, _ = index_query.shape
540
+ key_len = index_key.shape[1]
541
+ num_blocks = (key_len + self.block_size - 1) // self.block_size
542
+ selected_count = min(max(self.num_remote_blocks, 1), num_blocks)
543
+ block_starts = torch.arange(
544
+ num_blocks,
545
+ device=index_query.device,
546
+ ) * self.block_size
547
+ block_ends = block_starts + self.block_size
548
+ selected_chunks = []
549
+
550
+ for query_start in range(0, query_len, self.query_chunk_size):
551
+ query_end = min(
552
+ query_start + self.query_chunk_size,
553
+ query_len,
554
+ )
555
+ chunk_len = query_end - query_start
556
+ query_chunk = index_query[
557
+ :, :, query_start:query_end, :
558
+ ]
559
+ per_group_block_scores = torch.full(
560
+ (
561
+ batch_size,
562
+ num_groups,
563
+ chunk_len,
564
+ num_blocks,
565
+ ),
566
+ float("-inf"),
567
+ dtype=torch.float32,
568
+ device=index_query.device,
569
+ )
570
+ # Match the Megatron max-route oracle while bounding the eager
571
+ # token-score temporary for 256K contexts.
572
+ key_blocks_per_chunk = max(
573
+ 1,
574
+ 8192 // self.block_size,
575
+ )
576
+ for block_start in range(
577
+ 0,
578
+ num_blocks,
579
+ key_blocks_per_chunk,
580
+ ):
581
+ block_end = min(
582
+ block_start + key_blocks_per_chunk,
583
+ num_blocks,
584
+ )
585
+ key_start = block_start * self.block_size
586
+ key_end = min(
587
+ block_end * self.block_size,
588
+ key_len,
589
+ )
590
+ token_scores = torch.einsum(
591
+ "bgqd,bkd->bgqk",
592
+ query_chunk.float(),
593
+ index_key[:, key_start:key_end].float(),
594
+ )
595
+ token_scores.mul_(self.indexer.index_softmax_scale)
596
+ token_scores.masked_fill_(
597
+ ~allowed[
598
+ :,
599
+ None,
600
+ query_start:query_end,
601
+ key_start:key_end,
602
+ ],
603
+ float("-inf"),
604
+ )
605
+ block_count = block_end - block_start
606
+ padding = (
607
+ block_count * self.block_size
608
+ - (key_end - key_start)
609
+ )
610
+ if padding:
611
+ token_scores = F.pad(
612
+ token_scores,
613
+ (0, padding),
614
+ value=float("-inf"),
615
+ )
616
+ per_group_block_scores[
617
+ ...,
618
+ block_start:block_end,
619
+ ] = token_scores.view(
620
+ batch_size,
621
+ num_groups,
622
+ chunk_len,
623
+ block_count,
624
+ self.block_size,
625
+ ).amax(dim=-1)
626
+ route_scores = per_group_block_scores.permute(
627
+ 0, 2, 1, 3
628
+ ).contiguous()
629
+ current_block = torch.div(
630
+ query_positions[:, query_start:query_end],
631
+ self.block_size,
632
+ rounding_mode="floor",
633
+ )
634
+ local_start_block = (
635
+ current_block - self.num_local_blocks + 1
636
+ ).clamp(min=0)
637
+ local_start = local_start_block * self.block_size
638
+ eligible = (
639
+ block_starts.view(1, 1, -1)
640
+ >= self.prefix_window_size
641
+ ) & (
642
+ block_ends.view(1, 1, -1)
643
+ <= local_start.unsqueeze(-1)
644
+ )
645
+ route_scores.masked_fill_(
646
+ ~eligible.unsqueeze(2),
647
+ float("-inf"),
648
+ )
649
+ values, indices = route_scores.topk(
650
+ selected_count,
651
+ dim=-1,
652
+ )
653
+ indices = torch.where(
654
+ torch.isfinite(values),
655
+ indices,
656
+ -1,
657
+ )
658
+ if selected_count < self.num_remote_blocks:
659
+ indices = F.pad(
660
+ indices,
661
+ (0, self.num_remote_blocks - selected_count),
662
+ value=-1,
663
+ )
664
+ selected_chunks.append(indices)
665
+
666
+ return torch.cat(selected_chunks, dim=1).contiguous()
667
+
668
+ def _build_block_aligned_route(self, remote_blocks, query_positions):
669
+ """Add the same fixed prefix/local blocks to every group route."""
670
+ if remote_blocks.ndim != 4:
671
+ raise ValueError(
672
+ "per-group remote blocks must be [batch, query, groups, remote]"
673
+ )
674
+ batch_size, query_len, num_groups, _ = remote_blocks.shape
675
+ current_block = torch.div(
676
+ query_positions,
677
+ self.block_size,
678
+ rounding_mode="floor",
679
+ ).unsqueeze(-1).unsqueeze(-1)
680
+
681
+ prefix = torch.arange(
682
+ self.num_prefix_blocks,
683
+ device=remote_blocks.device,
684
+ dtype=remote_blocks.dtype,
685
+ ).view(1, 1, 1, -1)
686
+ prefix = prefix.expand(
687
+ batch_size, query_len, num_groups, -1
688
+ )
689
+ prefix = torch.where(prefix <= current_block, prefix, -1)
690
+
691
+ local_offsets = torch.arange(
692
+ 1 - self.num_local_blocks,
693
+ 1,
694
+ device=remote_blocks.device,
695
+ dtype=remote_blocks.dtype,
696
+ ).view(1, 1, 1, -1)
697
+ local = current_block.to(remote_blocks.dtype) + local_offsets
698
+ local = local.expand(
699
+ batch_size, query_len, num_groups, -1
700
+ )
701
+ local = torch.where(local >= self.num_prefix_blocks, local, -1)
702
+ return torch.cat(
703
+ (prefix, remote_blocks, local),
704
+ dim=-1,
705
+ ).contiguous()
706
+
707
+ def _sparse_attention(
708
+ self,
709
+ query_states,
710
+ key_states,
711
+ value_states,
712
+ selected_blocks,
713
+ allowed,
714
+ ):
715
+ batch_size, num_query_heads, query_len, head_dim = (
716
+ query_states.shape
717
+ )
718
+ num_groups = key_states.shape[1]
719
+ key_len = key_states.shape[2]
720
+ if selected_blocks.ndim != 4 or selected_blocks.shape[:3] != (
721
+ batch_size,
722
+ query_len,
723
+ num_groups,
724
+ ):
725
+ raise ValueError(
726
+ "selected blocks must be [batch, query, groups, blocks]"
727
+ )
728
+ heads_per_group = num_query_heads // num_groups
729
+ selected_tokens = selected_blocks.shape[-1] * self.block_size
730
+ block_offsets = torch.arange(
731
+ self.block_size,
732
+ device=query_states.device,
733
+ )
734
+ output = torch.empty(
735
+ batch_size,
736
+ num_query_heads,
737
+ query_len,
738
+ head_dim,
739
+ dtype=value_states.dtype,
740
+ device=value_states.device,
741
+ )
742
+
743
+ for query_start in range(0, query_len, self.query_chunk_size):
744
+ query_end = min(
745
+ query_start + self.query_chunk_size,
746
+ query_len,
747
+ )
748
+ chunk_len = query_end - query_start
749
+ selected_chunk = selected_blocks[
750
+ :, query_start:query_end
751
+ ].permute(0, 2, 1, 3)
752
+ token_positions = (
753
+ selected_chunk.unsqueeze(-1)
754
+ * self.block_size
755
+ + block_offsets
756
+ ).reshape(
757
+ batch_size,
758
+ num_groups,
759
+ chunk_len,
760
+ selected_tokens,
761
+ )
762
+ in_range = (
763
+ (token_positions >= 0)
764
+ & (token_positions < key_len)
765
+ )
766
+ gather_positions = token_positions.clamp(
767
+ min=0,
768
+ max=key_len - 1
769
+ )
770
+ allowed_chunk = allowed[
771
+ :, None, query_start:query_end
772
+ ].expand(
773
+ batch_size,
774
+ num_groups,
775
+ chunk_len,
776
+ key_len,
777
+ )
778
+ valid = in_range & torch.gather(
779
+ allowed_chunk,
780
+ 3,
781
+ gather_positions,
782
+ )
783
+ gather_index = gather_positions[..., None].expand(
784
+ batch_size,
785
+ num_groups,
786
+ chunk_len,
787
+ selected_tokens,
788
+ head_dim,
789
+ )
790
+ selected_key = torch.gather(
791
+ key_states.unsqueeze(2).expand(
792
+ batch_size,
793
+ num_groups,
794
+ chunk_len,
795
+ key_len,
796
+ head_dim,
797
+ ),
798
+ 3,
799
+ gather_index,
800
+ )
801
+ selected_value = torch.gather(
802
+ value_states.unsqueeze(2).expand(
803
+ batch_size,
804
+ num_groups,
805
+ chunk_len,
806
+ key_len,
807
+ head_dim,
808
+ ),
809
+ 3,
810
+ gather_index,
811
+ )
812
+ grouped_query = query_states[
813
+ :, :, query_start:query_end
814
+ ].reshape(
815
+ batch_size,
816
+ num_groups,
817
+ heads_per_group,
818
+ chunk_len,
819
+ head_dim,
820
+ )
821
+ # Accumulate scores in fp32 like the Triton kernels; a bf16 einsum
822
+ # output quantizes logits of ~1e3 to steps of several units.
823
+ scores = torch.einsum(
824
+ "bghqd,bgqkd->bghqk",
825
+ grouped_query.float(),
826
+ selected_key.float(),
827
+ )
828
+ scores.mul_(self.softmax_scale)
829
+ scores.masked_fill_(
830
+ ~valid[:, :, None],
831
+ float("-inf"),
832
+ )
833
+ probabilities = torch.softmax(
834
+ scores,
835
+ dim=-1,
836
+ dtype=torch.float32,
837
+ )
838
+ probabilities = torch.nan_to_num(
839
+ probabilities,
840
+ nan=0.0,
841
+ ).to(selected_value.dtype)
842
+ context = torch.einsum(
843
+ "bghqk,bgqkd->bghqd",
844
+ probabilities,
845
+ selected_value,
846
+ )
847
+ output[
848
+ :, :, query_start:query_end
849
+ ] = context.reshape(
850
+ batch_size,
851
+ num_query_heads,
852
+ chunk_len,
853
+ head_dim,
854
+ )
855
+
856
+ return output
857
+
858
+ def forward(
859
+ self, hidden_states, attention_mask=None, position_ids=None,
860
+ past_key_value=None, output_attentions=False, use_cache=False,
861
+ cache_position=None, position_embeddings=None,
862
+ ):
863
+ if output_attentions:
864
+ raise ValueError(
865
+ "block_sparse_gqa does not materialize dense attention weights"
866
+ )
867
+ batch_size, query_len, _ = hidden_states.shape
868
+ query_states = self.q_proj(hidden_states).view(
869
+ batch_size,
870
+ query_len,
871
+ self.num_heads,
872
+ self.head_dim,
873
+ )
874
+ key_states = self.k_proj(hidden_states).view(
875
+ batch_size,
876
+ query_len,
877
+ self.num_key_value_heads,
878
+ self.head_dim,
879
+ )
880
+ value_states = self.v_proj(hidden_states).view(
881
+ batch_size,
882
+ query_len,
883
+ self.num_key_value_heads,
884
+ self.head_dim,
885
+ )
886
+
887
+ if self.qk_layernorm:
888
+ query_states = self.q_layernorm(query_states)
889
+ key_states = self.k_layernorm(key_states)
890
+
891
+ query_states = query_states.transpose(1, 2)
892
+ key_states = key_states.transpose(1, 2)
893
+ value_states = value_states.transpose(1, 2)
894
+
895
+ if position_embeddings is None:
896
+ cos, sin = self.rotary_emb(value_states, position_ids)
897
+ else:
898
+ cos, sin = position_embeddings
899
+ query_states, key_states = apply_rotary_pos_emb(
900
+ query_states,
901
+ key_states,
902
+ cos,
903
+ sin,
904
+ position_ids,
905
+ )
906
+ index_query, index_key = self.indexer.project_qk(
907
+ hidden_states,
908
+ cos,
909
+ sin,
910
+ )
911
+
912
+ if past_key_value is not None:
913
+ if not isinstance(
914
+ past_key_value,
915
+ MaxwellMoeBlockSparseGQACache,
916
+ ):
917
+ raise TypeError(
918
+ "block_sparse_gqa requires "
919
+ "MaxwellMoeBlockSparseGQACache"
920
+ )
921
+ cache_kwargs = {
922
+ "sin": sin,
923
+ "cos": cos,
924
+ "cache_position": cache_position,
925
+ }
926
+ key_states, value_states = past_key_value.update(
927
+ key_states,
928
+ value_states,
929
+ self.layer_idx,
930
+ cache_kwargs,
931
+ )
932
+ index_key = past_key_value.update_indexer_key_state(
933
+ index_key,
934
+ self.layer_idx,
935
+ )
936
+ if key_states.shape[-2] != index_key.shape[1]:
937
+ raise RuntimeError(
938
+ "main KV and indexer-K cache lengths diverged: "
939
+ f"{key_states.shape[-2]} != {index_key.shape[1]}"
940
+ )
941
+
942
+ query_positions = self._query_positions(
943
+ batch_size=batch_size,
944
+ query_len=query_len,
945
+ key_len=key_states.shape[-2],
946
+ device=query_states.device,
947
+ cache_position=cache_position,
948
+ )
949
+ allowed = self._allowed_keys(
950
+ query_positions=query_positions,
951
+ key_len=key_states.shape[-2],
952
+ attention_mask=attention_mask,
953
+ )
954
+ selected_blocks = self._select_blocks(
955
+ index_query,
956
+ index_key,
957
+ query_positions,
958
+ allowed,
959
+ )
960
+ route_blocks = self._build_block_aligned_route(
961
+ selected_blocks,
962
+ query_positions,
963
+ )
964
+ attention_output = self._sparse_attention(
965
+ query_states,
966
+ key_states,
967
+ value_states,
968
+ route_blocks,
969
+ allowed,
970
+ )
971
+ attention_output = attention_output.transpose(
972
+ 1, 2
973
+ ).contiguous().reshape(
974
+ batch_size,
975
+ query_len,
976
+ self.num_heads * self.head_dim,
977
+ )
978
+ return self.o_proj(attention_output), None, past_key_value
979
+
980
+
981
+ class MaxwellMoeFlashAttention2(MaxwellMoeAttention):
982
+ def __init__(self, *args, **kwargs):
983
+ super().__init__(*args, **kwargs)
984
+ self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
985
+
986
+ def forward(
987
+ self, hidden_states, attention_mask=None, position_ids=None,
988
+ past_key_value=None, output_attentions=False, use_cache=False,
989
+ cache_position=None, position_embeddings=None,
990
+ ):
991
+ bsz, q_len, _ = hidden_states.size()
992
+
993
+ query_states = self.q_proj(hidden_states)
994
+ key_states = self.k_proj(hidden_states)
995
+ value_states = self.v_proj(hidden_states)
996
+
997
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim)
998
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim)
999
+
1000
+ if self.qk_layernorm:
1001
+ query_states = self.q_layernorm(query_states)
1002
+ key_states = self.k_layernorm(key_states)
1003
+
1004
+ query_states = query_states.transpose(1, 2)
1005
+ key_states = key_states.transpose(1, 2)
1006
+ value_states = value_states.view(
1007
+ bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
1008
+
1009
+ if position_embeddings is None:
1010
+ cos, sin = self.rotary_emb(value_states, position_ids)
1011
+ else:
1012
+ cos, sin = position_embeddings
1013
+ query_states, key_states = apply_rotary_pos_emb(
1014
+ query_states, key_states, cos, sin)
1015
+
1016
+ if past_key_value is not None:
1017
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
1018
+ key_states, value_states = past_key_value.update(
1019
+ key_states, value_states, self.layer_idx, cache_kwargs)
1020
+
1021
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
1022
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
1023
+ dropout_rate = 0.0 if not self.training else self.attention_dropout
1024
+
1025
+ input_dtype = query_states.dtype
1026
+ if input_dtype == torch.float32:
1027
+ if torch.is_autocast_enabled():
1028
+ target_dtype = torch.get_autocast_gpu_dtype()
1029
+ elif hasattr(self.config, "_pre_quantization_dtype"):
1030
+ target_dtype = self.config._pre_quantization_dtype
1031
+ else:
1032
+ target_dtype = self.q_proj.weight.dtype
1033
+ query_states = query_states.to(target_dtype)
1034
+ key_states = key_states.to(target_dtype)
1035
+ value_states = value_states.to(target_dtype)
1036
+
1037
+ query_states = query_states.transpose(1, 2)
1038
+ key_states = key_states.transpose(1, 2)
1039
+ value_states = value_states.transpose(1, 2)
1040
+
1041
+ attn_output = _flash_attention_forward(
1042
+ query_states, key_states, value_states, attention_mask, q_len,
1043
+ position_ids=position_ids, dropout=dropout_rate,
1044
+ sliding_window=None, is_causal=self.is_causal,
1045
+ use_top_left_mask=self._flash_attn_uses_top_left_mask,
1046
+ )
1047
+
1048
+ attn_output = attn_output.reshape(
1049
+ bsz, q_len, self.num_heads * self.head_dim).contiguous()
1050
+ attn_output = self.o_proj(attn_output)
1051
+
1052
+ if not output_attentions:
1053
+ attn_weights = None
1054
+ return attn_output, attn_weights, past_key_value
1055
+
1056
+
1057
+ class MaxwellMoeSdpaAttention(MaxwellMoeAttention):
1058
+ def forward(
1059
+ self, hidden_states, attention_mask=None, position_ids=None,
1060
+ past_key_value=None, output_attentions=False, use_cache=False,
1061
+ cache_position=None, position_embeddings=None,
1062
+ ):
1063
+ if output_attentions:
1064
+ return super().forward(
1065
+ hidden_states=hidden_states, attention_mask=attention_mask,
1066
+ position_ids=position_ids, past_key_value=past_key_value,
1067
+ output_attentions=output_attentions, use_cache=use_cache,
1068
+ )
1069
+
1070
+ bsz, q_len, _ = hidden_states.size()
1071
+
1072
+ query_states = self.q_proj(hidden_states)
1073
+ key_states = self.k_proj(hidden_states)
1074
+ value_states = self.v_proj(hidden_states)
1075
+
1076
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim)
1077
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim)
1078
+
1079
+ if self.qk_layernorm:
1080
+ query_states = self.q_layernorm(query_states)
1081
+ key_states = self.k_layernorm(key_states)
1082
+
1083
+ query_states = query_states.transpose(1, 2)
1084
+ key_states = key_states.transpose(1, 2)
1085
+ value_states = value_states.view(
1086
+ bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
1087
+
1088
+ if position_embeddings is None:
1089
+ cos, sin = self.rotary_emb(value_states, position_ids)
1090
+ else:
1091
+ cos, sin = position_embeddings
1092
+ query_states, key_states = apply_rotary_pos_emb(
1093
+ query_states, key_states, cos, sin)
1094
+
1095
+ if past_key_value is not None:
1096
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
1097
+ key_states, value_states = past_key_value.update(
1098
+ key_states, value_states, self.layer_idx, cache_kwargs)
1099
+
1100
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
1101
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
1102
+
1103
+ causal_mask = attention_mask
1104
+ if attention_mask is not None:
1105
+ causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
1106
+
1107
+ if query_states.device.type == "cuda" and attention_mask is not None:
1108
+ query_states = query_states.contiguous()
1109
+ key_states = key_states.contiguous()
1110
+ value_states = value_states.contiguous()
1111
+
1112
+ is_causal = True if causal_mask is None and q_len > 1 else False
1113
+ attn_output = torch.nn.functional.scaled_dot_product_attention(
1114
+ query_states, key_states, value_states,
1115
+ attn_mask=causal_mask,
1116
+ dropout_p=self.attention_dropout if self.training else 0.0,
1117
+ is_causal=is_causal,
1118
+ )
1119
+
1120
+ attn_output = attn_output.transpose(1, 2).contiguous()
1121
+ attn_output = attn_output.view(bsz, q_len, self.num_heads * self.head_dim)
1122
+ attn_output = self.o_proj(attn_output)
1123
+
1124
+ return attn_output, None, past_key_value
1125
+
1126
+
1127
+ class MaxwellMoeTeAttention(MaxwellMoeAttention):
1128
+ """Attention using Transformer Engine's DotProductAttention kernel."""
1129
+
1130
+ def __init__(self, config, layer_idx=None):
1131
+ super().__init__(config, layer_idx)
1132
+ try:
1133
+ import transformer_engine.pytorch as te
1134
+ self.te_attn = te.DotProductAttention(
1135
+ num_attention_heads=self.num_heads,
1136
+ kv_channels=self.head_dim,
1137
+ num_gqa_groups=self.num_key_value_heads,
1138
+ attention_dropout=self.attention_dropout,
1139
+ qkv_format="bshd",
1140
+ attn_mask_type="causal",
1141
+ softmax_scale=1.0 / math.sqrt(self.head_dim),
1142
+ layer_number=layer_idx or 1,
1143
+ )
1144
+ except ImportError:
1145
+ raise ImportError("transformer_engine is required for attn_implementation='te'")
1146
+
1147
+ def forward(
1148
+ self, hidden_states, attention_mask=None, position_ids=None,
1149
+ past_key_value=None, output_attentions=False, use_cache=False,
1150
+ cache_position=None, position_embeddings=None,
1151
+ ):
1152
+ bsz, q_len, _ = hidden_states.size()
1153
+
1154
+ query_states = self.q_proj(hidden_states)
1155
+ key_states = self.k_proj(hidden_states)
1156
+ value_states = self.v_proj(hidden_states)
1157
+
1158
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim)
1159
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim)
1160
+ value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim)
1161
+
1162
+ if self.qk_layernorm:
1163
+ query_states = self.q_layernorm(query_states)
1164
+ key_states = self.k_layernorm(key_states)
1165
+
1166
+ if position_embeddings is None:
1167
+ cos, sin = self.rotary_emb(value_states, position_ids)
1168
+ else:
1169
+ cos, sin = position_embeddings
1170
+
1171
+ # RoPE expects [bsz, heads, seq, dim]
1172
+ q_t = query_states.transpose(1, 2)
1173
+ k_t = key_states.transpose(1, 2)
1174
+ q_t, k_t = apply_rotary_pos_emb(q_t, k_t, cos, sin)
1175
+ query_states = q_t.transpose(1, 2).contiguous()
1176
+ key_states = k_t.transpose(1, 2).contiguous()
1177
+
1178
+ if past_key_value is not None:
1179
+ k_for_cache = key_states.transpose(1, 2)
1180
+ v_for_cache = value_states.transpose(1, 2)
1181
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
1182
+ k_for_cache, v_for_cache = past_key_value.update(
1183
+ k_for_cache, v_for_cache, self.layer_idx, cache_kwargs)
1184
+ key_states = k_for_cache.transpose(1, 2).contiguous()
1185
+ value_states = v_for_cache.transpose(1, 2).contiguous()
1186
+
1187
+ # TE DotProductAttention expects [bsz, seq, heads, dim] for bshd format
1188
+ attn_output = self.te_attn(query_states, key_states, value_states)
1189
+
1190
+ attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.head_dim)
1191
+ attn_output = self.o_proj(attn_output)
1192
+ return attn_output, None, past_key_value
1193
+
1194
+
1195
+ MAXWELLMOE_ATTENTION_CLASSES = {
1196
+ "eager": MaxwellMoeAttention,
1197
+ "flash_attention_2": MaxwellMoeFlashAttention2,
1198
+ "sdpa": MaxwellMoeSdpaAttention,
1199
+ "te": MaxwellMoeTeAttention,
1200
+ }
1201
+
1202
+
1203
+ # ---------------------------------------------------------------------------
1204
+ # mHC (Manifold-Constrained Hyper-Connections)
1205
+ # ---------------------------------------------------------------------------
1206
+
1207
+ def _sinkhorn_iterations(M: torch.Tensor, num_iterations: int, eps: float = 0.0):
1208
+ """TileKernels-aligned Sinkhorn-Knopp projection.
1209
+
1210
+ Mirrors the TileKernels mhc_sinkhorn_kernel forward pass:
1211
+ 1. M = softmax(logits, dim=-1) + eps
1212
+ 2. M = M / (M.sum(dim=-2) + eps) # col norm
1213
+ 3. repeat-1 times:
1214
+ M = M / (M.sum(dim=-1) + eps) # row norm
1215
+ M = M / (M.sum(dim=-2) + eps) # col norm
1216
+
1217
+ Input M is the softmax output from _compute_mappings.
1218
+ Megatron --use-tile-mhc passes eps=0.0 (tile_mhc_kernels.py:169-170).
1219
+ DeepSeek V4 HuggingFace uses eps=1e-6 (config.hc_eps).
1220
+ """
1221
+ # softmax + eps (softmax already applied in _compute_mappings)
1222
+ M = M + eps
1223
+
1224
+ # comb = comb / (comb.sum(-2) + eps)
1225
+ M = M / (M.sum(dim=-2, keepdim=True) + eps)
1226
+
1227
+ for _ in range(num_iterations - 1):
1228
+ # comb = comb / (comb.sum(-1) + eps)
1229
+ M = M / (M.sum(dim=-1, keepdim=True) + eps)
1230
+ # comb = comb / (comb.sum(-2) + eps)
1231
+ M = M / (M.sum(dim=-2, keepdim=True) + eps)
1232
+
1233
+ return M
1234
+
1235
+
1236
+ class HyperConnectionModule(nn.Module):
1237
+ def __init__(self, config: MaxwellMoeConfig):
1238
+ super().__init__()
1239
+ self.n = config.num_residual_streams
1240
+ self.hidden_size = config.hidden_size
1241
+ self.sinkhorn_iterations = config.mhc_sinkhorn_iterations
1242
+ self.eps = 1e-6
1243
+ mix_dim = self.n * self.n + 2 * self.n
1244
+
1245
+ self.mapping_proj = nn.Linear(self.n * self.hidden_size, mix_dim, bias=False)
1246
+ self.alpha_pre = nn.Parameter(torch.full((1,), config.mhc_init_gating_factor))
1247
+ self.alpha_post = nn.Parameter(torch.full((1,), config.mhc_init_gating_factor))
1248
+ self.alpha_res = nn.Parameter(torch.full((1,), config.mhc_init_gating_factor))
1249
+ self.bias = nn.Parameter(torch.zeros(mix_dim))
1250
+
1251
+ def _compute_mappings(self, x):
1252
+ b, s, nC = x.shape
1253
+ n = self.n
1254
+ norm = x.float().norm(dim=-1, keepdim=True)
1255
+ r = 1.0 / (norm / math.sqrt(nC) + self.eps)
1256
+ proj = F.linear(x.float(), self.mapping_proj.weight.float())
1257
+ alpha = torch.cat([
1258
+ self.alpha_pre.expand(n),
1259
+ self.alpha_post.expand(n),
1260
+ self.alpha_res.expand(n * n),
1261
+ ], dim=-1).float()
1262
+ h = r.float() * proj * alpha + self.bias.float()
1263
+ h_pre = (h[..., :n].sigmoid() + self.eps).to(x.dtype)
1264
+ h_post = (h[..., n:2*n].sigmoid() * 2).to(x.dtype)
1265
+ h_res_logits = h[..., 2*n:]
1266
+ h_res = torch.softmax(h_res_logits.view(b, s, n, n), dim=-1)
1267
+ h_res = _sinkhorn_iterations(h_res, self.sinkhorn_iterations, eps=self.eps)
1268
+ h_res = h_res.to(x.dtype)
1269
+ return h_pre, h_post, h_res
1270
+
1271
+ def aggregate(self, x, h_pre):
1272
+ b, s, _ = x.shape
1273
+ C = self.hidden_size
1274
+ x_streams = x.view(b, s, self.n, C).float()
1275
+ return (x_streams * h_pre.float().unsqueeze(-1)).sum(dim=2).to(x.dtype)
1276
+
1277
+ def expand_and_mix(self, layer_output, residual, h_post, h_res):
1278
+ b, s, C = layer_output.shape
1279
+ n = self.n
1280
+ res_streams = residual.view(b, s, n, C).float()
1281
+ h_res_batched = h_res.view(b * s, n, n).float()
1282
+ res_batched = res_streams.view(b * s, n, C)
1283
+ mixed = torch.bmm(h_res_batched.transpose(-1, -2), res_batched).view(b, s, n, C)
1284
+ expanded = h_post.float().unsqueeze(-1) * layer_output.float().unsqueeze(2)
1285
+ return (expanded + mixed).to(residual.dtype).view(b, s, n * C)
1286
+
1287
+ @staticmethod
1288
+ def input_expand(x, n):
1289
+ b, s, C = x.shape
1290
+ return x.unsqueeze(2).expand(b, s, n, C).contiguous().view(b, s, n * C)
1291
+
1292
+ @staticmethod
1293
+ def output_contract(x, n):
1294
+ b, s, nC = x.shape
1295
+ C = nC // n
1296
+ return x.view(b, s, n, C).mean(dim=2)
1297
+
1298
+
1299
+ class HCHead(nn.Module):
1300
+ """Learned n-stream to 1-stream contraction replacing simple averaging."""
1301
+
1302
+ def __init__(self, config: MaxwellMoeConfig):
1303
+ super().__init__()
1304
+ self.n = config.num_residual_streams
1305
+ self.hidden_size = config.hidden_size
1306
+ self.eps = 1e-6
1307
+ self.hc_head_fn = nn.Parameter(
1308
+ torch.zeros(self.n, self.n * self.hidden_size))
1309
+ self.hc_head_base = nn.Parameter(torch.zeros(self.n))
1310
+ self.hc_head_scale = nn.Parameter(torch.ones(1))
1311
+
1312
+ def forward(self, x):
1313
+ """
1314
+ Args:
1315
+ x: [b, s, n*C] n-stream hidden states
1316
+ Returns:
1317
+ [b, s, C] contracted single-stream
1318
+ """
1319
+ b, s, nC = x.shape
1320
+ n = self.n
1321
+ C = self.hidden_size
1322
+ dtype = x.dtype
1323
+
1324
+ x_flat = x.float()
1325
+ rsqrt = torch.rsqrt(x_flat.square().mean(-1, keepdim=True) + self.eps)
1326
+
1327
+ proj = F.linear(x_flat, self.hc_head_fn.float())
1328
+ mixes = proj * rsqrt
1329
+ weights = torch.sigmoid(mixes * self.hc_head_scale.float() + self.hc_head_base.float()) + self.eps
1330
+
1331
+ x_streams = x.view(b, s, n, C).float()
1332
+ out = (x_streams * weights.unsqueeze(-1)).sum(dim=2)
1333
+ return out.to(dtype)
1334
+
1335
+
1336
+ # ---------------------------------------------------------------------------
1337
+ # MoE Block with expert bias and sigmoid routing
1338
+ # ---------------------------------------------------------------------------
1339
+
1340
+ class MaxwellMoeSparseMoeBlock(nn.Module):
1341
+ def __init__(self, config):
1342
+ super().__init__()
1343
+ self.num_experts = config.num_experts
1344
+ self.top_k = config.num_experts_per_tok
1345
+ self.use_shared_expert_gate = config.use_shared_expert_gate
1346
+ self.router_score_function = getattr(config, 'router_score_function', 'softmax')
1347
+ self.router_dtype = getattr(config, 'moe_router_dtype', 'fp32')
1348
+
1349
+ self.gate = nn.Linear(config.hidden_size, config.num_experts, bias=False)
1350
+
1351
+ if getattr(config, 'moe_router_enable_expert_bias', False):
1352
+ self.expert_bias = nn.Parameter(torch.zeros(config.num_experts), requires_grad=False)
1353
+
1354
+ self.experts = nn.ModuleList(
1355
+ [MaxwellMoeMLP(config, intermediate_size=config.moe_intermediate_size)
1356
+ for _ in range(self.num_experts)])
1357
+
1358
+ if config.shared_expert_intermediate_size is not None:
1359
+ self.num_shared_experts = 1
1360
+ self.shared_expert = MaxwellMoeMLP(
1361
+ config, intermediate_size=config.shared_expert_intermediate_size)
1362
+ if self.use_shared_expert_gate:
1363
+ self.shared_expert_gate = nn.Linear(config.hidden_size, 1, bias=False)
1364
+ else:
1365
+ self.num_shared_experts = 0
1366
+
1367
+ def _router_dtype(self, input_dtype):
1368
+ if self.router_dtype == 'fp32':
1369
+ return torch.float32
1370
+ if self.router_dtype == 'fp64':
1371
+ return torch.float64
1372
+ return input_dtype
1373
+
1374
+ @staticmethod
1375
+ def _autocast_device_type(tensor):
1376
+ device_type = tensor.device.type
1377
+ return device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
1378
+
1379
+ def _compute_router_logits(self, hidden_states):
1380
+ router_dtype = self._router_dtype(hidden_states.dtype)
1381
+ device_type = self._autocast_device_type(hidden_states)
1382
+ with torch.autocast(device_type=device_type, enabled=False):
1383
+ return F.linear(hidden_states.to(router_dtype), self.gate.weight.to(router_dtype))
1384
+
1385
+ def _maintain_float32_expert_bias(self):
1386
+ if hasattr(self, 'expert_bias') and self.expert_bias.dtype != torch.float32:
1387
+ self.expert_bias.data = self.expert_bias.data.to(torch.float32)
1388
+
1389
+ def forward(self, hidden_states):
1390
+ self._maintain_float32_expert_bias()
1391
+
1392
+ batch_size, sequence_length, hidden_dim = hidden_states.shape
1393
+ hidden_states = hidden_states.view(-1, hidden_dim)
1394
+
1395
+ router_logits = self._compute_router_logits(hidden_states)
1396
+
1397
+ with torch.autocast(device_type=self._autocast_device_type(router_logits), enabled=False):
1398
+ if self.router_score_function == 'sigmoid':
1399
+ scores = torch.sigmoid(router_logits.float())
1400
+ if hasattr(self, 'expert_bias'):
1401
+ scores_for_routing = scores + self.expert_bias.float()
1402
+ _, selected_experts = torch.topk(scores_for_routing, self.top_k, dim=-1)
1403
+ topk_scores = torch.gather(scores, dim=1, index=selected_experts)
1404
+ else:
1405
+ topk_scores, selected_experts = torch.topk(scores, self.top_k, dim=-1)
1406
+ routing_weights = (
1407
+ topk_scores / (topk_scores.sum(dim=-1, keepdim=True) + 1e-20)
1408
+ ).type_as(router_logits)
1409
+ else:
1410
+ topk_routing_weights, selected_experts = torch.topk(
1411
+ router_logits, self.top_k, dim=-1)
1412
+ routing_weights = torch.softmax(
1413
+ topk_routing_weights, dim=1, dtype=torch.float).type_as(router_logits)
1414
+
1415
+ final_hidden_states = torch.zeros(
1416
+ (batch_size * sequence_length, hidden_dim),
1417
+ dtype=hidden_states.dtype, device=hidden_states.device,
1418
+ )
1419
+
1420
+ expert_mask = torch.nn.functional.one_hot(
1421
+ selected_experts, num_classes=self.num_experts).permute(2, 1, 0)
1422
+
1423
+ for expert_idx in range(self.num_experts):
1424
+ expert_layer = self.experts[expert_idx]
1425
+ idx, top_x = torch.where(expert_mask[expert_idx])
1426
+ if top_x.numel() == 0:
1427
+ continue
1428
+ current_state = hidden_states[None, top_x].reshape(-1, hidden_dim)
1429
+ current_hidden_states = (
1430
+ expert_layer(current_state) * routing_weights[top_x, idx, None])
1431
+ final_hidden_states.index_add_(
1432
+ 0, top_x, current_hidden_states.to(hidden_states.dtype))
1433
+
1434
+ if self.num_shared_experts:
1435
+ shared_expert_output = self.shared_expert(hidden_states)
1436
+ if self.use_shared_expert_gate:
1437
+ shared_expert_output = (
1438
+ F.sigmoid(self.shared_expert_gate(hidden_states))
1439
+ * shared_expert_output)
1440
+ final_hidden_states = final_hidden_states + shared_expert_output
1441
+
1442
+ final_hidden_states = final_hidden_states.reshape(
1443
+ batch_size, sequence_length, hidden_dim)
1444
+ return final_hidden_states, router_logits
1445
+
1446
+
1447
+ # ---------------------------------------------------------------------------
1448
+ # Decoder Layer
1449
+ # ---------------------------------------------------------------------------
1450
+
1451
+ class MaxwellMoeDecoderLayer(nn.Module):
1452
+ def __init__(self, config: MaxwellMoeConfig, layer_idx: int):
1453
+ super().__init__()
1454
+ self.hidden_size = config.hidden_size
1455
+ self.enable_hyper_connections = config.enable_hyper_connections
1456
+
1457
+ # Never let Transformers' backend auto-selection replace the
1458
+ # architecture-specific shared block route with plain dense SDPA.
1459
+ self.self_attn = MaxwellMoeBlockSparseGQAAttention(
1460
+ config, layer_idx
1461
+ )
1462
+
1463
+ is_dense = layer_idx in config.mlp_only_layers
1464
+ if is_dense or config.num_experts == 0:
1465
+ self.block_sparse_moe = MaxwellMoeMLP(
1466
+ config, intermediate_size=config.intermediate_size)
1467
+ else:
1468
+ self.block_sparse_moe = MaxwellMoeSparseMoeBlock(config)
1469
+
1470
+ self.input_layernorm = MaxwellMoeRMSNorm(
1471
+ config.hidden_size, eps=config.rms_norm_eps)
1472
+ self.post_attention_layernorm = MaxwellMoeRMSNorm(
1473
+ config.hidden_size, eps=config.rms_norm_eps)
1474
+
1475
+ if self.enable_hyper_connections:
1476
+ self.self_attn_hyper_connection = HyperConnectionModule(config)
1477
+ self.mlp_hyper_connection = HyperConnectionModule(config)
1478
+
1479
+ def forward(
1480
+ self, hidden_states, attention_mask=None, position_ids=None,
1481
+ past_key_value=None, output_attentions=False,
1482
+ output_router_logits=False, use_cache=False,
1483
+ cache_position=None, position_embeddings=None, **kwargs,
1484
+ ):
1485
+ if self.enable_hyper_connections:
1486
+ return self._forward_with_hc(
1487
+ hidden_states, attention_mask, position_ids, past_key_value,
1488
+ output_attentions, output_router_logits, use_cache,
1489
+ cache_position, position_embeddings, **kwargs)
1490
+ else:
1491
+ return self._forward_standard(
1492
+ hidden_states, attention_mask, position_ids, past_key_value,
1493
+ output_attentions, output_router_logits, use_cache,
1494
+ cache_position, position_embeddings, **kwargs)
1495
+
1496
+ def _forward_standard(
1497
+ self, hidden_states, attention_mask, position_ids, past_key_value,
1498
+ output_attentions, output_router_logits, use_cache,
1499
+ cache_position, position_embeddings, **kwargs,
1500
+ ):
1501
+ residual = hidden_states
1502
+ hidden_states = self.input_layernorm(hidden_states)
1503
+ hidden_states, self_attn_weights, present_key_value = self.self_attn(
1504
+ hidden_states=hidden_states, attention_mask=attention_mask,
1505
+ position_ids=position_ids, past_key_value=past_key_value,
1506
+ output_attentions=output_attentions, use_cache=use_cache,
1507
+ cache_position=cache_position, position_embeddings=position_embeddings,
1508
+ )
1509
+ hidden_states = residual + hidden_states
1510
+
1511
+ residual = hidden_states
1512
+ hidden_states = self.post_attention_layernorm(hidden_states)
1513
+ hidden_states = self.block_sparse_moe(hidden_states)
1514
+ if isinstance(hidden_states, tuple):
1515
+ hidden_states, router_logits = hidden_states
1516
+ else:
1517
+ router_logits = None
1518
+ hidden_states = residual + hidden_states
1519
+
1520
+ outputs = (hidden_states,)
1521
+ if output_attentions:
1522
+ outputs += (self_attn_weights,)
1523
+ if use_cache:
1524
+ outputs += (present_key_value,)
1525
+ if output_router_logits:
1526
+ outputs += (router_logits,)
1527
+ return outputs
1528
+
1529
+ def _forward_with_hc(
1530
+ self, hidden_states, attention_mask, position_ids, past_key_value,
1531
+ output_attentions, output_router_logits, use_cache,
1532
+ cache_position, position_embeddings, **kwargs,
1533
+ ):
1534
+ # === Self-Attention with mHC ===
1535
+ residual = hidden_states
1536
+ hc = self.self_attn_hyper_connection
1537
+ h_pre, h_post, h_res = hc._compute_mappings(hidden_states)
1538
+ aggregated = hc.aggregate(hidden_states, h_pre)
1539
+ aggregated = self.input_layernorm(aggregated)
1540
+ attn_output, self_attn_weights, present_key_value = self.self_attn(
1541
+ hidden_states=aggregated, attention_mask=attention_mask,
1542
+ position_ids=position_ids, past_key_value=past_key_value,
1543
+ output_attentions=output_attentions, use_cache=use_cache,
1544
+ cache_position=cache_position, position_embeddings=position_embeddings,
1545
+ )
1546
+ hidden_states = hc.expand_and_mix(attn_output, residual, h_post, h_res)
1547
+
1548
+ # === MLP with mHC ===
1549
+ residual = hidden_states
1550
+ hc_mlp = self.mlp_hyper_connection
1551
+ h_pre, h_post, h_res = hc_mlp._compute_mappings(hidden_states)
1552
+ aggregated = hc_mlp.aggregate(hidden_states, h_pre)
1553
+ aggregated = self.post_attention_layernorm(aggregated)
1554
+ mlp_output = self.block_sparse_moe(aggregated)
1555
+ if isinstance(mlp_output, tuple):
1556
+ mlp_output, router_logits = mlp_output
1557
+ else:
1558
+ router_logits = None
1559
+ hidden_states = hc_mlp.expand_and_mix(mlp_output, residual, h_post, h_res)
1560
+
1561
+ outputs = (hidden_states,)
1562
+ if output_attentions:
1563
+ outputs += (self_attn_weights,)
1564
+ if use_cache:
1565
+ outputs += (present_key_value,)
1566
+ if output_router_logits:
1567
+ outputs += (router_logits,)
1568
+ return outputs
1569
+
1570
+
1571
+ # ---------------------------------------------------------------------------
1572
+ # Model
1573
+ # ---------------------------------------------------------------------------
1574
+
1575
+ MAXWELLMOE_START_DOCSTRING = ""
1576
+ MAXWELLMOE_INPUTS_DOCSTRING = ""
1577
+
1578
+
1579
+ class MaxwellMoePreTrainedModel(PreTrainedModel):
1580
+ config_class = MaxwellMoeConfig
1581
+ base_model_prefix = "model"
1582
+ supports_gradient_checkpointing = True
1583
+ _no_split_modules = ["MaxwellMoeDecoderLayer"]
1584
+ _skip_keys_device_placement = "past_key_values"
1585
+ _supports_flash_attn_2 = False
1586
+ _supports_sdpa = False
1587
+ _supports_cache_class = True
1588
+
1589
+ def _init_weights(self, module):
1590
+ std = self.config.initializer_range
1591
+ if isinstance(module, nn.Linear):
1592
+ module.weight.data.normal_(mean=0.0, std=std)
1593
+ if module.bias is not None:
1594
+ module.bias.data.zero_()
1595
+ elif isinstance(module, nn.Embedding):
1596
+ module.weight.data.normal_(mean=0.0, std=std)
1597
+ if module.padding_idx is not None:
1598
+ module.weight.data[module.padding_idx].zero_()
1599
+
1600
+
1601
+ class MaxwellMoeModel(MaxwellMoePreTrainedModel):
1602
+ def __init__(self, config: MaxwellMoeConfig):
1603
+ super().__init__(config)
1604
+ self.padding_idx = config.pad_token_id
1605
+ self.vocab_size = config.vocab_size
1606
+ self.enable_hyper_connections = config.enable_hyper_connections
1607
+ self.num_residual_streams = config.num_residual_streams
1608
+
1609
+ self.embed_tokens = nn.Embedding(
1610
+ config.vocab_size, config.hidden_size, self.padding_idx)
1611
+ self.layers = nn.ModuleList(
1612
+ [MaxwellMoeDecoderLayer(config, layer_idx)
1613
+ for layer_idx in range(config.num_hidden_layers)])
1614
+ self._attn_implementation = config._attn_implementation
1615
+ self.norm = MaxwellMoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
1616
+ self.rotary_emb = MaxwellMoeRotaryEmbedding(config=config)
1617
+
1618
+ if self.enable_hyper_connections:
1619
+ self.hc_head = HCHead(config)
1620
+
1621
+ self.gradient_checkpointing = False
1622
+ self.post_init()
1623
+
1624
+ def get_input_embeddings(self):
1625
+ return self.embed_tokens
1626
+
1627
+ def set_input_embeddings(self, value):
1628
+ self.embed_tokens = value
1629
+
1630
+ @staticmethod
1631
+ def _normalize_block_sparse_gqa_attention_mask(attention_mask):
1632
+ """Accept only masks that do not alter physical sparse coordinates.
1633
+
1634
+ Block Sparse GQA defines its route in physical KV-cache coordinates.
1635
+ Padding and packed/custom masks need a
1636
+ logical-to-physical token map that this bounded HF oracle does not
1637
+ implement, so reject them instead of silently changing the route.
1638
+ """
1639
+ if attention_mask is None:
1640
+ return None
1641
+ if not isinstance(attention_mask, torch.Tensor):
1642
+ raise TypeError(
1643
+ "block_sparse_gqa attention_mask must be a torch.Tensor or None"
1644
+ )
1645
+ if attention_mask.dim() == 2:
1646
+ if not torch.all(attention_mask == 1).item():
1647
+ raise ValueError(
1648
+ "block_sparse_gqa HF does not support padding: a 2D "
1649
+ "attention_mask must contain only ones"
1650
+ )
1651
+ # Canonicalize the no-op mask so the model constructs its causal
1652
+ # additive mask in one place.
1653
+ return None
1654
+ if attention_mask.dim() == 4:
1655
+ if attention_mask.dtype == torch.bool:
1656
+ raise ValueError(
1657
+ "block_sparse_gqa HF does not support 4D boolean masks or "
1658
+ "packed inputs"
1659
+ )
1660
+ if not torch.all(attention_mask == 0).item():
1661
+ raise ValueError(
1662
+ "block_sparse_gqa HF does not support nontrivial 4D "
1663
+ "additive masks, padding, or packed inputs"
1664
+ )
1665
+ # An all-zero additive mask is a no-op. Canonicalize it to None
1666
+ # so it cannot accidentally replace the internal causal mask.
1667
+ return None
1668
+ raise ValueError(
1669
+ "block_sparse_gqa HF supports only an all-ones 2D mask, an "
1670
+ "all-zero 4D additive mask, or no attention mask"
1671
+ )
1672
+
1673
+ @staticmethod
1674
+ def _validate_block_sparse_gqa_cache(past_key_values):
1675
+ """Require main K/V and indexer K to remain synchronized."""
1676
+ if past_key_values is None:
1677
+ return
1678
+ if not isinstance(past_key_values, MaxwellMoeBlockSparseGQACache):
1679
+ raise TypeError(
1680
+ "block_sparse_gqa requires MaxwellMoeBlockSparseGQACache; "
1681
+ "ordinary, legacy, static, sliding, and quantized caches omit "
1682
+ "historical indexer K"
1683
+ )
1684
+ main_length = past_key_values.get_seq_length()
1685
+ for layer_idx, state in enumerate(
1686
+ past_key_values.indexer_key_states
1687
+ ):
1688
+ if state is not None and int(state.shape[1]) != main_length:
1689
+ raise ValueError(
1690
+ "block_sparse_gqa cache is inconsistent at layer "
1691
+ f"{layer_idx}: main length={main_length}, "
1692
+ f"indexer length={state.shape[1]}"
1693
+ )
1694
+
1695
+ def forward(
1696
+ self,
1697
+ input_ids=None, attention_mask=None, position_ids=None,
1698
+ past_key_values=None, inputs_embeds=None, use_cache=None,
1699
+ output_attentions=None, output_hidden_states=None,
1700
+ output_router_logits=None, return_dict=None,
1701
+ cache_position=None, output_multistream=False,
1702
+ ):
1703
+ output_attentions = (output_attentions if output_attentions is not None
1704
+ else self.config.output_attentions)
1705
+ output_router_logits = (output_router_logits if output_router_logits is not None
1706
+ else self.config.output_router_logits)
1707
+ output_hidden_states = (output_hidden_states if output_hidden_states is not None
1708
+ else self.config.output_hidden_states)
1709
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
1710
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1711
+
1712
+ if (input_ids is None) ^ (inputs_embeds is not None):
1713
+ raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
1714
+
1715
+ if self.gradient_checkpointing and self.training:
1716
+ raise ValueError(
1717
+ "block_sparse_gqa HF is an inference reference and does not "
1718
+ "support gradient checkpointing"
1719
+ )
1720
+
1721
+ if use_cache:
1722
+ if past_key_values is None:
1723
+ past_key_values = MaxwellMoeBlockSparseGQACache()
1724
+ elif not isinstance(
1725
+ past_key_values, MaxwellMoeBlockSparseGQACache
1726
+ ):
1727
+ existing_length = (
1728
+ past_key_values.get_seq_length()
1729
+ if isinstance(past_key_values, Cache)
1730
+ else len(past_key_values)
1731
+ if isinstance(past_key_values, (tuple, list))
1732
+ else None
1733
+ )
1734
+ if existing_length not in (0, None):
1735
+ raise ValueError(
1736
+ "block_sparse_gqa cannot restore a non-empty cache "
1737
+ "without historical indexer K"
1738
+ )
1739
+ if existing_length is None:
1740
+ raise TypeError(
1741
+ "block_sparse_gqa received an unsupported cache type"
1742
+ )
1743
+ past_key_values = MaxwellMoeBlockSparseGQACache()
1744
+
1745
+ self._validate_block_sparse_gqa_cache(past_key_values)
1746
+ attention_mask = self._normalize_block_sparse_gqa_attention_mask(
1747
+ attention_mask
1748
+ )
1749
+
1750
+ if inputs_embeds is None:
1751
+ inputs_embeds = self.embed_tokens(input_ids)
1752
+
1753
+ if cache_position is None:
1754
+ past_seen_tokens = (past_key_values.get_seq_length()
1755
+ if past_key_values is not None else 0)
1756
+ cache_position = torch.arange(
1757
+ past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1],
1758
+ device=inputs_embeds.device)
1759
+ if position_ids is None:
1760
+ position_ids = cache_position.unsqueeze(0)
1761
+
1762
+ causal_mask = self._update_causal_mask(
1763
+ attention_mask, inputs_embeds, cache_position,
1764
+ past_key_values, output_attentions)
1765
+
1766
+ hidden_states = inputs_embeds
1767
+
1768
+ if self.enable_hyper_connections:
1769
+ hidden_states = HyperConnectionModule.input_expand(
1770
+ hidden_states, self.num_residual_streams)
1771
+
1772
+ position_embeddings = self.rotary_emb(hidden_states, position_ids)
1773
+
1774
+ all_hidden_states = () if output_hidden_states else None
1775
+ all_self_attns = () if output_attentions else None
1776
+ all_router_logits = () if output_router_logits else None
1777
+ next_decoder_cache = None
1778
+
1779
+ for decoder_layer in self.layers:
1780
+ if output_hidden_states:
1781
+ all_hidden_states += (hidden_states,)
1782
+
1783
+ if self.gradient_checkpointing and self.training:
1784
+ layer_outputs = self._gradient_checkpointing_func(
1785
+ decoder_layer.__call__,
1786
+ hidden_states, causal_mask, position_ids, past_key_values,
1787
+ output_attentions, output_router_logits, use_cache,
1788
+ cache_position, position_embeddings,
1789
+ )
1790
+ else:
1791
+ layer_outputs = decoder_layer(
1792
+ hidden_states, attention_mask=causal_mask,
1793
+ position_ids=position_ids, past_key_value=past_key_values,
1794
+ output_attentions=output_attentions,
1795
+ output_router_logits=output_router_logits,
1796
+ use_cache=use_cache, cache_position=cache_position,
1797
+ position_embeddings=position_embeddings,
1798
+ )
1799
+
1800
+ hidden_states = layer_outputs[0]
1801
+ if use_cache:
1802
+ next_decoder_cache = layer_outputs[2 if output_attentions else 1]
1803
+ if output_attentions:
1804
+ all_self_attns += (layer_outputs[1],)
1805
+ if output_router_logits and layer_outputs[-1] is not None:
1806
+ all_router_logits += (layer_outputs[-1],)
1807
+
1808
+ multistream_hidden_states = None
1809
+ if output_multistream and self.enable_hyper_connections:
1810
+ multistream_hidden_states = hidden_states
1811
+
1812
+ if self.enable_hyper_connections:
1813
+ hidden_states = self.hc_head(hidden_states)
1814
+
1815
+ hidden_states = self.norm(hidden_states)
1816
+
1817
+ if output_hidden_states:
1818
+ all_hidden_states += (hidden_states,)
1819
+
1820
+ next_cache = next_decoder_cache if use_cache else None
1821
+
1822
+ if not return_dict:
1823
+ result = tuple(v for v in [
1824
+ hidden_states, next_cache, all_hidden_states,
1825
+ all_self_attns, all_router_logits
1826
+ ] if v is not None)
1827
+ if output_multistream:
1828
+ return result, multistream_hidden_states, position_embeddings, causal_mask, position_ids, cache_position
1829
+ return result
1830
+ model_output = MoeModelOutputWithPast(
1831
+ last_hidden_state=hidden_states,
1832
+ past_key_values=next_cache,
1833
+ hidden_states=all_hidden_states,
1834
+ attentions=all_self_attns,
1835
+ router_logits=all_router_logits,
1836
+ )
1837
+ if output_multistream:
1838
+ return model_output, multistream_hidden_states, position_embeddings, causal_mask, position_ids, cache_position
1839
+ return model_output
1840
+
1841
+ def _update_causal_mask(self, attention_mask, input_tensor, cache_position,
1842
+ past_key_values, output_attentions):
1843
+ if self.config._attn_implementation == "flash_attention_2":
1844
+ if attention_mask is not None and 0.0 in attention_mask:
1845
+ return attention_mask
1846
+ return None
1847
+
1848
+ past_seen_tokens = (past_key_values.get_seq_length()
1849
+ if past_key_values is not None else 0)
1850
+ using_static_cache = isinstance(past_key_values, StaticCache)
1851
+ if (self.config._attn_implementation == "sdpa"
1852
+ and not using_static_cache
1853
+ and not output_attentions):
1854
+ if AttentionMaskConverter._ignore_causal_mask_sdpa(
1855
+ attention_mask, inputs_embeds=input_tensor,
1856
+ past_key_values_length=past_seen_tokens,
1857
+ sliding_window=None,
1858
+ is_training=self.training,
1859
+ ):
1860
+ return None
1861
+
1862
+ dtype, device = input_tensor.dtype, input_tensor.device
1863
+ min_dtype = torch.finfo(dtype).min
1864
+ sequence_length = input_tensor.shape[1]
1865
+
1866
+ if using_static_cache:
1867
+ target_length = past_key_values.get_max_cache_shape()
1868
+ else:
1869
+ target_length = (
1870
+ attention_mask.shape[-1] if isinstance(attention_mask, torch.Tensor)
1871
+ else past_seen_tokens + sequence_length + 1)
1872
+
1873
+ causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position(
1874
+ attention_mask, sequence_length=sequence_length,
1875
+ target_length=target_length, dtype=dtype, device=device,
1876
+ cache_position=cache_position, batch_size=input_tensor.shape[0],
1877
+ config=self.config, past_key_values=past_key_values,
1878
+ )
1879
+
1880
+ if (self.config._attn_implementation == "sdpa"
1881
+ and attention_mask is not None
1882
+ and attention_mask.device.type == "cuda"
1883
+ and not output_attentions):
1884
+ causal_mask = AttentionMaskConverter._unmask_unattended(
1885
+ causal_mask, min_dtype)
1886
+
1887
+ return causal_mask
1888
+
1889
+ @staticmethod
1890
+ def _prepare_4d_causal_attention_mask_with_cache_position(
1891
+ attention_mask, sequence_length, target_length, dtype, device,
1892
+ cache_position, batch_size, config, past_key_values,
1893
+ ):
1894
+ if attention_mask is not None and attention_mask.dim() == 4:
1895
+ causal_mask = attention_mask
1896
+ else:
1897
+ min_dtype = torch.finfo(dtype).min
1898
+ causal_mask = torch.full(
1899
+ (sequence_length, target_length),
1900
+ fill_value=min_dtype, dtype=dtype, device=device)
1901
+ diagonal_attend_mask = (
1902
+ torch.arange(target_length, device=device)
1903
+ > cache_position.reshape(-1, 1))
1904
+ causal_mask *= diagonal_attend_mask
1905
+ causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
1906
+ if attention_mask is not None:
1907
+ causal_mask = causal_mask.clone()
1908
+ if attention_mask.shape[-1] > target_length:
1909
+ attention_mask = attention_mask[:, :target_length]
1910
+ mask_length = attention_mask.shape[-1]
1911
+ padding_mask = (
1912
+ causal_mask[:, :, :, :mask_length]
1913
+ + attention_mask[:, None, None, :]) == 0
1914
+ causal_mask[:, :, :, :mask_length] = causal_mask[
1915
+ :, :, :, :mask_length].masked_fill(padding_mask, min_dtype)
1916
+ return causal_mask
1917
+
1918
+
1919
+ # ---------------------------------------------------------------------------
1920
+ # MTP (Multi-Token Prediction)
1921
+ # ---------------------------------------------------------------------------
1922
+ #
1923
+ # The MTP layer's inner transformer layer uses plain (non block-sparse) GQA
1924
+ # attention -- confirmed against the actual mcore checkpoint tensors
1925
+ # (mtp.layers.0.transformer_layer.self_attention.* has no indexer.*
1926
+ # counterpart). So MaxwellMoeMTPDecoderLayer reuses this file's existing
1927
+ # plain attention classes (MAXWELLMOE_ATTENTION_CLASSES) rather than the
1928
+ # block-sparse MaxwellMoeDecoderLayer used by the main 45-layer decoder.
1929
+
1930
+
1931
+ class MaxwellMoeMTPDecoderLayer(nn.Module):
1932
+ def __init__(self, config: MaxwellMoeConfig, layer_idx: int):
1933
+ super().__init__()
1934
+ self.hidden_size = config.hidden_size
1935
+ self.enable_hyper_connections = config.enable_hyper_connections
1936
+
1937
+ self.self_attn = MAXWELLMOE_ATTENTION_CLASSES[
1938
+ config._attn_implementation](config, layer_idx)
1939
+
1940
+ is_dense = layer_idx in config.mlp_only_layers
1941
+ if is_dense or config.num_experts == 0:
1942
+ self.block_sparse_moe = MaxwellMoeMLP(
1943
+ config, intermediate_size=config.intermediate_size)
1944
+ else:
1945
+ self.block_sparse_moe = MaxwellMoeSparseMoeBlock(config)
1946
+
1947
+ self.input_layernorm = MaxwellMoeRMSNorm(
1948
+ config.hidden_size, eps=config.rms_norm_eps)
1949
+ self.post_attention_layernorm = MaxwellMoeRMSNorm(
1950
+ config.hidden_size, eps=config.rms_norm_eps)
1951
+
1952
+ if self.enable_hyper_connections:
1953
+ self.self_attn_hyper_connection = HyperConnectionModule(config)
1954
+ self.mlp_hyper_connection = HyperConnectionModule(config)
1955
+
1956
+ def forward(
1957
+ self, hidden_states, attention_mask=None, position_ids=None,
1958
+ past_key_value=None, output_attentions=False,
1959
+ output_router_logits=False, use_cache=False,
1960
+ cache_position=None, position_embeddings=None, **kwargs,
1961
+ ):
1962
+ if self.enable_hyper_connections:
1963
+ return self._forward_with_hc(
1964
+ hidden_states, attention_mask, position_ids, past_key_value,
1965
+ output_attentions, output_router_logits, use_cache,
1966
+ cache_position, position_embeddings, **kwargs)
1967
+ else:
1968
+ return self._forward_standard(
1969
+ hidden_states, attention_mask, position_ids, past_key_value,
1970
+ output_attentions, output_router_logits, use_cache,
1971
+ cache_position, position_embeddings, **kwargs)
1972
+
1973
+ def _forward_standard(
1974
+ self, hidden_states, attention_mask, position_ids, past_key_value,
1975
+ output_attentions, output_router_logits, use_cache,
1976
+ cache_position, position_embeddings, **kwargs,
1977
+ ):
1978
+ residual = hidden_states
1979
+ hidden_states = self.input_layernorm(hidden_states)
1980
+ hidden_states, self_attn_weights, present_key_value = self.self_attn(
1981
+ hidden_states=hidden_states, attention_mask=attention_mask,
1982
+ position_ids=position_ids, past_key_value=past_key_value,
1983
+ output_attentions=output_attentions, use_cache=use_cache,
1984
+ cache_position=cache_position, position_embeddings=position_embeddings,
1985
+ )
1986
+ hidden_states = residual + hidden_states
1987
+
1988
+ residual = hidden_states
1989
+ hidden_states = self.post_attention_layernorm(hidden_states)
1990
+ hidden_states = self.block_sparse_moe(hidden_states)
1991
+ if isinstance(hidden_states, tuple):
1992
+ hidden_states, router_logits = hidden_states
1993
+ else:
1994
+ router_logits = None
1995
+ hidden_states = residual + hidden_states
1996
+
1997
+ outputs = (hidden_states,)
1998
+ if output_attentions:
1999
+ outputs += (self_attn_weights,)
2000
+ if use_cache:
2001
+ outputs += (present_key_value,)
2002
+ if output_router_logits:
2003
+ outputs += (router_logits,)
2004
+ return outputs
2005
+
2006
+ def _forward_with_hc(
2007
+ self, hidden_states, attention_mask, position_ids, past_key_value,
2008
+ output_attentions, output_router_logits, use_cache,
2009
+ cache_position, position_embeddings, **kwargs,
2010
+ ):
2011
+ residual = hidden_states
2012
+ hc = self.self_attn_hyper_connection
2013
+ h_pre, h_post, h_res = hc._compute_mappings(hidden_states)
2014
+ aggregated = hc.aggregate(hidden_states, h_pre)
2015
+ aggregated = self.input_layernorm(aggregated)
2016
+ attn_output, self_attn_weights, present_key_value = self.self_attn(
2017
+ hidden_states=aggregated, attention_mask=attention_mask,
2018
+ position_ids=position_ids, past_key_value=past_key_value,
2019
+ output_attentions=output_attentions, use_cache=use_cache,
2020
+ cache_position=cache_position, position_embeddings=position_embeddings,
2021
+ )
2022
+ hidden_states = hc.expand_and_mix(attn_output, residual, h_post, h_res)
2023
+
2024
+ residual = hidden_states
2025
+ hc_mlp = self.mlp_hyper_connection
2026
+ h_pre, h_post, h_res = hc_mlp._compute_mappings(hidden_states)
2027
+ aggregated = hc_mlp.aggregate(hidden_states, h_pre)
2028
+ aggregated = self.post_attention_layernorm(aggregated)
2029
+ mlp_output = self.block_sparse_moe(aggregated)
2030
+ if isinstance(mlp_output, tuple):
2031
+ mlp_output, router_logits = mlp_output
2032
+ else:
2033
+ router_logits = None
2034
+ hidden_states = hc_mlp.expand_and_mix(mlp_output, residual, h_post, h_res)
2035
+
2036
+ outputs = (hidden_states,)
2037
+ if output_attentions:
2038
+ outputs += (self_attn_weights,)
2039
+ if use_cache:
2040
+ outputs += (present_key_value,)
2041
+ if output_router_logits:
2042
+ outputs += (router_logits,)
2043
+ return outputs
2044
+
2045
+
2046
+ class MaxwellMoeMTPLayer(nn.Module):
2047
+ """Multi-Token Prediction layer.
2048
+
2049
+ Takes the multi-stream hidden states from the decoder, combines with
2050
+ rolled (next-position) token embeddings via separate projections,
2051
+ passes through a full MoE transformer layer, and contracts back to
2052
+ a single-stream representation for the shared lm_head.
2053
+ """
2054
+
2055
+ def __init__(self, config: MaxwellMoeConfig, layer_idx: int):
2056
+ super().__init__()
2057
+ self.config = config
2058
+ n = config.num_residual_streams
2059
+ h = config.hidden_size
2060
+
2061
+ self.enorm = MaxwellMoeRMSNorm(h, eps=config.rms_norm_eps)
2062
+ self.hnorm = MaxwellMoeRMSNorm(h, eps=config.rms_norm_eps)
2063
+ self.e_proj = nn.Linear(h, h, bias=False)
2064
+ self.h_proj = nn.Linear(h, h, bias=False)
2065
+
2066
+ mtp_layer_idx = config.num_hidden_layers + layer_idx
2067
+ self.decoder_layer = MaxwellMoeMTPDecoderLayer(config, layer_idx=mtp_layer_idx)
2068
+
2069
+ self.hc_head = HCHead(config)
2070
+ self.final_layernorm = MaxwellMoeRMSNorm(h, eps=config.rms_norm_eps)
2071
+
2072
+ def forward(
2073
+ self,
2074
+ multistream_hidden_states,
2075
+ input_ids,
2076
+ position_ids,
2077
+ embed_tokens,
2078
+ attention_mask=None,
2079
+ position_embeddings=None,
2080
+ cache_position=None,
2081
+ ):
2082
+ """
2083
+ Args:
2084
+ multistream_hidden_states: [b, s, n*h] multi-stream from decoder
2085
+ input_ids: [b, s] current token IDs
2086
+ position_ids: [b, s] or [1, s]
2087
+ embed_tokens: embedding layer (shared with main model)
2088
+ attention_mask: causal mask
2089
+ position_embeddings: (cos, sin) from rotary embedding
2090
+ Returns:
2091
+ hidden_states: [b, s, h] contracted single-stream output
2092
+ mtp_multistream: [b, s, n*h] multi-stream output (for chaining)
2093
+ rolled_input_ids: [b, s] rolled input IDs
2094
+ rolled_position_ids: [b, s] rolled position IDs
2095
+ """
2096
+ b, s, nC = multistream_hidden_states.shape
2097
+ n = self.config.num_residual_streams
2098
+ h = self.config.hidden_size
2099
+
2100
+ rolled_input_ids = torch.roll(input_ids, shifts=-1, dims=-1)
2101
+ rolled_input_ids[:, -1] = self.config.pad_token_id
2102
+ rolled_position_ids = torch.roll(position_ids, shifts=-1, dims=-1)
2103
+ rolled_position_ids[:, -1] = 0
2104
+
2105
+ decoder_input = embed_tokens(rolled_input_ids)
2106
+ decoder_input = self.enorm(decoder_input)
2107
+
2108
+ hs_streams = multistream_hidden_states.view(b, s, n, h)
2109
+ hs_streams = self.hnorm(hs_streams)
2110
+
2111
+ e_out = self.e_proj(decoder_input)
2112
+ h_out = self.h_proj(hs_streams)
2113
+
2114
+ e_out = e_out.unsqueeze(2).expand(b, s, n, h)
2115
+ hidden_states = (e_out + h_out).reshape(b, s, n * h)
2116
+
2117
+ layer_outputs = self.decoder_layer(
2118
+ hidden_states,
2119
+ attention_mask=attention_mask,
2120
+ position_ids=position_ids,
2121
+ position_embeddings=position_embeddings,
2122
+ cache_position=cache_position,
2123
+ )
2124
+ mtp_multistream = layer_outputs[0]
2125
+
2126
+ hidden_states = self.hc_head(mtp_multistream)
2127
+ hidden_states = self.final_layernorm(hidden_states)
2128
+
2129
+ return hidden_states, mtp_multistream, rolled_input_ids, rolled_position_ids
2130
+
2131
+
2132
+ @dataclass
2133
+ class MTPCausalLMOutput(ModelOutput):
2134
+ loss: Optional[torch.FloatTensor] = None
2135
+ logits: torch.FloatTensor = None
2136
+ mtp_logits: Optional[torch.FloatTensor] = None
2137
+ past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
2138
+ hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
2139
+ attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
2140
+ router_logits: Optional[Tuple[torch.FloatTensor]] = None
2141
+
2142
+
2143
+ class MaxwellMoeForCausalLM(MaxwellMoePreTrainedModel, GenerationMixin):
2144
+ """MaxwellMoE with Multi-Token Prediction head for speculative decoding."""
2145
+
2146
+ _tied_weights_keys = ["lm_head.weight"]
2147
+
2148
+ def __init__(self, config):
2149
+ super().__init__(config)
2150
+ self.model = MaxwellMoeModel(config)
2151
+ self.vocab_size = config.vocab_size
2152
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
2153
+ self.router_aux_loss_coef = config.router_aux_loss_coef
2154
+ self.num_experts = config.num_experts
2155
+ self.num_experts_per_tok = config.num_experts_per_tok
2156
+
2157
+ self.mtp_num_layers = getattr(config, 'mtp_num_layers', 0)
2158
+ if self.mtp_num_layers > 0:
2159
+ self.mtp_layers = nn.ModuleList([
2160
+ MaxwellMoeMTPLayer(config, layer_idx=i)
2161
+ for i in range(self.mtp_num_layers)
2162
+ ])
2163
+ else:
2164
+ self.mtp_layers = nn.ModuleList()
2165
+
2166
+ self.post_init()
2167
+
2168
+ def get_input_embeddings(self):
2169
+ return self.model.embed_tokens
2170
+
2171
+ def set_input_embeddings(self, value):
2172
+ self.model.embed_tokens = value
2173
+
2174
+ def get_output_embeddings(self):
2175
+ return self.lm_head
2176
+
2177
+ def set_output_embeddings(self, new_embeddings):
2178
+ self.lm_head = new_embeddings
2179
+
2180
+ def set_decoder(self, decoder):
2181
+ self.model = decoder
2182
+
2183
+ def get_decoder(self):
2184
+ return self.model
2185
+
2186
+ def _fix_expert_bias_precision(self):
2187
+ """Restore decoder and MTP expert_bias to fp32 from safetensors (first forward).
2188
+
2189
+ from_pretrained(dtype=bf16) truncates expert_bias, and the per-expert
2190
+ spread (range ~0.005) is below bf16's quantization step (~0.06), so
2191
+ routing collapses without it.
2192
+ """
2193
+ self._expert_bias_fp32_done = True
2194
+ import json, os
2195
+ from safetensors import safe_open
2196
+ model_dir = self.config._name_or_path
2197
+ idx_path = os.path.join(model_dir, "model.safetensors.index.json")
2198
+ if not os.path.exists(idx_path):
2199
+ return
2200
+ with open(idx_path) as f:
2201
+ weight_map = json.load(f)["weight_map"]
2202
+ for layer_idx, layer in enumerate(self.model.layers):
2203
+ moe = getattr(layer, 'block_sparse_moe', None)
2204
+ if moe is None or not hasattr(moe, 'expert_bias'):
2205
+ continue
2206
+ key = f"model.layers.{layer_idx}.block_sparse_moe.expert_bias"
2207
+ shard = weight_map.get(key)
2208
+ if shard is None:
2209
+ continue
2210
+ with safe_open(os.path.join(model_dir, shard), framework="pt") as f:
2211
+ moe.expert_bias.data = f.get_tensor(key).float().to(moe.expert_bias.device)
2212
+ for mtp_idx, mtp_layer in enumerate(self.mtp_layers):
2213
+ moe = getattr(mtp_layer.decoder_layer, 'block_sparse_moe', None)
2214
+ if moe is None or not hasattr(moe, 'expert_bias'):
2215
+ continue
2216
+ key = f"mtp_layers.{mtp_idx}.decoder_layer.block_sparse_moe.expert_bias"
2217
+ shard = weight_map.get(key)
2218
+ if shard is None:
2219
+ continue
2220
+ with safe_open(os.path.join(model_dir, shard), framework="pt") as f:
2221
+ moe.expert_bias.data = f.get_tensor(key).float().to(moe.expert_bias.device)
2222
+
2223
+ def forward(
2224
+ self,
2225
+ input_ids=None, attention_mask=None, position_ids=None,
2226
+ past_key_values=None, inputs_embeds=None, labels=None,
2227
+ use_cache=None, output_attentions=None, output_hidden_states=None,
2228
+ output_router_logits=None, return_dict=None,
2229
+ cache_position=None, num_logits_to_keep=0,
2230
+ output_mtp_logits=False,
2231
+ ):
2232
+ if not getattr(self, '_expert_bias_fp32_done', False):
2233
+ self._fix_expert_bias_precision()
2234
+ output_attentions = (output_attentions if output_attentions is not None
2235
+ else self.config.output_attentions)
2236
+ output_router_logits = (output_router_logits if output_router_logits is not None
2237
+ else self.config.output_router_logits)
2238
+ output_hidden_states = (output_hidden_states if output_hidden_states is not None
2239
+ else self.config.output_hidden_states)
2240
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
2241
+
2242
+ need_mtp = output_mtp_logits and len(self.mtp_layers) > 0
2243
+
2244
+ if need_mtp:
2245
+ result = self.model(
2246
+ input_ids=input_ids, attention_mask=attention_mask,
2247
+ position_ids=position_ids, past_key_values=past_key_values,
2248
+ inputs_embeds=inputs_embeds, use_cache=use_cache,
2249
+ output_attentions=output_attentions,
2250
+ output_hidden_states=output_hidden_states,
2251
+ output_router_logits=output_router_logits,
2252
+ return_dict=return_dict, cache_position=cache_position,
2253
+ output_multistream=True,
2254
+ )
2255
+ if return_dict:
2256
+ outputs, multistream_hidden_states, position_embeddings, causal_mask, resolved_position_ids, resolved_cache_position = result
2257
+ else:
2258
+ base_result, multistream_hidden_states, position_embeddings, causal_mask, resolved_position_ids, resolved_cache_position = result
2259
+ outputs = base_result
2260
+ else:
2261
+ outputs = self.model(
2262
+ input_ids=input_ids, attention_mask=attention_mask,
2263
+ position_ids=position_ids, past_key_values=past_key_values,
2264
+ inputs_embeds=inputs_embeds, use_cache=use_cache,
2265
+ output_attentions=output_attentions,
2266
+ output_hidden_states=output_hidden_states,
2267
+ output_router_logits=output_router_logits,
2268
+ return_dict=return_dict, cache_position=cache_position,
2269
+ )
2270
+
2271
+ hidden_states = outputs[0] if not return_dict else outputs.last_hidden_state
2272
+ lm_input = hidden_states[:, -num_logits_to_keep:, :]
2273
+ logits = torch.nn.functional.linear(lm_input.float(), self.lm_head.weight.float(), self.lm_head.bias if self.lm_head.bias is not None else None)
2274
+
2275
+ mtp_logits = None
2276
+ if need_mtp and multistream_hidden_states is not None:
2277
+ cur_input_ids = input_ids
2278
+ cur_position_ids = resolved_position_ids
2279
+ cur_multistream = multistream_hidden_states
2280
+ for mtp_layer in self.mtp_layers:
2281
+ mtp_hidden, cur_multistream, cur_input_ids, cur_position_ids = mtp_layer(
2282
+ multistream_hidden_states=cur_multistream,
2283
+ input_ids=cur_input_ids,
2284
+ position_ids=cur_position_ids,
2285
+ embed_tokens=self.model.embed_tokens,
2286
+ attention_mask=causal_mask,
2287
+ position_embeddings=position_embeddings,
2288
+ cache_position=resolved_cache_position,
2289
+ )
2290
+ mtp_logits = torch.nn.functional.linear(mtp_hidden.float(), self.lm_head.weight.float(), self.lm_head.bias if self.lm_head.bias is not None else None)
2291
+
2292
+ loss = None
2293
+ if labels is not None:
2294
+ loss = self.loss_function(logits, labels, self.vocab_size)
2295
+
2296
+ if not return_dict:
2297
+ output = (logits,) + outputs[1:]
2298
+ if mtp_logits is not None:
2299
+ output = output + (mtp_logits,)
2300
+ return (loss,) + output if loss is not None else output
2301
+
2302
+ return MTPCausalLMOutput(
2303
+ loss=loss,
2304
+ logits=logits,
2305
+ mtp_logits=mtp_logits,
2306
+ past_key_values=outputs.past_key_values,
2307
+ hidden_states=outputs.hidden_states,
2308
+ attentions=outputs.attentions,
2309
+ router_logits=outputs.router_logits,
2310
+ )
special_tokens_map.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "eos_token": "</s>",
4
+ "pad_token": "<pad>",
5
+ "unk_token": "<unk>",
6
+ "additional_special_tokens": [
7
+ "<|maxwell_start|>",
8
+ "<|maxwell_end|>"
9
+ ]
10
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "200000": {
30
+ "content": "<|maxwell_start|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "200001": {
38
+ "content": "<|maxwell_end|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "200002": {
46
+ "content": "<tool_call>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": false
52
+ },
53
+ "200003": {
54
+ "content": "</tool_call>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": false
60
+ },
61
+ "200004": {
62
+ "content": "<tool_response>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": false
68
+ },
69
+ "200005": {
70
+ "content": "</tool_response>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": false
76
+ },
77
+ "200006": {
78
+ "content": "<think>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": false
84
+ },
85
+ "200007": {
86
+ "content": "</think>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": false
92
+ },
93
+ "3": {
94
+ "content": "<pad>",
95
+ "lstrip": false,
96
+ "normalized": false,
97
+ "rstrip": false,
98
+ "single_word": false,
99
+ "special": true
100
+ }
101
+ },
102
+ "additional_special_tokens": [
103
+ "<|maxwell_start|>",
104
+ "<|maxwell_end|>"
105
+ ],
106
+ "bos_token": "<s>",
107
+ "clean_up_tokenization_spaces": false,
108
+ "eos_token": "</s>",
109
+ "model_max_length": 524288,
110
+ "pad_token": "<pad>",
111
+ "padding_side": "right",
112
+ "spaces_between_special_tokens": false,
113
+ "split_special_tokens": false,
114
+ "tokenizer_class": "PreTrainedTokenizerFast",
115
+ "unk_token": "<unk>"
116
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff