Upload verified plain block FP8 Agnes Preview weights

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

chat_template.jinja

@@ -0,0 +1,170 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- set reasoning_instructions = '' %}
+{%- if enable_thinking is undefined or enable_thinking is true %}
+    {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}
+    {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}
+        {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}
+    {%- endif %}
+    {%- if resolved_reasoning_effort == 'xhigh' %}
+        {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}
+    {%- elif resolved_reasoning_effort == 'low' %}
+        {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}
+    {%- endif %}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {%- if reasoning_instructions %}
+        {{- reasoning_instructions + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '')  + content + '<|im_end|>\n' }}
+        {%- elif reasoning_instructions %}
+            {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
+        {%- endif %}
+    {%- elif reasoning_instructions %}
+        {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined and tool_call.arguments != '' %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,458 @@
+{
+  "model_type": "agnes",
+  "architectures": [
+    "AgnesForConditionalGeneration"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_agnes.AgnesConfig",
+    "AutoModel": "modeling_agnes.AgnesModel",
+    "AutoModelForCausalLM": "modeling_agnes.AgnesForConditionalGeneration",
+    "AutoModelForImageTextToText": "modeling_agnes.AgnesForConditionalGeneration"
+  },
+  "text_config": {
+    "model_type": "agnes_text",
+    "hidden_size": 5120,
+    "num_hidden_layers": 72,
+    "intermediate_size": 19456,
+    "parallel_ffn_intermediate_size": 0,
+    "vocab_size": 248320,
+    "layer_types": [
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_delta_attention",
+      "agnes_global_attention"
+    ],
+    "global_attention_interval": 4,
+    "num_attention_heads": 24,
+    "num_key_value_heads": 4,
+    "head_dim": 256,
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_output_gate": true,
+    "partial_rotary_factor": 0.25,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 48,
+    "linear_key_head_dim": 128,
+    "linear_value_head_dim": 128,
+    "linear_conv_kernel_dim": 4,
+    "output_gate_type": "swish",
+    "mamba_ssm_dtype": "float32",
+    "hidden_act": "silu",
+    "rms_norm_eps": 1e-06,
+    "initializer_range": 0.02,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        11,
+        11,
+        10
+      ],
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "max_position_embeddings": 262144,
+    "mtp_num_hidden_layers": 1,
+    "mtp_use_dedicated_embeddings": false,
+    "bos_token_id": 248044,
+    "eos_token_id": 248044,
+    "pad_token_id": null,
+    "tie_word_embeddings": false,
+    "use_cache": true,
+    "dtype": "bfloat16",
+    "agnes_original_intermediate_size": 17408,
+    "agnes_original_parallel_ffn_intermediate_size": 2048
+  },
+  "vision_config": {
+    "model_type": "agnes_vision",
+    "depth": 27,
+    "hidden_size": 1152,
+    "intermediate_size": 4304,
+    "num_heads": 16,
+    "out_hidden_size": 5120,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2,
+    "in_channels": 3,
+    "num_position_embeddings": 2304,
+    "deepstack_visual_indexes": [],
+    "hidden_act": "gelu_pytorch_tanh",
+    "initializer_range": 0.02
+  },
+  "image_token_id": 248056,
+  "video_token_id": 248057,
+  "vision_start_token_id": 248053,
+  "vision_end_token_id": 248054,
+  "tie_word_embeddings": false,
+  "quantization_config": {
+    "quant_method": "fp8",
+    "fmt": "e4m3",
+    "activation_scheme": "dynamic",
+    "weight_block_size": [
+      128,
+      128
+    ],
+    "modules_to_not_convert": [
+      "lm_head",
+      "model.language_model.layers.0.delta_attn",
+      "model.language_model.layers.0.linear_attn",
+      "model.language_model.layers.1.delta_attn",
+      "model.language_model.layers.1.linear_attn",
+      "model.language_model.layers.10.delta_attn",
+      "model.language_model.layers.10.linear_attn",
+      "model.language_model.layers.11.delta_attn",
+      "model.language_model.layers.11.linear_attn",
+      "model.language_model.layers.12.delta_attn",
+      "model.language_model.layers.12.linear_attn",
+      "model.language_model.layers.13.delta_attn",
+      "model.language_model.layers.13.linear_attn",
+      "model.language_model.layers.14.delta_attn",
+      "model.language_model.layers.14.linear_attn",
+      "model.language_model.layers.15.delta_attn",
+      "model.language_model.layers.15.linear_attn",
+      "model.language_model.layers.16.delta_attn",
+      "model.language_model.layers.16.linear_attn",
+      "model.language_model.layers.17.delta_attn",
+      "model.language_model.layers.17.linear_attn",
+      "model.language_model.layers.18.delta_attn",
+      "model.language_model.layers.18.linear_attn",
+      "model.language_model.layers.19.delta_attn",
+      "model.language_model.layers.19.linear_attn",
+      "model.language_model.layers.2.delta_attn",
+      "model.language_model.layers.2.linear_attn",
+      "model.language_model.layers.20.delta_attn",
+      "model.language_model.layers.20.linear_attn",
+      "model.language_model.layers.21.delta_attn",
+      "model.language_model.layers.21.linear_attn",
+      "model.language_model.layers.22.delta_attn",
+      "model.language_model.layers.22.linear_attn",
+      "model.language_model.layers.23.delta_attn",
+      "model.language_model.layers.23.linear_attn",
+      "model.language_model.layers.24.delta_attn",
+      "model.language_model.layers.24.linear_attn",
+      "model.language_model.layers.25.delta_attn",
+      "model.language_model.layers.25.linear_attn",
+      "model.language_model.layers.26.delta_attn",
+      "model.language_model.layers.26.linear_attn",
+      "model.language_model.layers.27.delta_attn",
+      "model.language_model.layers.27.linear_attn",
+      "model.language_model.layers.28.delta_attn",
+      "model.language_model.layers.28.linear_attn",
+      "model.language_model.layers.29.delta_attn",
+      "model.language_model.layers.29.linear_attn",
+      "model.language_model.layers.3.delta_attn",
+      "model.language_model.layers.3.linear_attn",
+      "model.language_model.layers.30.delta_attn",
+      "model.language_model.layers.30.linear_attn",
+      "model.language_model.layers.31.delta_attn",
+      "model.language_model.layers.31.linear_attn",
+      "model.language_model.layers.32.delta_attn",
+      "model.language_model.layers.32.linear_attn",
+      "model.language_model.layers.33.delta_attn",
+      "model.language_model.layers.33.linear_attn",
+      "model.language_model.layers.34.delta_attn",
+      "model.language_model.layers.34.linear_attn",
+      "model.language_model.layers.35.delta_attn",
+      "model.language_model.layers.35.linear_attn",
+      "model.language_model.layers.36.delta_attn",
+      "model.language_model.layers.36.linear_attn",
+      "model.language_model.layers.37.delta_attn",
+      "model.language_model.layers.37.linear_attn",
+      "model.language_model.layers.38.delta_attn",
+      "model.language_model.layers.38.linear_attn",
+      "model.language_model.layers.39.delta_attn",
+      "model.language_model.layers.39.linear_attn",
+      "model.language_model.layers.4.delta_attn",
+      "model.language_model.layers.4.linear_attn",
+      "model.language_model.layers.40.delta_attn",
+      "model.language_model.layers.40.linear_attn",
+      "model.language_model.layers.41.delta_attn",
+      "model.language_model.layers.41.linear_attn",
+      "model.language_model.layers.42.delta_attn",
+      "model.language_model.layers.42.linear_attn",
+      "model.language_model.layers.43.delta_attn",
+      "model.language_model.layers.43.linear_attn",
+      "model.language_model.layers.44.delta_attn",
+      "model.language_model.layers.44.linear_attn",
+      "model.language_model.layers.45.delta_attn",
+      "model.language_model.layers.45.linear_attn",
+      "model.language_model.layers.46.delta_attn",
+      "model.language_model.layers.46.linear_attn",
+      "model.language_model.layers.47.delta_attn",
+      "model.language_model.layers.47.linear_attn",
+      "model.language_model.layers.48.delta_attn",
+      "model.language_model.layers.48.linear_attn",
+      "model.language_model.layers.49.delta_attn",
+      "model.language_model.layers.49.linear_attn",
+      "model.language_model.layers.5.delta_attn",
+      "model.language_model.layers.5.linear_attn",
+      "model.language_model.layers.50.delta_attn",
+      "model.language_model.layers.50.linear_attn",
+      "model.language_model.layers.51.delta_attn",
+      "model.language_model.layers.51.linear_attn",
+      "model.language_model.layers.52.delta_attn",
+      "model.language_model.layers.52.linear_attn",
+      "model.language_model.layers.53.delta_attn",
+      "model.language_model.layers.53.linear_attn",
+      "model.language_model.layers.54.delta_attn",
+      "model.language_model.layers.54.linear_attn",
+      "model.language_model.layers.55.delta_attn",
+      "model.language_model.layers.55.linear_attn",
+      "model.language_model.layers.56.delta_attn",
+      "model.language_model.layers.56.linear_attn",
+      "model.language_model.layers.57.delta_attn",
+      "model.language_model.layers.57.linear_attn",
+      "model.language_model.layers.58.delta_attn",
+      "model.language_model.layers.58.linear_attn",
+      "model.language_model.layers.59.delta_attn",
+      "model.language_model.layers.59.linear_attn",
+      "model.language_model.layers.6.delta_attn",
+      "model.language_model.layers.6.linear_attn",
+      "model.language_model.layers.60.delta_attn",
+      "model.language_model.layers.60.linear_attn",
+      "model.language_model.layers.61.delta_attn",
+      "model.language_model.layers.61.linear_attn",
+      "model.language_model.layers.62.delta_attn",
+      "model.language_model.layers.62.linear_attn",
+      "model.language_model.layers.63.delta_attn",
+      "model.language_model.layers.63.linear_attn",
+      "model.language_model.layers.64.delta_attn",
+      "model.language_model.layers.64.linear_attn",
+      "model.language_model.layers.65.delta_attn",
+      "model.language_model.layers.65.linear_attn",
+      "model.language_model.layers.66.delta_attn",
+      "model.language_model.layers.66.linear_attn",
+      "model.language_model.layers.67.delta_attn",
+      "model.language_model.layers.67.linear_attn",
+      "model.language_model.layers.68.delta_attn",
+      "model.language_model.layers.68.linear_attn",
+      "model.language_model.layers.69.delta_attn",
+      "model.language_model.layers.69.linear_attn",
+      "model.language_model.layers.7.delta_attn",
+      "model.language_model.layers.7.linear_attn",
+      "model.language_model.layers.70.delta_attn",
+      "model.language_model.layers.70.linear_attn",
+      "model.language_model.layers.71.delta_attn",
+      "model.language_model.layers.71.linear_attn",
+      "model.language_model.layers.8.delta_attn",
+      "model.language_model.layers.8.linear_attn",
+      "model.language_model.layers.9.delta_attn",
+      "model.language_model.layers.9.linear_attn",
+      "model.layers.0.delta_attn",
+      "model.layers.0.linear_attn",
+      "model.layers.1.delta_attn",
+      "model.layers.1.linear_attn",
+      "model.layers.10.delta_attn",
+      "model.layers.10.linear_attn",
+      "model.layers.11.delta_attn",
+      "model.layers.11.linear_attn",
+      "model.layers.12.delta_attn",
+      "model.layers.12.linear_attn",
+      "model.layers.13.delta_attn",
+      "model.layers.13.linear_attn",
+      "model.layers.14.delta_attn",
+      "model.layers.14.linear_attn",
+      "model.layers.15.delta_attn",
+      "model.layers.15.linear_attn",
+      "model.layers.16.delta_attn",
+      "model.layers.16.linear_attn",
+      "model.layers.17.delta_attn",
+      "model.layers.17.linear_attn",
+      "model.layers.18.delta_attn",
+      "model.layers.18.linear_attn",
+      "model.layers.19.delta_attn",
+      "model.layers.19.linear_attn",
+      "model.layers.2.delta_attn",
+      "model.layers.2.linear_attn",
+      "model.layers.20.delta_attn",
+      "model.layers.20.linear_attn",
+      "model.layers.21.delta_attn",
+      "model.layers.21.linear_attn",
+      "model.layers.22.delta_attn",
+      "model.layers.22.linear_attn",
+      "model.layers.23.delta_attn",
+      "model.layers.23.linear_attn",
+      "model.layers.24.delta_attn",
+      "model.layers.24.linear_attn",
+      "model.layers.25.delta_attn",
+      "model.layers.25.linear_attn",
+      "model.layers.26.delta_attn",
+      "model.layers.26.linear_attn",
+      "model.layers.27.delta_attn",
+      "model.layers.27.linear_attn",
+      "model.layers.28.delta_attn",
+      "model.layers.28.linear_attn",
+      "model.layers.29.delta_attn",
+      "model.layers.29.linear_attn",
+      "model.layers.3.delta_attn",
+      "model.layers.3.linear_attn",
+      "model.layers.30.delta_attn",
+      "model.layers.30.linear_attn",
+      "model.layers.31.delta_attn",
+      "model.layers.31.linear_attn",
+      "model.layers.32.delta_attn",
+      "model.layers.32.linear_attn",
+      "model.layers.33.delta_attn",
+      "model.layers.33.linear_attn",
+      "model.layers.34.delta_attn",
+      "model.layers.34.linear_attn",
+      "model.layers.35.delta_attn",
+      "model.layers.35.linear_attn",
+      "model.layers.36.delta_attn",
+      "model.layers.36.linear_attn",
+      "model.layers.37.delta_attn",
+      "model.layers.37.linear_attn",
+      "model.layers.38.delta_attn",
+      "model.layers.38.linear_attn",
+      "model.layers.39.delta_attn",
+      "model.layers.39.linear_attn",
+      "model.layers.4.delta_attn",
+      "model.layers.4.linear_attn",
+      "model.layers.40.delta_attn",
+      "model.layers.40.linear_attn",
+      "model.layers.41.delta_attn",
+      "model.layers.41.linear_attn",
+      "model.layers.42.delta_attn",
+      "model.layers.42.linear_attn",
+      "model.layers.43.delta_attn",
+      "model.layers.43.linear_attn",
+      "model.layers.44.delta_attn",
+      "model.layers.44.linear_attn",
+      "model.layers.45.delta_attn",
+      "model.layers.45.linear_attn",
+      "model.layers.46.delta_attn",
+      "model.layers.46.linear_attn",
+      "model.layers.47.delta_attn",
+      "model.layers.47.linear_attn",
+      "model.layers.48.delta_attn",
+      "model.layers.48.linear_attn",
+      "model.layers.49.delta_attn",
+      "model.layers.49.linear_attn",
+      "model.layers.5.delta_attn",
+      "model.layers.5.linear_attn",
+      "model.layers.50.delta_attn",
+      "model.layers.50.linear_attn",
+      "model.layers.51.delta_attn",
+      "model.layers.51.linear_attn",
+      "model.layers.52.delta_attn",
+      "model.layers.52.linear_attn",
+      "model.layers.53.delta_attn",
+      "model.layers.53.linear_attn",
+      "model.layers.54.delta_attn",
+      "model.layers.54.linear_attn",
+      "model.layers.55.delta_attn",
+      "model.layers.55.linear_attn",
+      "model.layers.56.delta_attn",
+      "model.layers.56.linear_attn",
+      "model.layers.57.delta_attn",
+      "model.layers.57.linear_attn",
+      "model.layers.58.delta_attn",
+      "model.layers.58.linear_attn",
+      "model.layers.59.delta_attn",
+      "model.layers.59.linear_attn",
+      "model.layers.6.delta_attn",
+      "model.layers.6.linear_attn",
+      "model.layers.60.delta_attn",
+      "model.layers.60.linear_attn",
+      "model.layers.61.delta_attn",
+      "model.layers.61.linear_attn",
+      "model.layers.62.delta_attn",
+      "model.layers.62.linear_attn",
+      "model.layers.63.delta_attn",
+      "model.layers.63.linear_attn",
+      "model.layers.64.delta_attn",
+      "model.layers.64.linear_attn",
+      "model.layers.65.delta_attn",
+      "model.layers.65.linear_attn",
+      "model.layers.66.delta_attn",
+      "model.layers.66.linear_attn",
+      "model.layers.67.delta_attn",
+      "model.layers.67.linear_attn",
+      "model.layers.68.delta_attn",
+      "model.layers.68.linear_attn",
+      "model.layers.69.delta_attn",
+      "model.layers.69.linear_attn",
+      "model.layers.7.delta_attn",
+      "model.layers.7.linear_attn",
+      "model.layers.70.delta_attn",
+      "model.layers.70.linear_attn",
+      "model.layers.71.delta_attn",
+      "model.layers.71.linear_attn",
+      "model.layers.8.delta_attn",
+      "model.layers.8.linear_attn",
+      "model.layers.9.delta_attn",
+      "model.layers.9.linear_attn",
+      "model.visual",
+      "mtp",
+      "visual"
+    ]
+  }
+}

configuration_agnes.py

@@ -0,0 +1,203 @@
+# Copyright 2026 Agnes AI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Agnes 3.0 Flash model configuration."""
+
+from huggingface_hub.dataclasses import strict
+
+from transformers.configuration_utils import PreTrainedConfig
+from transformers.modeling_rope_utils import RopeParameters
+from transformers.utils import auto_docstring
+
+
+LAYER_GLOBAL = "agnes_global_attention"
+LAYER_DELTA = "agnes_delta_attention"
+LAYER_TYPES = (LAYER_GLOBAL, LAYER_DELTA)
+
+
+@auto_docstring
+@strict
+class AgnesTextConfig(PreTrainedConfig):
+    r"""
+    linear_num_key_heads (`int`, *optional*, defaults to 16):
+        Number of key heads in the delta-rule (recurrent) attention layers.
+    linear_num_value_heads (`int`, *optional*, defaults to 32):
+        Number of value heads in the delta-rule attention layers.
+    linear_key_head_dim (`int`, *optional*, defaults to 128):
+        Per-head key width in the delta-rule attention layers.
+    linear_value_head_dim (`int`, *optional*, defaults to 128):
+        Per-head value width in the delta-rule attention layers.
+    linear_conv_kernel_dim (`int`, *optional*, defaults to 4):
+        Kernel width of the causal depthwise convolution feeding the delta-rule layers.
+    parallel_ffn_intermediate_size (`int`, *optional*, defaults to 0):
+        Width of the dense SwiGLU branch that runs beside the main MLP in every
+        decoder layer and is summed into the same residual. 0 disables the branch.
+    """
+
+    model_type = "agnes_text"
+    base_config_key = "text_config"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"}
+
+    # core dimensions
+    hidden_size: int = 4096
+    num_hidden_layers: int = 32
+    intermediate_size: int = 12288
+    parallel_ffn_intermediate_size: int = 0
+    vocab_size: int = 248320
+
+    # layer plan
+    layer_types: list[str] | None = None
+
+    # global attention
+    num_attention_heads: int = 16
+    num_key_value_heads: int = 4
+    head_dim: int = 256
+    attention_bias: bool = False
+    attention_dropout: float | int = 0.0
+
+    # delta-rule attention
+    linear_num_key_heads: int = 16
+    linear_num_value_heads: int = 32
+    linear_key_head_dim: int = 128
+    linear_value_head_dim: int = 128
+    linear_conv_kernel_dim: int = 4
+
+    # ffn / norm / init
+    hidden_act: str = "silu"
+    rms_norm_eps: float = 1e-6
+    initializer_range: float = 0.02
+
+    # positions
+    rope_parameters: RopeParameters | dict | None = None
+    max_position_embeddings: int = 32768
+
+    # tokens / runtime
+    bos_token_id: int | None = None
+    eos_token_id: int | list[int] | None = None
+    pad_token_id: int | None = None
+    tie_word_embeddings: bool = False
+    use_cache: bool = True
+
+    base_model_pp_plan = {
+        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+    base_model_tp_plan = {
+        "layers.*.global_attn.q_proj": "colwise",
+        "layers.*.global_attn.k_proj": "colwise",
+        "layers.*.global_attn.v_proj": "colwise",
+        "layers.*.global_attn.o_proj": "rowwise",
+        "layers.*.global_attn.q_norm": "replicated_with_grad_allreduce",
+        "layers.*.global_attn.k_norm": "replicated_with_grad_allreduce",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj": "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+        "layers.*.mlp.parallel_ffn.gate_proj": "colwise",
+        "layers.*.mlp.parallel_ffn.up_proj": "colwise",
+        "layers.*.mlp.parallel_ffn.down_proj": "rowwise",
+    }
+
+    def __post_init__(self, **kwargs):
+        # rotary embedding covers a quarter of each head unless told otherwise
+        kwargs.setdefault("partial_rotary_factor", 0.25)
+        if self.layer_types is None:
+            period = kwargs.pop("global_attention_interval", kwargs.pop("full_attention_interval", 4))
+            plan = []
+            for i in range(self.num_hidden_layers):
+                is_global = (i + 1) % period == 0
+                plan.append(LAYER_GLOBAL if is_global else LAYER_DELTA)
+            self.layer_types = plan
+        super().__post_init__(**kwargs)
+
+    def validate_layer_type(self):
+        """The layer plan uses this model's own type names, so the generic
+        allow-list in the base class does not apply."""
+        plan = self.layer_types
+        if plan is None:
+            return
+        unknown = sorted({t for t in plan if t not in LAYER_TYPES})
+        if unknown:
+            raise ValueError(f"`layer_types` entries must be in {LAYER_TYPES}, got {unknown}")
+        if self.num_hidden_layers is not None and self.num_hidden_layers != len(plan):
+            raise ValueError(
+                f"`num_hidden_layers` ({self.num_hidden_layers}) must equal the number of `layer_types` ({len(plan)})"
+            )
+
+
+@auto_docstring
+@strict
+class AgnesVisionConfig(PreTrainedConfig):
+    r"""
+    out_hidden_size (`int`, *optional*, defaults to 3584):
+        Width of the merged visual tokens handed to the language model.
+    num_position_embeddings (`int`, *optional*, defaults to 2304):
+        Size of the learned 2-D position table (a square grid, side = sqrt of this).
+    """
+
+    model_type = "agnes_vision"
+    base_config_key = "vision_config"
+
+    depth: int = 27
+    hidden_size: int = 1152
+    intermediate_size: int = 4304
+    num_heads: int = 16
+    out_hidden_size: int = 3584
+    patch_size: int | list[int] | tuple[int, int] = 16
+    spatial_merge_size: int = 2
+    temporal_patch_size: int | list[int] | tuple[int, int] = 2
+    in_channels: int = 3
+    num_position_embeddings: int = 2304
+    hidden_act: str = "gelu_pytorch_tanh"
+    initializer_range: float = 0.02
+
+
+@auto_docstring
+@strict
+class AgnesConfig(PreTrainedConfig):
+    r"""
+    Example:
+
+    ```python
+    >>> from transformers import AutoConfig
+
+    >>> configuration = AutoConfig.from_pretrained("<model dir>", trust_remote_code=True)
+    >>> configuration.text_config.num_hidden_layers
+    72
+    ```"""
+
+    model_type = "agnes"
+    sub_configs = {"text_config": AgnesTextConfig, "vision_config": AgnesVisionConfig}
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    text_config: dict | PreTrainedConfig | None = None
+    vision_config: dict | PreTrainedConfig | None = None
+
+    image_token_id: int = 248056
+    video_token_id: int = 248057
+    vision_start_token_id: int = 248053
+    vision_end_token_id: int = 248054
+    tie_word_embeddings: bool = False
+
+    def __post_init__(self, **kwargs):
+        for key, cls in self.sub_configs.items():
+            value = getattr(self, key)
+            if isinstance(value, dict):
+                setattr(self, key, cls(**value))
+            elif value is None:
+                setattr(self, key, cls())
+        super().__post_init__(**kwargs)
+
+
+__all__ = ["AgnesConfig", "AgnesTextConfig", "AgnesVisionConfig", "LAYER_GLOBAL", "LAYER_DELTA", "LAYER_TYPES"]

generation_config.json

@@ -0,0 +1,12 @@
+{
+    "bos_token_id": 248044,
+    "do_sample": true,
+    "eos_token_id": [
+        248046,
+        248044
+    ],
+    "pad_token_id": 248044,
+    "temperature": 1.0,
+    "top_k": 20,
+    "top_p": 0.95
+}

image_processing_agnes.py

@@ -0,0 +1,190 @@
+# Copyright 2026 Agnes AI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Image processor for Agnes 3.0 Flash: dynamic-resolution patching."""
+
+import math
+from collections.abc import Iterable
+
+import torch
+from torchvision.transforms.v2 import functional as tvF
+
+from transformers.image_processing_backends import TorchvisionBackend
+from transformers.image_processing_utils import BatchFeature
+from transformers.image_transforms import group_images_by_shape, reorder_images
+from transformers.image_utils import ImageInput, PILImageResampling, SizeDict
+from transformers.processing_utils import ImagesKwargs, Unpack
+from transformers.utils import TensorType, auto_docstring
+
+
+class AgnesImageProcessorKwargs(ImagesKwargs, total=False):
+    r"""
+    min_pixels (`int`, *optional*, defaults to `256 * 256`):
+        Lower bound on the pixel count after resizing.
+    max_pixels (`int`, *optional*, defaults to `4096 * 4096`):
+        Upper bound on the pixel count after resizing.
+    patch_size (`int`, *optional*, defaults to 16):
+        Spatial patch size of the vision tower.
+    temporal_patch_size (`int`, *optional*, defaults to 2):
+        Temporal patch size of the vision tower (images are duplicated to fill it).
+    merge_size (`int`, *optional*, defaults to 2):
+        Side of the patch square merged into one language-model token.
+    """
+
+    min_pixels: int
+    max_pixels: int
+    patch_size: int
+    temporal_patch_size: int
+    merge_size: int
+
+
+def fit_to_grid(height: int, width: int, factor: int = 32, min_pixels: int = 256 * 256, max_pixels: int = 4096 * 4096):
+    """Pick a (height, width) that is a multiple of `factor` on both sides,
+    keeps the pixel count inside [min_pixels, max_pixels] and stays as close
+    as possible to the original aspect ratio."""
+    if max(height, width) / min(height, width) > 200:
+        raise ValueError(f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}")
+    h = round(height / factor) * factor
+    w = round(width / factor) * factor
+    if h * w > max_pixels:
+        scale = math.sqrt((height * width) / max_pixels)
+        h = max(factor, math.floor(height / scale / factor) * factor)
+        w = max(factor, math.floor(width / scale / factor) * factor)
+    elif h * w < min_pixels:
+        scale = math.sqrt(min_pixels / (height * width))
+        h = math.ceil(height * scale / factor) * factor
+        w = math.ceil(width * scale / factor) * factor
+    return h, w
+
+
+@auto_docstring
+class AgnesImageProcessor(TorchvisionBackend):
+    do_resize = True
+    resample = PILImageResampling.BICUBIC
+    size = {"shortest_edge": 256 * 256, "longest_edge": 4096 * 4096}
+    default_to_square = False
+    do_rescale = True
+    do_normalize = True
+    image_mean = [0.5, 0.5, 0.5]
+    image_std = [0.5, 0.5, 0.5]
+    do_convert_rgb = True
+    patch_size = 16
+    temporal_patch_size = 2
+    merge_size = 2
+    valid_kwargs = AgnesImageProcessorKwargs
+    model_input_names = ["pixel_values", "image_grid_thw"]
+
+    def __init__(self, **kwargs: Unpack[AgnesImageProcessorKwargs]):
+        size = kwargs.pop("size", None)
+        min_pixels = kwargs.pop("min_pixels", None)
+        max_pixels = kwargs.pop("max_pixels", None)
+        size = self.size if size is None else size
+        # min_pixels / max_pixels are the older spelling of the two size keys
+        if min_pixels is not None:
+            size["shortest_edge"] = min_pixels
+            size.pop("min_pixels", None)
+        if max_pixels is not None:
+            size["longest_edge"] = max_pixels
+            size.pop("max_pixels", None)
+        if "shortest_edge" not in size or "longest_edge" not in size:
+            raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.")
+        super().__init__(size=size, **kwargs)
+
+    def _standardize_kwargs(
+        self,
+        size: int | Iterable[int] | dict[str, int] | SizeDict | None = None,
+        min_pixels: int | None = None,
+        max_pixels: int | None = None,
+        **kwargs,
+    ) -> dict:
+        if min_pixels is not None and max_pixels is not None:
+            size = SizeDict(shortest_edge=min_pixels, longest_edge=max_pixels)
+        kwargs = super()._standardize_kwargs(size=size, **kwargs)
+        size = kwargs.get("size", self.size)
+        if not size.shortest_edge or not size.longest_edge:
+            raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.")
+        return kwargs
+
+    @auto_docstring
+    def preprocess(self, images: ImageInput, **kwargs: Unpack[AgnesImageProcessorKwargs]) -> BatchFeature:
+        return super().preprocess(images, **kwargs)
+
+    def _preprocess(
+        self,
+        images: list["torch.Tensor"],
+        do_resize: bool,
+        size: SizeDict,
+        resample: "PILImageResampling | tvF.InterpolationMode | int | None",
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: float | list[float] | None,
+        image_std: float | list[float] | None,
+        patch_size: int,
+        temporal_patch_size: int,
+        merge_size: int,
+        disable_grouping: bool | None,
+        return_tensors: str | TensorType | None,
+        **kwargs,
+    ) -> BatchFeature:
+        # 1. resize, batched per input shape
+        by_shape, order = group_images_by_shape(images, disable_grouping=disable_grouping)
+        resized = {}
+        for shape, batch in by_shape.items():
+            height, width = batch.shape[-2:]
+            if do_resize:
+                new_h, new_w = fit_to_grid(
+                    height, width, factor=patch_size * merge_size,
+                    min_pixels=size.shortest_edge, max_pixels=size.longest_edge,
+                )
+                batch = self.resize(image=batch, size=SizeDict(height=new_h, width=new_w), resample=resample)
+            resized[shape] = batch
+        images = reorder_images(resized, order)
+
+        # 2. normalise and cut into merge-ordered patches, batched per resized shape
+        by_shape, order = group_images_by_shape(images, disable_grouping=disable_grouping)
+        flat = {}
+        grids = {}
+        for shape, batch in by_shape.items():
+            new_h, new_w = batch.shape[-2:]
+            px = self.rescale_and_normalize(batch, do_rescale, rescale_factor, do_normalize, image_mean, image_std)
+            n, c = px.shape[:2]
+            gh, gw = new_h // patch_size, new_w // patch_size
+            px = px.reshape(n, c, gh // merge_size, merge_size, patch_size, gw // merge_size, merge_size, patch_size)
+            # -> [n, gh/merge, gw/merge, merge, merge, c, patch, patch]: patches of one
+            # merge square end up adjacent in the flattened sequence
+            px = px.permute(0, 2, 5, 3, 6, 1, 4, 7)
+            px = (
+                px.unsqueeze(6)
+                .expand(-1, -1, -1, -1, -1, -1, temporal_patch_size, -1, -1)
+                .reshape(n, gh * gw, c * temporal_patch_size * patch_size * patch_size)
+            )
+            flat[shape] = px
+            grids[shape] = [[1, gh, gw]] * n
+
+        pixel_values = torch.cat(reorder_images(flat, order), dim=0)
+        image_grid_thw = torch.tensor(reorder_images(grids, order), dtype=torch.long)
+        return BatchFeature(data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}, tensor_type=return_tensors)
+
+    def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None):
+        """Number of vision patches an image of this size produces (used by
+        serving engines to lay out placeholders without running the processor)."""
+        min_pixels = images_kwargs["min_pixels"] if "min_pixels" in images_kwargs else self.size["shortest_edge"]
+        max_pixels = images_kwargs["max_pixels"] if "max_pixels" in images_kwargs else self.size["longest_edge"]
+        patch_size = images_kwargs.get("patch_size", self.patch_size)
+        merge_size = images_kwargs.get("merge_size", self.merge_size)
+        new_h, new_w = fit_to_grid(height, width, patch_size * merge_size, min_pixels=min_pixels, max_pixels=max_pixels)
+        return (new_h // patch_size) * (new_w // patch_size)
+
+
+__all__ = ["AgnesImageProcessor"]

inference_manifest.json

@@ -0,0 +1,184 @@
+{
+  "files": [
+    {
+      "name": "LICENSE",
+      "size": 11357,
+      "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4"
+    },
+    {
+      "name": "chat_template.jinja",
+      "size": 8952,
+      "sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041"
+    },
+    {
+      "name": "config.json",
+      "size": 17515,
+      "sha256": "3d657d45e1d0b619e97be53ef426a372364854e00bde11cafa2b31eee8026063"
+    },
+    {
+      "name": "configuration_agnes.py",
+      "size": 7516,
+      "sha256": "b504c8a6052933ef46f5e9df0d52a09a45b6df41f1f3a2168bfdb4718d7c2342"
+    },
+    {
+      "name": "generation_config.json",
+      "size": 202,
+      "sha256": "e70c136c1b78ddc1fb0905bac8e733a4dc448d4f852a5dd75143fffc70be550e"
+    },
+    {
+      "name": "image_processing_agnes.py",
+      "size": 8700,
+      "sha256": "e7c131d219cd1f8f80730c5b650c32ce107a76f3a5168e12b4a35f754f5253ca"
+    },
+    {
+      "name": "merges.txt",
+      "size": 3353259,
+      "sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d"
+    },
+    {
+      "name": "model-00001.safetensors",
+      "size": 2542796928,
+      "sha256": "54d83c1d36631de231876217a8e0c2483eccee8746369a482b79442bdfc5d958"
+    },
+    {
+      "name": "model-00002.safetensors",
+      "size": 4439047312,
+      "sha256": "e63715ddcf2d7de72a3561034d9f791c3dae259f73fc5e8bd49e47a3108940a6"
+    },
+    {
+      "name": "model-00003.safetensors",
+      "size": 4422852680,
+      "sha256": "b41468b596836b51cfa561687154e9dd4a6f54a77ab5b87e65f7bf2f6d3e4312"
+    },
+    {
+      "name": "model-00004.safetensors",
+      "size": 4422852680,
+      "sha256": "2ca193e4db478721f503d35a8588b065b5ada8c1f553b7256082e5de8869ebd6"
+    },
+    {
+      "name": "model-00005.safetensors",
+      "size": 4495263520,
+      "sha256": "05814cf21a497469ea0c2597c8bf13575447148b25374185016ac13b02c3447b"
+    },
+    {
+      "name": "model-00006.safetensors",
+      "size": 4422852672,
+      "sha256": "fe0c5a9a1301686cbca7e7940ea8c1c6f406c6444c3077dab87891b7425c1f5b"
+    },
+    {
+      "name": "model-00007.safetensors",
+      "size": 4459566912,
+      "sha256": "2b2c237285b8f059de1cd8efd15683d4fae3df7f3dfbb3b1a46d15143645720a"
+    },
+    {
+      "name": "model-00008.safetensors",
+      "size": 4459576904,
+      "sha256": "ea8005e19a879b173fda7ba9029a459f0e0f29099921cd0c51f2b11f2af31f18"
+    },
+    {
+      "name": "model-00009.safetensors",
+      "size": 4417634288,
+      "sha256": "7765ac98fc25e68b7a46052fffe35cf4d0fe6ab3337ba2eb489cfc678295fe03"
+    },
+    {
+      "name": "model-00010.safetensors",
+      "size": 4343701536,
+      "sha256": "9c0f78f02d06afc1e68e54f41a71f628e5965ba58a7c7c0b1877030d0dc35791"
+    },
+    {
+      "name": "model-00011.safetensors",
+      "size": 356557464,
+      "sha256": "116fa885398765cc312c0cf99fe92b533a3a7924d381193757adf2d67f4ac888"
+    },
+    {
+      "name": "model.safetensors.index.json",
+      "size": 138501,
+      "sha256": "eaf5133c51efdf9fac4bcc06f75887a13a47719740ce7bc81f185b6caa7c93d7"
+    },
+    {
+      "name": "modeling_agnes.py",
+      "size": 70576,
+      "sha256": "21eb5cf1c2f6686253000937c668030934bebb53cf1ce423ccd4c2eb47a074c5"
+    },
+    {
+      "name": "preprocessor_config.json",
+      "size": 476,
+      "sha256": "362459e4ba0bcb92bfdfb2a592063368976106eb10c5e9acdb946d727c08f4c8"
+    },
+    {
+      "name": "processing_agnes.py",
+      "size": 5849,
+      "sha256": "98316a130c4f67e4d5bef9ebe80aabf908666cdf989ee7e5e5949c071f26ffdd"
+    },
+    {
+      "name": "serve.sh",
+      "size": 1481,
+      "sha256": "1a9f4db808b9f0ddc9e04b6408a81ff13924e2d39a7daa34076149170e4407bd"
+    },
+    {
+      "name": "sglang_patch/agnes_sglang_config.py",
+      "size": 3174,
+      "sha256": "959819231c778f901d97cd8d2c1d4379ea5a5497486b1df156fb9239a1873a15"
+    },
+    {
+      "name": "sglang_patch/apply_patch.py",
+      "size": 4832,
+      "sha256": "97c3524bb895b9f2159c8d68467767797ead9ae9458407a667c55e7c8657d8e6"
+    },
+    {
+      "name": "sglang_patch/nightly-dev-20260908-20ca564b/sglang/srt/configs/agnes.py",
+      "size": 3174,
+      "sha256": "959819231c778f901d97cd8d2c1d4379ea5a5497486b1df156fb9239a1873a15"
+    },
+    {
+      "name": "sglang_patch/nightly-dev-20260908-20ca564b/sglang/srt/models/qwen3_5.py",
+      "size": 110335,
+      "sha256": "a489dcb3dc15e0b4c7a48ace8b1a9e2055dc49bc92fd59cc937cdb6648735c2d"
+    },
+    {
+      "name": "sglang_patch/nightly-dev-20260908-20ca564b/sglang/srt/utils/hf_transformers/common.py",
+      "size": 25576,
+      "sha256": "62ab258bf96c1b586c42b41c6ff43e96549f902f8feddf84bd23b8921f01d575"
+    },
+    {
+      "name": "sglang_patch/v0.5.19/sglang/srt/configs/agnes.py",
+      "size": 3174,
+      "sha256": "959819231c778f901d97cd8d2c1d4379ea5a5497486b1df156fb9239a1873a15"
+    },
+    {
+      "name": "sglang_patch/v0.5.19/sglang/srt/models/qwen3_5.py",
+      "size": 109787,
+      "sha256": "dc61db0d04f5b3c56a13276f35148229e3dee85c5eacbe7220af09393be9ed7f"
+    },
+    {
+      "name": "sglang_patch/v0.5.19/sglang/srt/utils/hf_transformers/common.py",
+      "size": 23485,
+      "sha256": "6ac72227e51040e22b692821a01bf36555d0b298f735740fece328a1817df9e4"
+    },
+    {
+      "name": "tokenizer.json",
+      "size": 9835269,
+      "sha256": "f7ffe83fabdd9d1861e1d52e1ebf982a5c8accab07a0e867132d06ec6bdd3894"
+    },
+    {
+      "name": "tokenizer_config.json",
+      "size": 18884,
+      "sha256": "914f96a0624c33e2d899a042413c957df6082176b6b2990f7c87045b7182263e"
+    },
+    {
+      "name": "video_preprocessor_config.json",
+      "size": 460,
+      "sha256": "2ec70c5d9e01c6b0c14b882913b6319da78e1efa49037d20b143d59ea37455c1"
+    },
+    {
+      "name": "video_processing_agnes.py",
+      "size": 8832,
+      "sha256": "b1c1f1849736a823903a7a3eb7f2b57ad81ee15131f65a4ee616ec32b9655b28"
+    },
+    {
+      "name": "vocab.json",
+      "size": 6722759,
+      "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003"
+    }
+  ]
+}

model-00001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54d83c1d36631de231876217a8e0c2483eccee8746369a482b79442bdfc5d958
+size 2542796928

model-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e63715ddcf2d7de72a3561034d9f791c3dae259f73fc5e8bd49e47a3108940a6
+size 4439047312

model-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b41468b596836b51cfa561687154e9dd4a6f54a77ab5b87e65f7bf2f6d3e4312
+size 4422852680

model-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ca193e4db478721f503d35a8588b065b5ada8c1f553b7256082e5de8869ebd6
+size 4422852680

model-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05814cf21a497469ea0c2597c8bf13575447148b25374185016ac13b02c3447b
+size 4495263520

model-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe0c5a9a1301686cbca7e7940ea8c1c6f406c6444c3077dab87891b7425c1f5b
+size 4422852672

model-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b2c237285b8f059de1cd8efd15683d4fae3df7f3dfbb3b1a46d15143645720a
+size 4459566912

model-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea8005e19a879b173fda7ba9029a459f0e0f29099921cd0c51f2b11f2af31f18
+size 4459576904

model-00009.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7765ac98fc25e68b7a46052fffe35cf4d0fe6ab3337ba2eb489cfc678295fe03
+size 4417634288

model-00010.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c0f78f02d06afc1e68e54f41a71f628e5965ba58a7c7c0b1877030d0dc35791
+size 4343701536

model-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:116fa885398765cc312c0cf99fe92b533a3a7924d381193757adf2d67f4ac888
+size 356557464

modeling_agnes.py

@@ -0,0 +1,1559 @@
+# Copyright 2026 Agnes AI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Agnes 3.0 Flash.
+
+A hybrid decoder: three delta-rule recurrent layers followed by one global
+attention layer, repeated; every layer carries a main SwiGLU MLP plus a
+parallel SwiGLU branch summed into the same residual.  A vision tower feeds
+merged patch tokens into the language model through placeholder tokens.
+"""
+
+import itertools
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any, Optional
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from transformers import initialization as init
+from transformers.activations import ACT2FN
+from transformers.cache_utils import LAYER_TYPE_CACHE_MAPPING, Cache, DynamicCache, DynamicLayer, LinearAttentionLayer
+from transformers.generation import GenerationMixin
+from transformers.masking_utils import LAYER_PATTERN_TO_MASK_FUNCTION_MAPPING, create_causal_mask
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
+from transformers.modeling_layers import GradientCheckpointingLayer
+from transformers.modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, CausalLMOutputWithPast
+from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from transformers.processing_utils import Unpack
+from transformers.utils import TransformersKwargs, auto_docstring, can_return_tuple, logging, torch_compilable_check
+from transformers.utils.generic import (
+    accepts_precomputed_kwargs,
+    is_flash_attention_requested,
+    maybe_autocast,
+    merge_with_config_defaults,
+)
+from transformers.utils.import_utils import is_causal_conv1d_available, is_flash_linear_attention_available
+from transformers.utils.output_capturing import capture_outputs
+from transformers.vision_utils import get_vision_bilinear_indices_and_weights, get_vision_cu_seqlens, get_vision_position_ids
+
+from .configuration_agnes import LAYER_DELTA, LAYER_GLOBAL, AgnesConfig, AgnesTextConfig, AgnesVisionConfig
+
+
+logger = logging.get_logger(__name__)
+
+# Optional fused kernels.  When absent the pure-torch paths below are used;
+# both paths are numerically interchangeable.
+if is_causal_conv1d_available():
+    from causal_conv1d import causal_conv1d_fn, causal_conv1d_update
+else:
+    causal_conv1d_fn = causal_conv1d_update = None
+
+if is_flash_linear_attention_available():
+    from fla.modules import FusedRMSNormGated
+    from fla.ops.gated_delta_rule import chunk_gated_delta_rule, fused_recurrent_gated_delta_rule
+else:
+    FusedRMSNormGated = None
+    chunk_gated_delta_rule = fused_recurrent_gated_delta_rule = None
+
+_FUSED_DELTA_PATH = all((causal_conv1d_fn, causal_conv1d_update, chunk_gated_delta_rule, fused_recurrent_gated_delta_rule))
+
+
+# ---------------------------------------------------------------------------
+# Layer-type plumbing: the cache builds one cache layer per entry of
+# config.layer_types through a registry keyed by type name, and generation
+# picks a mask function the same way.  Both are told about this model's two
+# layer types here, at import time.
+# ---------------------------------------------------------------------------
+
+
+class AgnesGlobalCacheLayer(DynamicLayer):
+    layer_type = LAYER_GLOBAL
+
+
+class AgnesDeltaCacheLayer(LinearAttentionLayer):
+    layer_type = LAYER_DELTA
+
+
+# DynamicLayer subclasses self-register through CacheLayerMixin; the linear-attention
+# base does not share that hook, and an unknown type silently falls back to a plain
+# KV layer, so both entries are written explicitly.
+LAYER_TYPE_CACHE_MAPPING[LAYER_GLOBAL] = AgnesGlobalCacheLayer
+LAYER_TYPE_CACHE_MAPPING[LAYER_DELTA] = AgnesDeltaCacheLayer
+LAYER_PATTERN_TO_MASK_FUNCTION_MAPPING.setdefault(LAYER_GLOBAL, create_causal_mask)
+LAYER_PATTERN_TO_MASK_FUNCTION_MAPPING.setdefault(LAYER_DELTA, create_causal_mask)
+
+
+# ---------------------------------------------------------------------------
+# Normalisation
+# ---------------------------------------------------------------------------
+
+
+class _RMSNormFn(torch.autograd.Function):
+    """RMS normalisation with a one-centred scale, y = x / rms(x) * (1 + w).
+
+    The forward matches the reference op-for-op (fp32 statistics, cast back to
+    the input dtype at the end); the backward is written out analytically
+    instead of being traced through the forward graph.
+    """
+
+    @staticmethod
+    def forward(ctx, x, weight, eps):
+        xf = x.float()
+        inv = torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)
+        unit = xf * inv
+        out = unit * (1.0 + weight.float())
+        ctx.save_for_backward(unit, inv, weight)
+        ctx.in_dtype = x.dtype
+        return out.type_as(x)
+
+    @staticmethod
+    def backward(ctx, grad_out):
+        unit, inv, weight = ctx.saved_tensors
+        g = grad_out.float()
+        g_unit = g * (1.0 + weight.float())
+        grad_x = inv * (g_unit - unit * (g_unit * unit).mean(-1, keepdim=True))
+        grad_w = (g * unit).reshape(-1, unit.shape[-1]).sum(0)
+        return grad_x.to(ctx.in_dtype), grad_w.to(weight.dtype), None
+
+
+class AgnesRMSNorm(nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-6):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.zeros(dim))
+
+    def forward(self, x):
+        return _RMSNormFn.apply(x, self.weight, self.eps)
+
+    def extra_repr(self):
+        return f"{tuple(self.weight.shape)}, eps={self.eps}"
+
+
+class _GatedRMSNormFn(torch.autograd.Function):
+    """RMS normalisation followed by a learned scale and a SiLU gate.
+
+    Forward order (matters for bit-exactness): fp32 statistics, cast to the
+    input dtype, multiply by the weight in that dtype, then multiply by the
+    fp32 gate activation and cast back.
+    """
+
+    @staticmethod
+    def forward(ctx, x, weight, gate, eps):
+        in_dtype = x.dtype
+        xf = x.to(torch.float32)
+        inv = torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)
+        unit = xf * inv
+        scaled = weight * unit.to(in_dtype)
+        gate_f = gate.to(torch.float32)
+        act = F.silu(gate_f)
+        out = scaled * act
+        ctx.save_for_backward(unit, inv, weight, gate_f, scaled)
+        ctx.in_dtype = in_dtype
+        return out.to(in_dtype)
+
+    @staticmethod
+    def backward(ctx, grad_out):
+        unit, inv, weight, gate_f, scaled = ctx.saved_tensors
+        g = grad_out.float()
+        sig = torch.sigmoid(gate_f)
+        act = gate_f * sig
+        g_scaled = g * act
+        grad_w = (g_scaled * unit).reshape(-1, unit.shape[-1]).sum(0)
+        g_unit = g_scaled * weight.float()
+        grad_x = inv * (g_unit - unit * (g_unit * unit).mean(-1, keepdim=True))
+        grad_gate = g * scaled.float() * (sig * (1.0 + gate_f * (1.0 - sig)))
+        return grad_x.to(ctx.in_dtype), grad_w.to(weight.dtype), grad_gate.to(ctx.in_dtype), None
+
+
+class AgnesGatedNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6, **kwargs):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states, gate=None):
+        return _GatedRMSNormFn.apply(hidden_states, self.weight, gate, self.variance_epsilon)
+
+
+# ---------------------------------------------------------------------------
+# Feed-forward
+# ---------------------------------------------------------------------------
+
+
+class AgnesMLP(nn.Module):
+    """SwiGLU block.  With `parallel_size > 0` a second, narrower SwiGLU runs
+    on the same input and its output is added to the main one."""
+
+    def __init__(self, config, intermediate_size: int, parallel_size: int = 0):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = intermediate_size
+        self.gate_proj = nn.Linear(self.hidden_size, intermediate_size, bias=False)
+        self.up_proj = nn.Linear(self.hidden_size, intermediate_size, bias=False)
+        self.down_proj = nn.Linear(intermediate_size, self.hidden_size, bias=False)
+        self.act_fn = ACT2FN[config.hidden_act]
+        self.parallel_ffn = AgnesMLP(config, parallel_size) if parallel_size > 0 else None
+
+    def forward(self, x):
+        y = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+        if self.parallel_ffn is not None:
+            y = y + self.parallel_ffn(x)
+        return y
+
+
+# ---------------------------------------------------------------------------
+# Rotary position encodings
+# ---------------------------------------------------------------------------
+
+
+def _swap_halves(x):
+    """(a, b) -> (-b, a) along the last axis."""
+    half = x.shape[-1] // 2
+    return torch.cat((-x[..., half:], x[..., :half]), dim=-1)
+
+
+def _apply_rope(q, k, cos, sin, unsqueeze_dim=1):
+    """Rotate the leading `cos.shape[-1]` channels of q and k; the rest pass through."""
+    cos = cos.unsqueeze(unsqueeze_dim)
+    sin = sin.unsqueeze(unsqueeze_dim)
+    n_rot = cos.shape[-1]
+    q_rot, q_rest = q[..., :n_rot], q[..., n_rot:]
+    k_rot, k_rest = k[..., :n_rot], k[..., n_rot:]
+    q_rot = (q_rot * cos) + (_swap_halves(q_rot) * sin)
+    k_rot = (k_rot * cos) + (_swap_halves(k_rot) * sin)
+    return torch.cat([q_rot, q_rest], dim=-1), torch.cat([k_rot, k_rest], dim=-1)
+
+
+class AgnesRotaryEmbedding(nn.Module):
+    """Three-axis (text / height / width) rotary tables with interleaved sections."""
+
+    inv_freq: torch.Tensor
+
+    def __init__(self, config: AgnesTextConfig, device=None):
+        super().__init__()
+        self.config = config
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
+        self.rope_type = config.rope_parameters["rope_type"]
+        builder: Callable = self.compute_default_rope_parameters
+        if self.rope_type != "default":
+            builder = ROPE_INIT_FUNCTIONS[self.rope_type]
+        inv_freq, self.attention_scaling = builder(config, device)
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False)
+        self.mrope_section = config.rope_parameters.get("mrope_section", [11, 11, 10])
+
+    @staticmethod
+    def compute_default_rope_parameters(
+        config: AgnesTextConfig | None = None,
+        device: Optional["torch.device"] = None,
+        seq_len: int | None = None,
+    ) -> tuple["torch.Tensor", float]:
+        theta = config.rope_parameters["rope_theta"]
+        fraction = config.rope_parameters.get("partial_rotary_factor", 1.0)
+        head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
+        n_rot = int(head_dim * fraction)
+        exponents = torch.arange(0, n_rot, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / n_rot
+        inv_freq = 1.0 / (theta**exponents)
+        return inv_freq, 1.0
+
+    @torch.no_grad()
+    @dynamic_rope_update
+    def forward(self, x, position_ids):
+        if position_ids.ndim == 2:
+            position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)
+        freq_col = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1).to(x.device)
+        pos_row = position_ids[:, :, None, :].float()
+        dev = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
+        with maybe_autocast(device_type=dev, enabled=False):
+            angles = (freq_col.float() @ pos_row.float()).transpose(2, 3)
+            angles = self._interleave_axes(angles, self.mrope_section)
+            table = torch.cat((angles, angles), dim=-1)
+            cos = table.cos() * self.attention_scaling
+            sin = table.sin() * self.attention_scaling
+        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+    @staticmethod
+    def _interleave_axes(angles, section):
+        """Fold the H and W axes into the T table at every 2nd / 3rd frequency slot."""
+        merged = angles[0]
+        for axis, offset in ((1, 1), (2, 2)):
+            take = slice(offset, section[axis] * 3, 3)
+            merged[..., take] = angles[axis, ..., take]
+        return merged
+
+
+class AgnesVisionRotary(nn.Module):
+    inv_freq: torch.Tensor
+
+    def __init__(self, dim: int, theta: float = 10000.0) -> None:
+        super().__init__()
+        self.dim = dim
+        self.theta = theta
+        inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+    def forward(self, position_ids: torch.Tensor) -> torch.Tensor:
+        return (position_ids.unsqueeze(-1) * self.inv_freq).flatten(1)
+
+
+def _apply_vision_rope(q, k, cos, sin):
+    q_dtype, k_dtype = q.dtype, k.dtype
+    q, k = q.float(), k.float()
+    cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float()
+    q = (q * cos) + (_swap_halves(q) * sin)
+    k = (k * cos) + (_swap_halves(k) * sin)
+    return q.to(q_dtype), k.to(k_dtype)
+
+
+# ---------------------------------------------------------------------------
+# Global (softmax) attention
+# ---------------------------------------------------------------------------
+
+
+def _broadcast_kv(x: torch.Tensor, groups: int) -> torch.Tensor:
+    """Repeat each kv head `groups` times along the head axis."""
+    b, h, t, d = x.shape
+    if groups == 1:
+        return x
+    return x[:, :, None, :, :].expand(b, h, groups, t, d).reshape(b, h * groups, t, d)
+
+
+def _dense_attention(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    scaling: float,
+    dropout: float = 0.0,
+    **kwargs: Unpack[TransformersKwargs],
+):
+    k = _broadcast_kv(key, module.num_key_value_groups)
+    v = _broadcast_kv(value, module.num_key_value_groups)
+    scores = torch.matmul(query, k.transpose(2, 3)) * scaling
+    if attention_mask is not None:
+        scores = scores + attention_mask
+    probs = nn.functional.softmax(scores, dim=-1, dtype=torch.float32).to(query.dtype)
+    probs = nn.functional.dropout(probs, p=dropout, training=module.training)
+    out = torch.matmul(probs, v).transpose(1, 2).contiguous()
+    return out, probs
+
+
+class AgnesGlobalAttention(nn.Module):
+    """Grouped-query softmax attention with per-head q/k normalisation and a
+    sigmoid output gate produced alongside the queries."""
+
+    def __init__(self, config: AgnesTextConfig, layer_idx: int):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
+        self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
+        self.scaling = self.head_dim**-0.5
+        self.attention_dropout = config.attention_dropout
+        self.is_causal = True
+        q_width = config.num_attention_heads * self.head_dim
+        kv_width = config.num_key_value_heads * self.head_dim
+        self.q_proj = nn.Linear(config.hidden_size, q_width * 2, bias=config.attention_bias)
+        self.k_proj = nn.Linear(config.hidden_size, kv_width, bias=config.attention_bias)
+        self.v_proj = nn.Linear(config.hidden_size, kv_width, bias=config.attention_bias)
+        self.o_proj = nn.Linear(q_width, config.hidden_size, bias=config.attention_bias)
+        self.q_norm = AgnesRMSNorm(self.head_dim, eps=config.rms_norm_eps)
+        self.k_norm = AgnesRMSNorm(self.head_dim, eps=config.rms_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        attention_mask: torch.Tensor | None,
+        past_key_values: Cache | None = None,
+        **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> tuple[torch.Tensor, torch.Tensor | None]:
+        lead = hidden_states.shape[:-1]
+        per_head = (*lead, -1, self.head_dim)
+
+        q, gate = torch.chunk(self.q_proj(hidden_states).view(*lead, -1, self.head_dim * 2), 2, dim=-1)
+        gate = gate.reshape(*lead, -1)
+        q = self.q_norm(q.view(per_head)).transpose(1, 2)
+        k = self.k_norm(self.k_proj(hidden_states).view(per_head)).transpose(1, 2)
+        v = self.v_proj(hidden_states).view(per_head).transpose(1, 2)
+
+        cos, sin = position_embeddings
+        q, k = _apply_rope(q, k, cos, sin)
+        if past_key_values is not None:
+            k, v = past_key_values.update(k, v, self.layer_idx)
+
+        attend: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(self.config._attn_implementation, _dense_attention)
+        out, probs = attend(
+            self, q, k, v, attention_mask,
+            dropout=0.0 if not self.training else self.attention_dropout,
+            scaling=self.scaling,
+            **kwargs,
+        )
+        out = out.reshape(*lead, -1).contiguous()
+        out = out * torch.sigmoid(gate)
+        return self.o_proj(out), probs
+
+
+# ---------------------------------------------------------------------------
+# Delta-rule (recurrent) attention
+# ---------------------------------------------------------------------------
+
+
+def _zero_padded_positions(hidden_states, attention_mask):
+    """Mask out padding tokens so they leave no trace in the recurrent state."""
+    if attention_mask is not None and attention_mask.shape[1] > 1 and attention_mask.shape[0] > 1:
+        dt = hidden_states.dtype
+        hidden_states = (hidden_states * attention_mask[:, :, None]).to(dt)
+    return hidden_states
+
+
+def _causal_conv_step(hidden_states, conv_state, weight, bias=None, activation=None):
+    """One decode step of the depthwise causal conv, updating `conv_state` in place."""
+    _, channels, steps = hidden_states.shape
+    window = conv_state.shape[-1]
+    joined = torch.cat([conv_state, hidden_states], dim=-1).to(weight.dtype)
+    conv_state.copy_(joined[:, :, -window:])
+    y = F.conv1d(joined, weight.unsqueeze(1), bias, padding=0, groups=channels)
+    y = F.silu(y[:, :, -steps:])
+    return y.to(hidden_states.dtype)
+
+
+def _unit_normalize(x: torch.FloatTensor, dim: int = -1, eps: float = 1e-6):
+    return x * torch.rsqrt((x * x).sum(dim=dim, keepdim=True) + eps)
+
+
+def _delta_rule_chunked(
+    query, key, value, g, beta, chunk_size=64, initial_state=None, output_final_state=False,
+    use_qk_l2norm_in_kernel=False, **kwargs,
+):
+    """Chunk-parallel gated delta rule (prefill path)."""
+    out_dtype = query.dtype
+    if use_qk_l2norm_in_kernel:
+        query = _unit_normalize(query, dim=-1, eps=1e-6)
+        key = _unit_normalize(key, dim=-1, eps=1e-6)
+    query, key, value, beta, g = [t.transpose(1, 2).contiguous().to(torch.float32) for t in (query, key, value, beta, g)]
+
+    bsz, heads, seq, dk = key.shape
+    dv = value.shape[-1]
+    pad = (chunk_size - seq % chunk_size) % chunk_size
+    query = F.pad(query, (0, 0, 0, pad))
+    key = F.pad(key, (0, 0, 0, pad))
+    value = F.pad(value, (0, 0, 0, pad))
+    beta = F.pad(beta, (0, pad))
+    g = F.pad(g, (0, pad))
+    padded = seq + pad
+    query = query * (1 / (query.shape[-1] ** 0.5))
+
+    v_beta = value * beta.unsqueeze(-1)
+    k_beta = key * beta.unsqueeze(-1)
+    query, key, value, k_beta, v_beta = [
+        t.reshape(t.shape[0], t.shape[1], -1, chunk_size, t.shape[-1]) for t in (query, key, value, k_beta, v_beta)
+    ]
+    g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size)
+    upper = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0)
+
+    g = g.cumsum(dim=-1)
+    decay = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril()
+    solve = -((k_beta @ key.transpose(-1, -2)) * decay).masked_fill(upper, 0)
+    for i in range(1, chunk_size):
+        row = solve[..., i, :i].clone()
+        block = solve[..., :i, :i].clone()
+        solve[..., i, :i] = row + (row.unsqueeze(-1) * block).sum(-2)
+    solve = solve + torch.eye(chunk_size, dtype=solve.dtype, device=solve.device)
+    value = solve @ v_beta
+    k_decayed = solve @ (k_beta * g.exp().unsqueeze(-1))
+    state = (
+        torch.zeros(bsz, heads, dk, dv, dtype=value.dtype, device=value.device)
+        if initial_state is None
+        else initial_state.to(value)
+    )
+    out = torch.zeros_like(value)
+    strict_upper = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=1)
+
+    for i in range(0, padded // chunk_size):
+        q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i]
+        local = q_i @ k_i.transpose(-1, -2) * decay[:, :, i]
+        v_pred = (k_decayed[:, :, i]) @ state
+        v_res = v_i - v_pred
+        carried = (q_i * g[:, :, i, :, None].exp()) @ state
+        out[:, :, i] = carried + local @ v_res
+        state = (
+            state * g[:, :, i, -1, None, None].exp()
+            + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) @ v_res
+        )
+
+    if not output_final_state:
+        state = None
+    out = out.reshape(out.shape[0], out.shape[1], -1, out.shape[-1])
+    out = out[:, :, :seq]
+    return out.transpose(1, 2).contiguous().to(out_dtype), state
+
+
+def _delta_rule_stepwise(query, key, value, g, beta, initial_state, output_final_state, use_qk_l2norm_in_kernel=False):
+    """Token-by-token gated delta rule (decode path)."""
+    out_dtype = query.dtype
+    if use_qk_l2norm_in_kernel:
+        query = _unit_normalize(query, dim=-1, eps=1e-6)
+        key = _unit_normalize(key, dim=-1, eps=1e-6)
+    query, key, value, beta, g = [t.transpose(1, 2).contiguous().to(torch.float32) for t in (query, key, value, beta, g)]
+
+    bsz, heads, seq, dk = key.shape
+    dv = value.shape[-1]
+    query = query * (1 / (query.shape[-1] ** 0.5))
+
+    out = torch.zeros(bsz, heads, seq, dv, dtype=value.dtype, device=value.device)
+    state = (
+        torch.zeros(bsz, heads, dk, dv, dtype=value.dtype, device=value.device)
+        if initial_state is None
+        else initial_state.to(value)
+    )
+    for t in range(seq):
+        q_t, k_t, v_t = query[:, :, t], key[:, :, t], value[:, :, t]
+        decay_t = g[:, :, t].exp().unsqueeze(-1).unsqueeze(-1)
+        beta_t = beta[:, :, t].unsqueeze(-1)
+        state = state * decay_t
+        recalled = (state * k_t.unsqueeze(-1)).sum(dim=-2)
+        correction = (v_t - recalled) * beta_t
+        state = state + k_t.unsqueeze(-1) * correction.unsqueeze(-2)
+        out[:, :, t] = (state * q_t.unsqueeze(-1)).sum(dim=-2)
+
+    if not output_final_state:
+        state = None
+    return out.transpose(1, 2).contiguous().to(out_dtype), state
+
+
+class AgnesDeltaAttention(nn.Module):
+    """Gated delta-rule recurrent attention with a causal depthwise conv on
+    the projected q/k/v and a gated RMS norm on the output."""
+
+    def __init__(self, config: AgnesTextConfig, layer_idx: int):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        self.num_v_heads = config.linear_num_value_heads
+        self.num_k_heads = config.linear_num_key_heads
+        self.head_k_dim = config.linear_key_head_dim
+        self.head_v_dim = config.linear_value_head_dim
+        self.key_dim = self.head_k_dim * self.num_k_heads
+        self.value_dim = self.head_v_dim * self.num_v_heads
+        self.conv_kernel_size = config.linear_conv_kernel_dim
+        self.layer_idx = layer_idx
+        self.activation = config.hidden_act
+        self.act = ACT2FN[config.hidden_act]
+        self.layer_norm_epsilon = config.rms_norm_eps
+
+        self.conv_dim = self.key_dim * 2 + self.value_dim
+        self.conv1d = nn.Conv1d(
+            in_channels=self.conv_dim, out_channels=self.conv_dim, bias=False,
+            kernel_size=self.conv_kernel_size, groups=self.conv_dim, padding=self.conv_kernel_size - 1,
+        )
+        self.dt_bias = nn.Parameter(torch.ones(self.num_v_heads))
+        self.A_log = nn.Parameter(torch.log(torch.empty(self.num_v_heads).uniform_(0, 16)))
+
+        if FusedRMSNormGated is None:
+            self.norm = AgnesGatedNorm(self.head_v_dim, eps=self.layer_norm_epsilon)
+        else:
+            self.norm = FusedRMSNormGated(
+                self.head_v_dim, eps=self.layer_norm_epsilon, activation=self.activation,
+                device=torch.cuda.current_device(),
+                dtype=config.dtype if config.dtype is not None else torch.get_default_dtype(),
+            )
+        self.out_proj = nn.Linear(self.value_dim, self.hidden_size, bias=False)
+
+        self.causal_conv1d_fn = causal_conv1d_fn
+        self.causal_conv1d_update = causal_conv1d_update or _causal_conv_step
+        self.chunk_gated_delta_rule = chunk_gated_delta_rule or _delta_rule_chunked
+        self.recurrent_gated_delta_rule = fused_recurrent_gated_delta_rule or _delta_rule_stepwise
+        if not _FUSED_DELTA_PATH:
+            logger.warning_once(
+                "Fused delta-rule kernels not found (flash-linear-attention / causal-conv1d); "
+                "using the pure-torch implementation."
+            )
+
+        self.in_proj_qkv = nn.Linear(self.hidden_size, self.key_dim * 2 + self.value_dim, bias=False)
+        self.in_proj_z = nn.Linear(self.hidden_size, self.value_dim, bias=False)
+        self.in_proj_b = nn.Linear(self.hidden_size, self.num_v_heads, bias=False)
+        self.in_proj_a = nn.Linear(self.hidden_size, self.num_v_heads, bias=False)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cache_params: Cache | None = None,
+        attention_mask: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ):
+        hidden_states = _zero_padded_positions(hidden_states, attention_mask)
+        bsz, seq, _ = hidden_states.shape
+
+        resume = cache_params is not None and cache_params.has_previous_state(self.layer_idx)
+        if resume:
+            conv_state = cache_params.layers[self.layer_idx].conv_states
+            rec_state = cache_params.layers[self.layer_idx].recurrent_states
+
+        qkv = self.in_proj_qkv(hidden_states).transpose(1, 2)
+        z = self.in_proj_z(hidden_states).reshape(bsz, seq, -1, self.head_v_dim)
+        b = self.in_proj_b(hidden_states)
+        a = self.in_proj_a(hidden_states)
+
+        if resume and seq == 1:
+            qkv = self.causal_conv1d_update(qkv, conv_state, self.conv1d.weight.squeeze(1), self.conv1d.bias, self.activation)
+        else:
+            if resume:
+                qkv = torch.cat([conv_state, qkv], dim=-1)
+            if cache_params is not None:
+                cache_params.update_conv_state(F.pad(qkv, (self.conv_kernel_size - qkv.shape[-1], 0)), self.layer_idx)
+            if self.causal_conv1d_fn is not None:
+                qkv = self.causal_conv1d_fn(
+                    x=qkv, weight=self.conv1d.weight.squeeze(1), bias=self.conv1d.bias,
+                    activation=self.activation, seq_idx=kwargs.get("seq_idx"),
+                )
+            else:
+                qkv = F.silu(self.conv1d(qkv)[:, :, : qkv.shape[-1]])
+            if resume:
+                qkv = qkv[:, :, -seq:]
+
+        qkv = qkv.transpose(1, 2)
+        query, key, value = torch.split(qkv, [self.key_dim, self.key_dim, self.value_dim], dim=-1)
+        query = query.reshape(bsz, seq, -1, self.head_k_dim)
+        key = key.reshape(bsz, seq, -1, self.head_k_dim)
+        value = value.reshape(bsz, seq, -1, self.head_v_dim)
+
+        beta = b.sigmoid()
+        # fp32 keeps exp(A_log) finite under fp16 weights
+        g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
+        groups = self.num_v_heads // self.num_k_heads
+        if groups > 1:
+            query = query.repeat_interleave(groups, dim=2)
+            key = key.repeat_interleave(groups, dim=2)
+
+        if resume and seq == 1:
+            mixed, rec_state = self.recurrent_gated_delta_rule(
+                query, key, value, g=g, beta=beta, initial_state=rec_state,
+                output_final_state=cache_params is not None, use_qk_l2norm_in_kernel=True,
+            )
+        else:
+            mixed, rec_state = self.chunk_gated_delta_rule(
+                query, key, value, g=g, beta=beta,
+                initial_state=rec_state if resume else None,
+                output_final_state=cache_params is not None, use_qk_l2norm_in_kernel=True,
+                cu_seqlens=kwargs.get("cu_seq_lens_q"),
+            )
+        if cache_params is not None:
+            cache_params.update_recurrent_state(rec_state, self.layer_idx)
+
+        mixed = self.norm(mixed.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim))
+        return self.out_proj(mixed.reshape(bsz, seq, -1))
+
+
+# ---------------------------------------------------------------------------
+# Decoder layer / base class
+# ---------------------------------------------------------------------------
+
+
+class AgnesDecoderLayer(GradientCheckpointingLayer):
+    def __init__(self, config: AgnesTextConfig, layer_idx: int):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        self.layer_type = config.layer_types[layer_idx]
+        if self.layer_type == LAYER_DELTA:
+            self.delta_attn = AgnesDeltaAttention(config, layer_idx)
+        elif self.layer_type == LAYER_GLOBAL:
+            self.global_attn = AgnesGlobalAttention(config, layer_idx)
+        self.mlp = AgnesMLP(config, config.intermediate_size, getattr(config, "parallel_ffn_intermediate_size", 0) or 0)
+        self.input_layernorm = AgnesRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = AgnesRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> torch.FloatTensor:
+        skip = hidden_states
+        h = self.input_layernorm(hidden_states)
+        if self.layer_type == LAYER_DELTA:
+            h = self.delta_attn(hidden_states=h, cache_params=past_key_values, attention_mask=attention_mask, **kwargs)
+        elif self.layer_type == LAYER_GLOBAL:
+            h, _ = self.global_attn(
+                hidden_states=h, attention_mask=attention_mask, position_ids=position_ids,
+                past_key_values=past_key_values, position_embeddings=position_embeddings, **kwargs,
+            )
+        h = skip + h
+
+        skip = h
+        h = self.mlp(self.post_attention_layernorm(h))
+        return skip + h
+
+
+class AgnesPreTrainedModel(PreTrainedModel):
+    config: AgnesConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["AgnesDecoderLayer", "AgnesVisionBlock"]
+    _skip_keys_device_placement = ["past_key_values"]
+    _supports_flash_attn = True
+    _supports_sdpa = True
+    _keys_to_ignore_on_load_unexpected = [r"^mtp.*"]
+    _can_record_outputs = {
+        "hidden_states": AgnesDecoderLayer,
+        "attentions": AgnesGlobalAttention,
+    }
+    _is_stateful = True
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        super()._init_weights(module)
+        if isinstance(module, AgnesDeltaAttention):
+            init.ones_(module.dt_bias)
+            init.copy_(module.A_log, torch.empty_like(module.A_log).uniform_(0, 16).log_())
+        elif isinstance(module, AgnesRMSNorm):
+            # the norm scales by (1 + weight), so zero is the identity
+            init.zeros_(module.weight)
+        elif isinstance(module, AgnesVisionRotary):
+            inv_freq = 1.0 / (module.theta ** (torch.arange(0, module.dim, 2, dtype=torch.float) / module.dim))
+            init.copy_(module.inv_freq, inv_freq)
+
+
+# ---------------------------------------------------------------------------
+# Vision tower
+# ---------------------------------------------------------------------------
+
+
+class AgnesVisionPatchEmbed(nn.Module):
+    def __init__(self, config) -> None:
+        super().__init__()
+        self.patch_size = config.patch_size
+        self.temporal_patch_size = config.temporal_patch_size
+        self.in_channels = config.in_channels
+        self.embed_dim = config.hidden_size
+        kernel = [self.temporal_patch_size, self.patch_size, self.patch_size]
+        self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel, stride=kernel, bias=True)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        dt = self.proj.weight.dtype
+        patches = hidden_states.view(-1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size)
+        return self.proj(patches.to(dtype=dt)).view(-1, self.embed_dim)
+
+
+class AgnesVisionMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+        self.linear_fc1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=True)
+        self.linear_fc2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=True)
+        self.act_fn = ACT2FN[config.hidden_act]
+
+    def forward(self, hidden_state):
+        return self.linear_fc2(self.act_fn(self.linear_fc1(hidden_state)))
+
+
+class AgnesVisionAttention(nn.Module):
+    def __init__(self, config: AgnesVisionConfig) -> None:
+        super().__init__()
+        self.dim = config.hidden_size
+        self.num_heads = config.num_heads
+        self.head_dim = self.dim // self.num_heads
+        self.num_key_value_groups = 1
+        self.qkv = nn.Linear(self.dim, self.dim * 3, bias=True)
+        self.proj = nn.Linear(self.dim, self.dim)
+        self.scaling = self.head_dim**-0.5
+        self.config = config
+        self.attention_dropout = 0.0
+        self.is_causal = False
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cu_seqlens: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
+        **kwargs,
+    ) -> torch.Tensor:
+        n_tok = hidden_states.shape[0]
+        q, k, v = self.qkv(hidden_states).reshape(n_tok, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0)
+        cos, sin = position_embeddings
+        q, k = _apply_vision_rope(q, k, cos, sin)
+        q = q.transpose(0, 1).unsqueeze(0)
+        k = k.transpose(0, 1).unsqueeze(0)
+        v = v.transpose(0, 1).unsqueeze(0)
+
+        attend: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(self.config._attn_implementation, _dense_attention)
+        drop = 0.0 if not self.training else self.attention_dropout
+        if is_flash_attention_requested(self.config):
+            longest = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
+            out, _ = attend(
+                self, q, k, v, attention_mask=None, scaling=self.scaling, dropout=drop,
+                cu_seq_lens_q=cu_seqlens, cu_seq_lens_k=cu_seqlens, max_length_q=longest, max_length_k=longest,
+                is_causal=False, **kwargs,
+            )
+        else:
+            spans = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
+            pieces = [torch.split(t, spans, dim=2) for t in (q, k, v)]
+            out = torch.cat(
+                [
+                    attend(self, qi, ki, vi, attention_mask=None, scaling=self.scaling, dropout=drop, is_causal=False, **kwargs)[0]
+                    for qi, ki, vi in zip(*pieces)
+                ],
+                dim=1,
+            )
+        return self.proj(out.reshape(n_tok, -1).contiguous())
+
+
+class AgnesVisionBlock(GradientCheckpointingLayer):
+    def __init__(self, config, attn_implementation: str = "sdpa") -> None:
+        super().__init__()
+        self.norm1 = nn.LayerNorm(config.hidden_size, eps=1e-6)
+        self.norm2 = nn.LayerNorm(config.hidden_size, eps=1e-6)
+        self.attn = AgnesVisionAttention(config=config)
+        self.mlp = AgnesVisionMLP(config=config)
+
+    @auto_docstring
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cu_seqlens: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
+        **kwargs,
+    ) -> torch.Tensor:
+        r"""
+        cu_seqlens (`torch.Tensor`):
+            Cumulative sequence lengths used for packed variable-length attention.
+        """
+        hidden_states = hidden_states + self.attn(
+            self.norm1(hidden_states), cu_seqlens=cu_seqlens, position_embeddings=position_embeddings, **kwargs
+        )
+        return hidden_states + self.mlp(self.norm2(hidden_states))
+
+
+class AgnesVisionMerger(nn.Module):
+    def __init__(self, config: AgnesVisionConfig, use_postshuffle_norm=False) -> None:
+        super().__init__()
+        self.hidden_size = config.hidden_size * (config.spatial_merge_size**2)
+        self.use_postshuffle_norm = use_postshuffle_norm
+        self.norm = nn.LayerNorm(self.hidden_size if use_postshuffle_norm else config.hidden_size, eps=1e-6)
+        self.linear_fc1 = nn.Linear(self.hidden_size, self.hidden_size)
+        self.act_fn = nn.GELU()
+        self.linear_fc2 = nn.Linear(self.hidden_size, config.out_hidden_size)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.norm(x.view(-1, self.hidden_size) if self.use_postshuffle_norm else x).view(-1, self.hidden_size)
+        return self.linear_fc2(self.act_fn(self.linear_fc1(x)))
+
+
+class AgnesVisionModel(AgnesPreTrainedModel):
+    config: AgnesVisionConfig
+    config_class = AgnesVisionConfig
+    input_modalities = ("image", "video")
+    _can_record_outputs = {
+        "hidden_states": AgnesVisionBlock,
+        "attentions": AgnesVisionAttention,
+    }
+    _no_split_modules = ["AgnesVisionBlock"]
+
+    def __init__(self, config, *inputs, **kwargs) -> None:
+        super().__init__(config, *inputs, **kwargs)
+        self.spatial_merge_size = config.spatial_merge_size
+        self.patch_size = config.patch_size
+        self.spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size
+        self.patch_embed = AgnesVisionPatchEmbed(config=config)
+        self.pos_embed = nn.Embedding(config.num_position_embeddings, config.hidden_size)
+        self.num_grid_per_side = int(config.num_position_embeddings**0.5)
+        self.rotary_pos_emb = AgnesVisionRotary((config.hidden_size // config.num_heads) // 2)
+        self.blocks = nn.ModuleList([AgnesVisionBlock(config) for _ in range(config.depth)])
+        self.merger = AgnesVisionMerger(config=config, use_postshuffle_norm=False)
+        self.gradient_checkpointing = False
+        self.post_init()
+
+    @merge_with_config_defaults
+    @capture_outputs
+    def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor:
+        """
+        Args:
+            hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`):
+                Flattened patches.
+            grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`):
+                Temporal / height / width extent of every image or video.
+        """
+        idx, w = get_vision_bilinear_indices_and_weights(
+            grid_thw, num_grid_per_side=self.num_grid_per_side, spatial_merge_size=self.config.spatial_merge_size, kwargs=kwargs
+        )
+        pos_ids = get_vision_position_ids(grid_thw, self.spatial_merge_size, kwargs=kwargs)
+        cu_seqlens = get_vision_cu_seqlens(grid_thw, kwargs=kwargs)
+
+        tokens = self.patch_embed(hidden_states)
+        tokens = tokens + (self.pos_embed(idx) * w[:, :, None]).sum(0).to(tokens.dtype)
+        angles = self.rotary_pos_emb(pos_ids)
+
+        n_tok, _ = tokens.size()
+        tokens = tokens.reshape(n_tok, -1)
+        angles = angles.reshape(n_tok, -1)
+        table = torch.cat((angles, angles), dim=-1)
+        rope = (table.cos(), table.sin())
+
+        for block in self.blocks:
+            tokens = block(tokens, cu_seqlens=cu_seqlens, position_embeddings=rope, **kwargs)
+
+        return BaseModelOutputWithPooling(last_hidden_state=tokens, pooler_output=self.merger(tokens))
+
+
+# ---------------------------------------------------------------------------
+# Outputs
+# ---------------------------------------------------------------------------
+
+
+@auto_docstring
+@dataclass
+class AgnesModelOutput(BaseModelOutputWithPast):
+    r"""
+    rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*):
+        Offset between the multimodal rotary positions and plain sequence positions.
+    """
+
+    rope_deltas: torch.LongTensor | None = None
+
+
+@auto_docstring
+@dataclass
+class AgnesCausalLMOutput(CausalLMOutputWithPast):
+    r"""
+    rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*):
+        Offset between the multimodal rotary positions and plain sequence positions.
+    """
+
+    rope_deltas: torch.LongTensor | None = None
+
+
+# ---------------------------------------------------------------------------
+# Language model
+# ---------------------------------------------------------------------------
+
+
+class AgnesTextModel(AgnesPreTrainedModel):
+    config: AgnesTextConfig
+    config_class = AgnesTextConfig
+
+    def __init__(self, config: AgnesTextConfig):
+        super().__init__(config)
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
+        self.layers = nn.ModuleList([AgnesDecoderLayer(config, i) for i in range(config.num_hidden_layers)])
+        self.norm = AgnesRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.rotary_emb = AgnesRotaryEmbedding(config=config)
+        self.gradient_checkpointing = False
+        self.post_init()
+
+    @merge_with_config_defaults
+    @capture_outputs
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        use_cache: bool | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> BaseModelOutputWithPast:
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache(config=self.config)
+
+        # position ids carry four rows: plain text positions, then T / H / W
+        if position_ids is None:
+            offset = past_key_values.get_seq_length() if past_key_values is not None else 0
+            position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + offset
+            position_ids = position_ids.view(1, 1, -1).expand(4, inputs_embeds.shape[0], -1)
+        elif position_ids.ndim == 2:
+            position_ids = position_ids[None, ...].expand(4, position_ids.shape[0], -1)
+
+        if position_ids.ndim == 3 and position_ids.shape[0] == 4:
+            text_positions, position_ids = position_ids[0], position_ids[1:]
+        else:
+            text_positions = None
+
+        global_mask = create_causal_mask(
+            config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask,
+            past_key_values=past_key_values, position_ids=text_positions,
+        )
+        delta_mask = self._recurrent_mask(attention_mask, past_key_values)
+
+        h = inputs_embeds
+        rope = self.rotary_emb(h, position_ids)
+        for i, layer in enumerate(self.layers[: self.config.num_hidden_layers]):
+            mask = delta_mask if self.config.layer_types[i] == LAYER_DELTA else global_mask
+            h = layer(
+                h, position_embeddings=rope, attention_mask=mask, position_ids=text_positions,
+                past_key_values=past_key_values, use_cache=use_cache, **kwargs,
+            )
+        h = self.norm(h)
+        return AgnesModelOutput(last_hidden_state=h, past_key_values=past_key_values)
+
+    @staticmethod
+    def _recurrent_mask(attention_mask, past_key_values):
+        """Padding mask for the recurrent layers (left padding).  Not needed
+        once a cache holds prior state, or when nothing is masked."""
+        if past_key_values is not None and past_key_values.has_previous_state():
+            return None
+        if attention_mask is not None and torch.all(attention_mask == 1):
+            return None
+        return attention_mask
+
+
+@auto_docstring
+class AgnesModel(AgnesPreTrainedModel):
+    config: AgnesConfig
+    config_class = AgnesConfig
+    base_model_prefix = "model"
+    accepts_loss_kwargs = False
+    _no_split_modules = ["AgnesDecoderLayer", "AgnesVisionBlock"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.visual = AgnesVisionModel(config.vision_config)
+        self.language_model = AgnesTextModel(config.text_config)
+        self.rope_deltas = None
+        self.post_init()
+
+    def get_vision_position_ids(
+        self,
+        start_position: int,
+        grid_thw: list[int, int, int] | torch.Tensor,
+        temp_merge_size: int = 1,
+        spatial_merge_size: int = 1,
+        time_interval: int = 1,
+        device: str | torch.device | None = None,
+    ):
+        """Three-axis positions for the tokens of one image / video, offset by `start_position`."""
+        n_t = grid_thw[0].item() // temp_merge_size
+        n_h = grid_thw[1].item() // spatial_merge_size
+        n_w = grid_thw[2].item() // spatial_merge_size
+
+        t_axis = torch.arange(n_t, device=device) * time_interval
+        w_axis = torch.arange(n_w, device=device) + start_position
+        h_axis = torch.arange(n_h, device=device) + start_position
+
+        # repeat patterns define the raster order; keep them as is
+        w_axis = w_axis.repeat(n_h * n_t)
+        h_axis = h_axis.repeat_interleave(n_w).repeat(n_t)
+        t_axis = t_axis.repeat_interleave(n_h * n_w) + start_position
+        return torch.stack([t_axis, h_axis, w_axis], dim=0)
+
+    def get_rope_index(
+        self,
+        input_ids: torch.LongTensor,
+        mm_token_type_ids: torch.IntTensor,
+        image_grid_thw: torch.LongTensor | None = None,
+        video_grid_thw: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        **kwargs,
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        """Build (3, batch, seq) rotary positions: text runs advance all three
+        axes together, vision runs get grid positions.  Videos are split per
+        frame because frames are separated by timestamp tokens."""
+        if video_grid_thw is not None:
+            video_grid_thw = torch.repeat_interleave(video_grid_thw, video_grid_thw[:, 0], dim=0)
+            video_grid_thw[:, 0] = 1
+        merge = self.config.vision_config.spatial_merge_size
+
+        deltas = []
+        position_ids = torch.zeros(3, input_ids.shape[0], input_ids.shape[1], dtype=input_ids.dtype, device=input_ids.device)
+        grids = {
+            1: iter(image_grid_thw) if image_grid_thw is not None else None,
+            2: iter(video_grid_thw) if video_grid_thw is not None else None,
+        }
+
+        for b, ids in enumerate(input_ids):
+            kinds = mm_token_type_ids[b]
+            if attention_mask is not None:
+                keep = attention_mask[b].bool()
+                ids, kinds = ids[keep], kinds[keep]
+
+            runs = []
+            for kind, members in itertools.groupby(enumerate(kinds.tolist()), lambda x: x[1]):
+                members = list(members)
+                runs.append((kind, members[0][0], members[-1][0] + 1))
+
+            cursor = 0
+            pieces = []
+            for kind, lo, hi in runs:
+                if kind == 0:
+                    n = hi - lo
+                    pieces.append(torch.arange(n, device=input_ids.device).view(1, -1).expand(3, -1) + cursor)
+                    cursor += n
+                else:
+                    thw = next(grids[kind])
+                    pieces.append(self.get_vision_position_ids(cursor, thw, 1, merge, device=input_ids.device))
+                    cursor += max(thw[1], thw[2]) // merge
+            pos = torch.cat(pieces, dim=1).reshape(3, -1)
+            if attention_mask is not None:
+                position_ids[:, b, attention_mask[b].bool()] = pos.to(position_ids.device)
+            else:
+                position_ids[:, b] = pos.to(position_ids.device)
+            deltas.append(pos.max() + 1 - len(ids))
+        return position_ids, torch.tensor(deltas, device=input_ids.device).unsqueeze(1)
+
+    @accepts_precomputed_kwargs(modality="video")
+    @can_return_tuple
+    @auto_docstring
+    def get_video_features(
+        self,
+        pixel_values_videos: torch.FloatTensor,
+        video_grid_thw: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutputWithPooling:
+        r"""
+        pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
+            Video frames as patches.
+        video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
+            Temporal / height / width extent of every video.
+        """
+        return self.get_image_features(pixel_values_videos, video_grid_thw, **kwargs)
+
+    @accepts_precomputed_kwargs(modality="image")
+    @can_return_tuple
+    @auto_docstring
+    def get_image_features(
+        self,
+        pixel_values: torch.FloatTensor,
+        image_grid_thw: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutputWithPooling:
+        r"""
+        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
+            Images as patches.
+        image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
+            Temporal / height / width extent of every image.
+        """
+        pixel_values = pixel_values.type(self.visual.dtype)
+        vis: BaseModelOutputWithPooling = self.visual(pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs)
+        per_image = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist()
+        vis.pooler_output = torch.split(vis.pooler_output, per_image)
+        return vis
+
+    def get_placeholder_mask(
+        self,
+        input_ids: torch.LongTensor,
+        inputs_embeds: torch.FloatTensor,
+        image_features: torch.FloatTensor | None = None,
+        video_features: torch.FloatTensor | None = None,
+    ):
+        """Boolean masks over the image / video placeholder tokens, checked
+        against the number of visual features supplied."""
+        if input_ids is None:
+            embed = self.get_input_embeddings()
+            dev = inputs_embeds.device
+            image_mask = (inputs_embeds == embed(torch.tensor(self.config.image_token_id, dtype=torch.long, device=dev))).all(-1)
+            video_mask = (inputs_embeds == embed(torch.tensor(self.config.video_token_id, dtype=torch.long, device=dev))).all(-1)
+        else:
+            image_mask = input_ids == self.config.image_token_id
+            video_mask = input_ids == self.config.video_token_id
+
+        n_img = image_mask.sum()
+        image_mask = image_mask.unsqueeze(-1).to(inputs_embeds.device)
+        if image_features is not None:
+            torch_compilable_check(
+                n_img * inputs_embeds.shape[-1] == image_features.numel(),
+                f"Image features and image tokens do not match, tokens: {n_img}, features: {image_features.shape[0]}",
+            )
+        n_vid = video_mask.sum()
+        video_mask = video_mask.unsqueeze(-1).to(inputs_embeds.device)
+        if video_features is not None:
+            torch_compilable_check(
+                n_vid * inputs_embeds.shape[-1] == video_features.numel(),
+                f"Video features and video tokens do not match, tokens: {n_vid}, features: {video_features.shape[0]}",
+            )
+        return image_mask, video_mask
+
+    def compute_3d_position_ids(
+        self,
+        input_ids: torch.Tensor | None,
+        inputs_embeds: torch.Tensor | None,
+        image_grid_thw: torch.Tensor | None = None,
+        video_grid_thw: torch.Tensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        past_key_values: torch.Tensor | None = None,
+        mm_token_type_ids: torch.IntTensor | None = None,
+    ) -> torch.Tensor | None:
+        past_len = 0 if past_key_values is None else past_key_values.get_seq_length()
+        has_vision = image_grid_thw is not None or video_grid_thw is not None
+        if has_vision and mm_token_type_ids is None and input_ids is not None:
+            raise ValueError(
+                "Multimodal data was passed (via `image_grid_thw` or `video_grid_thw`) but `mm_token_type_ids` is "
+                "missing. Pass `mm_token_type_ids` (returned by the processor) so the 3-D rotary positions can be built."
+            )
+        fresh = input_ids is not None and mm_token_type_ids is not None and has_vision
+
+        if fresh and (self.rope_deltas is None or past_len == 0):
+            position_ids, self.rope_deltas = self.get_rope_index(
+                input_ids, image_grid_thw=image_grid_thw, video_grid_thw=video_grid_thw,
+                attention_mask=attention_mask, mm_token_type_ids=mm_token_type_ids,
+            )
+            return position_ids
+        # continuing generation, or embeds-only input: reuse the stored deltas
+        if self.rope_deltas is not None and (past_len > 0 or input_ids is None):
+            bsz, seq, _ = inputs_embeds.shape
+            if attention_mask is not None:
+                position_ids = attention_mask.long().cumsum(-1) - 1
+                position_ids = position_ids.masked_fill(attention_mask == 0, 0)
+                position_ids = position_ids.view(1, bsz, -1).repeat(3, 1, 1).to(inputs_embeds.device)
+            else:
+                position_ids = torch.arange(past_len, past_len + seq)
+                position_ids = position_ids.view(1, 1, -1).expand(3, bsz, -1).to(inputs_embeds.device)
+            delta = self.rope_deltas.repeat_interleave(bsz // self.rope_deltas.shape[0], dim=0)
+            return position_ids + delta.to(device=inputs_embeds.device)
+        return None
+
+    @auto_docstring
+    @can_return_tuple
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        pixel_values: torch.Tensor | None = None,
+        pixel_values_videos: torch.FloatTensor | None = None,
+        image_grid_thw: torch.LongTensor | None = None,
+        video_grid_thw: torch.LongTensor | None = None,
+        mm_token_type_ids: torch.IntTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | AgnesModelOutput:
+        r"""
+        image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
+            Temporal / height / width extent of every image.
+        video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
+            Temporal / height / width extent of every video.
+        """
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+        if inputs_embeds is None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+        if pixel_values is not None:
+            feats = torch.cat(
+                self.get_image_features(pixel_values, image_grid_thw, return_dict=True, **kwargs).pooler_output, dim=0
+            ).to(inputs_embeds.device, inputs_embeds.dtype)
+            mask, _ = self.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds, image_features=feats)
+            inputs_embeds = inputs_embeds.masked_scatter(mask, feats)
+
+        if pixel_values_videos is not None:
+            feats = torch.cat(
+                self.get_video_features(pixel_values_videos, video_grid_thw, return_dict=True, **kwargs).pooler_output, dim=0
+            ).to(inputs_embeds.device, inputs_embeds.dtype)
+            _, mask = self.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds, video_features=feats)
+            inputs_embeds = inputs_embeds.masked_scatter(mask, feats)
+
+        if position_ids is None:
+            position_ids = self.compute_3d_position_ids(
+                input_ids=input_ids, image_grid_thw=image_grid_thw, video_grid_thw=video_grid_thw,
+                inputs_embeds=inputs_embeds, attention_mask=attention_mask, past_key_values=past_key_values,
+                mm_token_type_ids=mm_token_type_ids,
+            )
+
+        out = self.language_model(
+            input_ids=None, position_ids=position_ids, attention_mask=attention_mask,
+            past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs,
+        )
+        return AgnesModelOutput(**out, rope_deltas=self.rope_deltas)
+
+
+@auto_docstring
+class AgnesForCausalLM(AgnesPreTrainedModel, GenerationMixin):
+    """Text-only head over `AgnesTextModel`."""
+
+    _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
+    _tp_plan = {"lm_head": "colwise_gather_output"}
+    _pp_plan = {"lm_head": (["hidden_states"], ["logits"])}
+    config: AgnesTextConfig
+    config_class = AgnesTextConfig
+    _keys_to_ignore_on_load_unexpected = [r"^mtp.*", r"^model.visual.*"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = AgnesTextModel(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        use_cache: bool | None = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> CausalLMOutputWithPast:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Targets for the language-modelling loss; -100 marks ignored positions.
+        """
+        out: BaseModelOutputWithPast = self.model(
+            input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids,
+            past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, **kwargs,
+        )
+        keep = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(out.last_hidden_state[:, keep, :])
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
+        return CausalLMOutputWithPast(
+            loss=loss, logits=logits, past_key_values=out.past_key_values,
+            hidden_states=out.hidden_states, attentions=out.attentions,
+        )
+
+
+class AgnesForConditionalGeneration(AgnesPreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {"lm_head.weight": "model.language_model.embed_tokens.weight"}
+    accepts_loss_kwargs = False
+    config: AgnesConfig
+    config_class = AgnesConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = AgnesModel(config)
+        self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False)
+        self.post_init()
+
+    @auto_docstring
+    def get_video_features(
+        self,
+        pixel_values_videos: torch.FloatTensor,
+        video_grid_thw: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutputWithPooling:
+        r"""
+        pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
+            Video frames as patches.
+        video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
+            Temporal / height / width extent of every video.
+        """
+        return self.model.get_video_features(pixel_values_videos, video_grid_thw, **kwargs)
+
+    @auto_docstring
+    def get_image_features(
+        self,
+        pixel_values: torch.FloatTensor,
+        image_grid_thw: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutputWithPooling:
+        r"""
+        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
+            Images as patches.
+        image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
+            Temporal / height / width extent of every image.
+        """
+        return self.model.get_image_features(pixel_values, image_grid_thw, **kwargs)
+
+    @can_return_tuple
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        pixel_values: torch.Tensor | None = None,
+        pixel_values_videos: torch.FloatTensor | None = None,
+        image_grid_thw: torch.LongTensor | None = None,
+        video_grid_thw: torch.LongTensor | None = None,
+        mm_token_type_ids: torch.IntTensor | None = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | AgnesCausalLMOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Targets for the language-modelling loss; -100 marks ignored positions.
+        image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
+            Temporal / height / width extent of every image.
+        video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
+            Temporal / height / width extent of every video.
+        """
+        out = self.model(
+            input_ids=input_ids, pixel_values=pixel_values, pixel_values_videos=pixel_values_videos,
+            image_grid_thw=image_grid_thw, video_grid_thw=video_grid_thw, position_ids=position_ids,
+            attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds,
+            mm_token_type_ids=mm_token_type_ids, **kwargs,
+        )
+        keep = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(out[0][:, keep, :])
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size)
+        return AgnesCausalLMOutput(
+            loss=loss, logits=logits, past_key_values=out.past_key_values,
+            hidden_states=out.hidden_states, attentions=out.attentions, rope_deltas=out.rope_deltas,
+        )
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        position_ids=None,
+        use_cache=True,
+        pixel_values=None,
+        pixel_values_videos=None,
+        image_grid_thw=None,
+        video_grid_thw=None,
+        is_first_iteration=False,
+        **kwargs,
+    ):
+        # pixels are consumed on the first step only; later steps run on the cache
+        inputs = super().prepare_inputs_for_generation(
+            input_ids, past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds,
+            position_ids=position_ids, pixel_values=pixel_values, pixel_values_videos=pixel_values_videos,
+            image_grid_thw=image_grid_thw, video_grid_thw=video_grid_thw, use_cache=use_cache,
+            is_first_iteration=is_first_iteration, **kwargs,
+        )
+        if not is_first_iteration and use_cache:
+            inputs["pixel_values"] = None
+            inputs["pixel_values_videos"] = None
+        return inputs
+
+    def _prepare_position_ids_for_generation(self, inputs_tensor, model_kwargs):
+        # four-row positions: text row on top of the three rotary axes
+        text_pos = super()._prepare_position_ids_for_generation(inputs_tensor, model_kwargs)
+
+        past_len = 0
+        if (cache := model_kwargs.get("past_key_values")) is not None:
+            past_len = cache.get_seq_length()
+        if past_len != 0 and self.model.rope_deltas is not None:
+            return text_pos[None, ...] + self.model.rope_deltas
+
+        if "input_ids" in model_kwargs and model_kwargs["input_ids"].shape[1] > 0:
+            inputs_tensor = model_kwargs["input_ids"]
+        is_ids = len(inputs_tensor.shape) == 2 and inputs_tensor.dtype in [torch.int, torch.long]
+        has_vision = model_kwargs.get("image_grid_thw") is not None or model_kwargs.get("video_grid_thw") is not None
+        if is_ids and model_kwargs.get("mm_token_type_ids") is not None and has_vision:
+            rest = {k: v for k, v in model_kwargs.items() if k != "input_ids"}
+            axes, self.model.rope_deltas = self.model.get_rope_index(inputs_tensor, **rest)
+        else:
+            axes = text_pos.unsqueeze(0).expand(3, -1, -1)
+            self.model.rope_deltas = torch.zeros(inputs_tensor.shape[0], 1, dtype=torch.long, device=inputs_tensor.device)
+        return torch.cat([text_pos[None, ...], axes], dim=0)
+
+    def _count_images_and_videos(
+        self,
+        input_ids: torch.LongTensor | None,
+        inputs_embeds: torch.Tensor | None = None,
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        """Per-sample number of images and of video frames, read off the
+        vision-start / placeholder tokens."""
+        img_id = self.config.image_token_id
+        vid_id = self.config.video_token_id
+        start_id = self.config.vision_start_token_id
+        if inputs_embeds is not None:
+            embed = self.get_input_embeddings()
+            dev = inputs_embeds.device
+            start = (inputs_embeds == embed(torch.tensor(start_id, dtype=torch.long, device=dev)))[..., 0]
+            img = (inputs_embeds == embed(torch.tensor(img_id, dtype=torch.long, device=dev)))[..., 0]
+            vid = (inputs_embeds == embed(torch.tensor(vid_id, dtype=torch.long, device=dev)))[..., 0]
+        else:
+            start = input_ids == start_id
+            img = input_ids == img_id
+            vid = input_ids == vid_id
+        after_start = torch.roll(start, shifts=1, dims=1)
+        return torch.sum(after_start & img, dim=1), torch.sum(after_start & vid, dim=1)
+
+    def _expand_inputs_for_generation(
+        self,
+        expand_size: int = 1,
+        is_encoder_decoder: bool = False,
+        input_ids: torch.LongTensor | None = None,
+        **model_kwargs,
+    ) -> tuple[torch.LongTensor, dict[str, Any]]:
+        # visual tensors have no batch axis (they are concatenated over samples),
+        # so they are expanded sample by sample using the per-sample counts
+        if expand_size == 1:
+            return input_ids, model_kwargs
+
+        visual_keys = ["pixel_values", "image_grid_thw", "pixel_values_videos", "video_grid_thw"]
+
+        def expand_visual(d):
+            image_grid_thw = model_kwargs.get("image_grid_thw", None)
+            video_grid_thw = model_kwargs.get("video_grid_thw", None)
+            n_img, n_vid = self._count_images_and_videos(input_ids, inputs_embeds=model_kwargs.get("inputs_embeds", None))
+
+            # n_vid counts frames (each frame carries a vision-start token); fold back to videos
+            if video_grid_thw is not None:
+                frame_cum = torch.cumsum(video_grid_thw[:, 0], dim=0)
+                token_cum = torch.cumsum(n_vid, dim=0)
+                edges = torch.searchsorted(frame_cum, token_cum)
+                n_vid = torch.diff(torch.cat([-edges.new_ones(1), edges]))
+
+            def tile(x, lengths, times):
+                parts = torch.split(x, lengths)
+                reps = [times] + [1] * (x.dim() - 1)
+                return torch.cat([p.repeat(*reps) for p in parts], dim=0)
+
+            for key in d:
+                if key == "pixel_values":
+                    per = torch.split(image_grid_thw, list(n_img))
+                    d[key] = tile(d[key], [torch.prod(s, dim=1).sum() for s in per], expand_size)
+                elif key == "image_grid_thw":
+                    d[key] = tile(d[key], list(n_img), expand_size)
+                elif key == "pixel_values_videos":
+                    per = torch.split(video_grid_thw, list(n_vid))
+                    d[key] = tile(d[key], [torch.prod(s, dim=1).sum() for s in per], expand_size)
+                elif key == "video_grid_thw":
+                    d[key] = tile(d[key], list(n_vid), expand_size)
+            return d
+
+        def expand_rest(d):
+            for key in d:
+                if key == "position_ids" and d[key].ndim == 3:
+                    d[key] = d[key].repeat_interleave(expand_size, dim=1)
+                elif d[key] is not None and isinstance(d[key], torch.Tensor) and key not in visual_keys:
+                    d[key] = d[key].repeat_interleave(expand_size, dim=0)
+            return d
+
+        model_kwargs = expand_visual(model_kwargs)
+        if input_ids is not None:
+            input_ids = input_ids.repeat_interleave(expand_size, dim=0)
+        model_kwargs = expand_rest(model_kwargs)
+        if is_encoder_decoder:
+            if model_kwargs.get("encoder_outputs") is None:
+                raise ValueError("If `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined.")
+            model_kwargs["encoder_outputs"] = expand_rest(model_kwargs["encoder_outputs"])
+        return input_ids, model_kwargs
+
+
+__all__ = [
+    "AgnesPreTrainedModel",
+    "AgnesVisionModel",
+    "AgnesTextModel",
+    "AgnesModel",
+    "AgnesForCausalLM",
+    "AgnesForConditionalGeneration",
+]

preprocessor_config.json

@@ -0,0 +1,25 @@
+{
+  "processor_class": "AgnesProcessor",
+  "image_processor_type": "AgnesImageProcessor",
+  "auto_map": {
+    "AutoProcessor": "processing_agnes.AgnesProcessor",
+    "AutoImageProcessor": "image_processing_agnes.AgnesImageProcessor"
+  },
+  "size": {
+    "longest_edge": 16777216,
+    "shortest_edge": 65536
+  },
+  "patch_size": 16,
+  "temporal_patch_size": 2,
+  "merge_size": 2,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ]
+}

processing_agnes.py

@@ -0,0 +1,116 @@
+# Copyright 2026 Agnes AI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Processor for Agnes 3.0 Flash: tokenizer + image processor + video processor."""
+
+import numpy as np
+
+from transformers.processing_utils import MultiModalData, ProcessingKwargs, ProcessorMixin
+from transformers.utils import auto_docstring, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class AgnesProcessorKwargs(ProcessingKwargs, total=False):
+    _defaults = {
+        "text_kwargs": {"padding": False, "return_token_type_ids": False, "return_mm_token_type_ids": True},
+        "videos_kwargs": {"return_metadata": True},
+    }
+
+
+@auto_docstring
+class AgnesProcessor(ProcessorMixin):
+    valid_processor_kwargs = AgnesProcessorKwargs
+
+    def __init__(self, image_processor=None, tokenizer=None, video_processor=None, chat_template=None, **kwargs):
+        self.image_token = getattr(tokenizer, "image_token", "<|image_pad|>")
+        self.video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
+        self.image_token_id = getattr(tokenizer, "image_token_id", None) or tokenizer.convert_tokens_to_ids(self.image_token)
+        self.video_token_id = getattr(tokenizer, "video_token_id", None) or tokenizer.convert_tokens_to_ids(self.video_token)
+        super().__init__(image_processor, tokenizer, video_processor, chat_template=chat_template)
+        self.vision_start_token = getattr(tokenizer, "vision_start_token", "<|vision_start|>")
+        self.vision_end_token = getattr(tokenizer, "vision_end_token", "<|vision_end|>")
+        self.vision_start_token_id = getattr(tokenizer, "vision_start_token_id", None) or tokenizer.convert_tokens_to_ids(
+            self.vision_start_token
+        )
+        self.vision_end_token_id = getattr(tokenizer, "vision_end_token_id", None) or tokenizer.convert_tokens_to_ids(
+            self.vision_end_token
+        )
+
+    def replace_image_token(self, image_inputs: dict, image_idx: int) -> str:
+        per_token = self.image_processor.merge_size**2
+        n = image_inputs["image_grid_thw"][image_idx].prod() // per_token
+        return self.image_token * n
+
+    def replace_video_token(self, video_inputs: dict, video_idx: int) -> str:
+        per_token = self.video_processor.merge_size**2
+        thw = video_inputs["video_grid_thw"][video_idx]
+        n_frames = thw[0]
+        per_frame = thw[1:].prod() // per_token
+        meta = video_inputs["video_metadata"][video_idx]
+        if meta.fps is None:
+            logger.warning_once(
+                "Frame timestamps are needed to build the video prompt but the `fps` of the input video could not be "
+                "inferred (no `video_metadata`, pre-sampled frames?). Defaulting to `fps=24`."
+            )
+        meta.fps = 24 if meta.fps is None else meta.fps
+        stamps = self._frame_timestamps(meta.frames_indices, meta.fps, self.video_processor.temporal_patch_size)
+        text = ""
+        for f in range(n_frames):
+            text += f"<{stamps[f]:.1f} seconds>"
+            text += self.vision_start_token + self.video_token * per_frame + self.vision_end_token
+        return text
+
+    def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs):
+        """Placeholder counts for inputs of the given sizes, without running the
+        processors on real pixels."""
+        data = {}
+        if image_sizes is not None:
+            ik = AgnesProcessorKwargs._defaults.get("images_kwargs", {})
+            ik.update(kwargs)
+            merge = ik.get("merge_size", None) or self.image_processor.merge_size
+            patches = [self.image_processor.get_number_of_image_patches(*s, ik) for s in image_sizes]
+            data.update({"num_image_tokens": [p // merge**2 for p in patches], "num_image_patches": patches})
+        if video_sizes is not None:
+            vk = AgnesProcessorKwargs._defaults.get("videos_kwargs", {})
+            vk.update(kwargs)
+            merge = vk.get("merge_size", None) or self.video_processor.merge_size
+            patches = [self.video_processor.get_number_of_video_patches(*s, vk) for s in video_sizes]
+            data["num_video_tokens"] = [p // merge**2 for p in patches]
+        return MultiModalData(**data)
+
+    def post_process_image_text_to_text(self, generated_outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False, **kwargs):
+        """Decode generated ids to text."""
+        return self.tokenizer.batch_decode(
+            generated_outputs, skip_special_tokens=skip_special_tokens,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs,
+        )
+
+    @property
+    def model_input_names(self):
+        return super().model_input_names + ["mm_token_type_ids"]
+
+    @staticmethod
+    def _frame_timestamps(indices: list[int] | np.ndarray, video_fps: float, merge_size: int = 2):
+        """One timestamp per temporal patch: the mean of the first and last
+        frame time inside the patch."""
+        if not isinstance(indices, list):
+            indices = indices.tolist()
+        if len(indices) % merge_size != 0:
+            indices.extend(indices[-1] for _ in range(merge_size - len(indices) % merge_size))
+        times = [i / video_fps for i in indices]
+        return [(times[i] + times[i + merge_size - 1]) / 2 for i in range(0, len(times), merge_size)]
+
+
+__all__ = ["AgnesProcessor"]

serve.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Serve Agnes 3.0 Flash with a stock sglang image.
+#
+# Tested image:  lmsysorg/sglang:nightly-dev-20260908-20ca564b
+#
+#   docker run --gpus all --shm-size 64g -p 30001:30002 \
+#       -v /path/to/agnes-3.0-flash:/model \
+#       lmsysorg/sglang:nightly-dev-20260908-20ca564b \
+#       bash /model/serve.sh [extra sglang arguments, e.g. --tp 2]
+#
+# The script overlays three files from sglang_patch/ onto the image's sglang
+# package (a prebuilt variant when one matches the installed version, else
+# apply_patch.py patches the installed files in place) and starts the server
+# on port 30002 inside the container.
+set -euo pipefail
+D="$(cd "$(dirname "$0")" && pwd)"
+PKG="$(python3 -c 'import sglang, os; print(os.path.dirname(sglang.__file__))')"
+VER="$(python3 -c 'import sglang; print(getattr(sglang, "__version__", ""))' 2>/dev/null || true)"
+
+case "$VER" in
+  *20ca564b*) VARIANT="nightly-dev-20260908-20ca564b" ;;
+  0.5.19*)    VARIANT="v0.5.19" ;;
+  *)          VARIANT="" ;;
+esac
+
+if [ -n "$VARIANT" ] && [ -d "$D/sglang_patch/$VARIANT/sglang" ]; then
+  echo "[serve.sh] sglang $VER -> prebuilt patch $VARIANT"
+  cp -r "$D/sglang_patch/$VARIANT/sglang/." "$PKG/"
+else
+  echo "[serve.sh] sglang $VER -> patching installed package in place"
+  echo "This checkpoint requires the documented pinned SGLang image." >&2
+  exit 1
+fi
+
+export AGNES_MODEL_PATH="$D"
+exec python3 -m sglang.launch_server --model-path "$D" --trust-remote-code --host 0.0.0.0 --port 8080 "$@"

sglang_patch/agnes_sglang_config.py

@@ -0,0 +1,57 @@
+# Agnes 3.0 Flash configuration for sglang.
+#
+# The HF checkpoint says model_type "agnes", names its layer types
+# agnes_delta_attention / agnes_global_attention and carries a parallel FFN
+# branch per layer.  The server runs it on its built-in hybrid
+# (delta-rule + global attention) implementation, so this class maps those
+# onto the fields that implementation reads; the checkpoint's tensor names
+# are translated while loading (see the model file patched by apply_patch.py).
+from sglang.srt.configs.qwen3_5 import Qwen3_5Config
+
+AGNES_DELTA = "agnes_delta_attention"
+AGNES_GLOBAL = "agnes_global_attention"
+
+
+class AgnesConfig(Qwen3_5Config):
+    model_type = "agnes"
+
+    def __init__(self, text_config=None, vision_config=None, **kwargs):
+        kwargs.pop("auto_map", None)  # the transformers remote code is not used in the server
+        if isinstance(text_config, dict):
+            text_config = dict(text_config)
+            text_config["model_type"] = "qwen3_5_text"
+            width = int(text_config.pop("parallel_ffn_intermediate_size", 0) or 0)
+            plan = text_config.pop("layer_types", None)
+            interval = text_config.pop("global_attention_interval", None)
+            if interval is None:
+                interval = text_config.pop("full_attention_interval", None)
+            if interval is None and plan:
+                interval = next(i + 1 for i, t in enumerate(plan) if t == AGNES_GLOBAL)
+            text_config["full_attention_interval"] = int(interval or 4)
+            main = int(text_config["intermediate_size"])
+            # the parallel branch is folded into the main MLP at load time
+            text_config["intermediate_size"] = main + width
+            text_config["agnes_main_intermediate_size"] = main
+            text_config["agnes_parallel_ffn_intermediate_size"] = width
+        if isinstance(vision_config, dict):
+            vision_config = dict(vision_config)
+            vision_config["model_type"] = "qwen3_5"
+        kwargs["architectures"] = ["Qwen3_5ForConditionalGeneration"]
+        super().__init__(text_config=text_config, vision_config=vision_config, **kwargs)
+        # every downstream check sees the built-in hybrid architecture
+        self.model_type = "qwen3_5"
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
+        # The weight loader needs the checkpoint directory to pick up the parallel
+        # branch tensors.  The path is written into the config *dict* before the
+        # object is built: the config reaches the worker processes through a
+        # to_dict round trip, which keeps fields that came in through __init__
+        # and drops attributes set afterwards (from_pretrained's own kwargs only
+        # override known fields, so they cannot carry it either).
+        path = str(pretrained_model_name_or_path)
+        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+        config_dict["agnes_model_path"] = path
+        if isinstance(config_dict.get("text_config"), dict):
+            config_dict["text_config"]["agnes_model_path"] = path
+        return cls.from_dict(config_dict, **kwargs)

sglang_patch/apply_patch.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Apply the Agnes patch to an sglang python package directory (the one that
+contains `srt/`).  Idempotent.
+
+  1. srt/configs/agnes.py                    new file
+  2. srt/utils/hf_transformers/common.py     register AgnesConfig in _CONFIG_REGISTRY
+  3. srt/models/qwen3_5.py                   load_weights: translate the Agnes checkpoint
+                                              (tensor prefixes, parallel-FFN fold)
+
+Usage: apply_patch.py <path/to/sglang>            e.g. .../site-packages/sglang
+"""
+
+import os
+import shutil
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+MARK = "# === agnes ==="
+
+TRANSLATE = '''
+# === agnes ===
+# Agnes 3.0 Flash checkpoints (config model_type "agnes") use their own tensor
+# prefixes and carry a parallel FFN branch per layer.  This generator sits at
+# the top of the weight stream and turns it into what the implementation below
+# expects: delta_attn -> linear_attn, global_attn -> self_attn, and the branch
+# concatenated onto the main gate / up (dim 0) and down (dim 1) projections,
+# matching the widened intermediate_size set by sglang.srt.configs.agnes.
+import json as _agnes_json
+import os as _agnes_os
+import re as _agnes_re
+
+_AGNES_MLP_RE = _agnes_re.compile(r"^(.*\\.layers\\.\\d+\\.mlp\\.)(gate_proj|up_proj|down_proj)\\.weight$")
+
+
+class _AgnesBranchReader:
+    def __init__(self, model_path):
+        from safetensors import safe_open
+
+        self._open = safe_open
+        self.path = model_path
+        index = _agnes_os.path.join(model_path, "model.safetensors.index.json")
+        self.weight_map = _agnes_json.load(open(index))["weight_map"]
+        self.handles = {}
+
+    def get(self, key):
+        fn = self.weight_map[key]
+        if fn not in self.handles:
+            self.handles[fn] = self._open(_agnes_os.path.join(self.path, fn), framework="pt", device="cpu")
+        return self.handles[fn].get_tensor(key)
+
+
+def _agnes_translate_weights(model, weights):
+    cfg = getattr(model.config, "text_config", None) or model.config
+    width = int(getattr(cfg, "agnes_parallel_ffn_intermediate_size", 0) or 0)
+    if width <= 0:
+        yield from weights
+        return
+    model_path = (
+        getattr(cfg, "agnes_model_path", None)
+        or getattr(model.config, "agnes_model_path", None)
+        or _agnes_os.environ.get("AGNES_MODEL_PATH")
+        or getattr(model.config, "_name_or_path", None)
+    )
+    if not model_path or not _agnes_os.path.isdir(model_path):
+        raise RuntimeError(
+            f"agnes: cannot locate the checkpoint directory (got {model_path!r}); "
+            "set AGNES_MODEL_PATH to the model directory"
+        )
+    reader = _AgnesBranchReader(model_path)
+    for name, w in weights:
+        if ".mlp.parallel_ffn." in name:
+            continue
+        m = _AGNES_MLP_RE.match(name)
+        if m and "visual" not in name and not name.startswith("mtp"):
+            extra = reader.get(f"{m.group(1)}parallel_ffn.{m.group(2)}.weight")
+            dim = 1 if m.group(2) == "down_proj" else 0
+            w = torch.cat([w, extra.to(device=w.device, dtype=w.dtype)], dim=dim)
+        name = name.replace(".delta_attn.", ".linear_attn.").replace(".global_attn.", ".self_attn.")
+        yield name, w
+# === /agnes ===
+
+'''
+
+REGISTER = '''
+# === agnes ===
+from sglang.srt.configs.agnes import AgnesConfig as _AgnesConfig
+
+_CONFIG_REGISTRY[_AgnesConfig.model_type] = _AgnesConfig
+'''
+
+
+def patch_file(path, edit):
+    src = open(path, encoding="utf-8").read()
+    if MARK in src:
+        return "already patched"
+    out = edit(src)
+    if out is None:
+        raise SystemExit(f"anchor not found in {path}")
+    open(path, "w", encoding="utf-8").write(out)
+    return "patched"
+
+
+def edit_model_file(src):
+    anchor = "QWEN3_5_KV_SCALE_MAPPER = WeightsMapper("
+    hook = "        weights = QWEN3_5_KV_SCALE_MAPPER.apply(weights)\n"
+    if anchor not in src or src.count(hook) < 1:
+        return None
+    src = src.replace(anchor, TRANSLATE + anchor, 1)
+    src = src.replace(hook, "        weights = _agnes_translate_weights(self, weights)\n" + hook)
+    return src
+
+
+def main():
+    if len(sys.argv) != 2:
+        sys.exit(__doc__)
+    pkg = os.path.abspath(sys.argv[1])
+    srt = os.path.join(pkg, "srt")
+    if not os.path.isdir(srt):
+        sys.exit(f"{pkg} does not contain srt/")
+    dst = os.path.join(srt, "configs", "agnes.py")
+    shutil.copy2(os.path.join(HERE, "agnes_sglang_config.py"), dst)
+    print(f"configs/agnes.py: installed")
+    print("utils/hf_transformers/common.py:", patch_file(
+        os.path.join(srt, "utils", "hf_transformers", "common.py"), lambda s: s.rstrip("\n") + "\n" + REGISTER))
+    print("models/qwen3_5.py:", patch_file(os.path.join(srt, "models", "qwen3_5.py"), edit_model_file))
+    print("APPLY_PATCH_OK")
+
+
+if __name__ == "__main__":
+    main()

sglang_patch/nightly-dev-20260908-20ca564b/sglang/srt/configs/agnes.py

@@ -0,0 +1,57 @@
+# Agnes 3.0 Flash configuration for sglang.
+#
+# The HF checkpoint says model_type "agnes", names its layer types
+# agnes_delta_attention / agnes_global_attention and carries a parallel FFN
+# branch per layer.  The server runs it on its built-in hybrid
+# (delta-rule + global attention) implementation, so this class maps those
+# onto the fields that implementation reads; the checkpoint's tensor names
+# are translated while loading (see the model file patched by apply_patch.py).
+from sglang.srt.configs.qwen3_5 import Qwen3_5Config
+
+AGNES_DELTA = "agnes_delta_attention"
+AGNES_GLOBAL = "agnes_global_attention"
+
+
+class AgnesConfig(Qwen3_5Config):
+    model_type = "agnes"
+
+    def __init__(self, text_config=None, vision_config=None, **kwargs):
+        kwargs.pop("auto_map", None)  # the transformers remote code is not used in the server
+        if isinstance(text_config, dict):
+            text_config = dict(text_config)
+            text_config["model_type"] = "qwen3_5_text"
+            width = int(text_config.pop("parallel_ffn_intermediate_size", 0) or 0)
+            plan = text_config.pop("layer_types", None)
+            interval = text_config.pop("global_attention_interval", None)
+            if interval is None:
+                interval = text_config.pop("full_attention_interval", None)
+            if interval is None and plan:
+                interval = next(i + 1 for i, t in enumerate(plan) if t == AGNES_GLOBAL)
+            text_config["full_attention_interval"] = int(interval or 4)
+            main = int(text_config["intermediate_size"])
+            # the parallel branch is folded into the main MLP at load time
+            text_config["intermediate_size"] = main + width
+            text_config["agnes_main_intermediate_size"] = main
+            text_config["agnes_parallel_ffn_intermediate_size"] = width
+        if isinstance(vision_config, dict):
+            vision_config = dict(vision_config)
+            vision_config["model_type"] = "qwen3_5"
+        kwargs["architectures"] = ["Qwen3_5ForConditionalGeneration"]
+        super().__init__(text_config=text_config, vision_config=vision_config, **kwargs)
+        # every downstream check sees the built-in hybrid architecture
+        self.model_type = "qwen3_5"
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
+        # The weight loader needs the checkpoint directory to pick up the parallel
+        # branch tensors.  The path is written into the config *dict* before the
+        # object is built: the config reaches the worker processes through a
+        # to_dict round trip, which keeps fields that came in through __init__
+        # and drops attributes set afterwards (from_pretrained's own kwargs only
+        # override known fields, so they cannot carry it either).
+        path = str(pretrained_model_name_or_path)
+        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+        config_dict["agnes_model_path"] = path
+        if isinstance(config_dict.get("text_config"), dict):
+            config_dict["text_config"]["agnes_model_path"] = path
+        return cls.from_dict(config_dict, **kwargs)

sglang_patch/nightly-dev-20260908-20ca564b/sglang/srt/utils/hf_transformers/common.py

@@ -0,0 +1,729 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Shared helpers used by config, tokenizer, and processor modules."""
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Type, Union
+
+import torch
+from huggingface_hub import snapshot_download
+
+from sglang.srt.configs import (
+    AfmoeConfig,
+    BailingHybridConfig,
+    ChatGLMConfig,
+    Cosmos3Config,
+    Cosmos3EdgeConfig,
+    Cosmos3EdgeProjectorConfig,
+    Cosmos3EdgeTextConfig,
+    Cosmos3EdgeVisionConfig,
+    DbrxConfig,
+    DeepseekVL2Config,
+    Dots3Config,
+    DotsOCRConfig,
+    DotsVLMConfig,
+    ExaoneConfig,
+    FalconH1Config,
+    Glm5NextConfig,
+    Glm5NextTextConfig,
+    GraniteMoeHybridConfig,
+    HYV4Config,
+    InklingAudioConfig,
+    InklingMMConfig,
+    InklingModelConfig,
+    InklingVisionConfig,
+    InternS2MobiusConfig,
+    InternS2MobiusTextConfig,
+    InternS2PreviewConfig,
+    JetNemotronConfig,
+    JetVLMConfig,
+    K2HorizonConfig,
+    KimiK3Config,
+    KimiK25Config,
+    KimiLinearConfig,
+    KimiVLConfig,
+    LagunaConfig,
+    LocateAnythingConfig,
+    LongcatFlashConfig,
+    MiniCPMHybridConfig,
+    MiniCPMV4_6Config,
+    MiniCPMV4_6VisionConfig,
+    MiniMaxM3VLConfig,
+    MultiModalityConfig,
+    MuseGlimmerAssistantConfig,
+    MuseGlimmerConfig,
+    NanbeigeConfig,
+    NemotronH_Nano_Omni_Reasoning_V3_Config,
+    NemotronH_Nano_VL_V2_Config,
+    NemotronHConfig,
+    NemotronHPuzzleConfig,
+    Olmo3Config,
+    Qwen3_5Config,
+    Qwen3_5MoeConfig,
+    Qwen3_5MoeTextConfig,
+    Qwen3_5TextConfig,
+    Qwen3NextConfig,
+    Spark2_5Config,
+    Step3p5Config,
+    Step3p7Config,
+    Step3VLConfig,
+    XllmConfig,
+)
+from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config
+from sglang.srt.configs.internvl import InternVLChatConfig
+from sglang.srt.utils import get_bool_env_var, logger, lru_cache_frozenset
+from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri
+
+from ..hf_transformers_patches import normalize_rope_scaling_compat
+
+if get_bool_env_var("SGLANG_USE_MODELSCOPE"):
+    from modelscope import AutoConfig, GenerationConfig
+else:
+    from transformers import AutoConfig, GenerationConfig
+
+from transformers import PretrainedConfig
+
+# ---------------------------------------------------------------------------
+# Config registry
+# ---------------------------------------------------------------------------
+
+_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
+    cls.model_type: cls
+    for cls in [
+        AfmoeConfig,
+        BailingHybridConfig,
+        ChatGLMConfig,
+        DbrxConfig,
+        ExaoneConfig,
+        DeepseekVL2Config,
+        MultiModalityConfig,
+        KimiVLConfig,
+        K2HorizonConfig,
+        LocateAnythingConfig,
+        InternVLChatConfig,
+        LagunaConfig,
+        Spark2_5Config,
+        Step3VLConfig,
+        LongcatFlashConfig,
+        Olmo3Config,
+        MuseGlimmerConfig,
+        MuseGlimmerAssistantConfig,
+        KimiK3Config,
+        Glm5NextConfig,
+        Glm5NextTextConfig,
+        KimiLinearConfig,
+        Qwen3NextConfig,
+        FalconH1Config,
+        GraniteMoeHybridConfig,
+        HYV4Config,
+        DotsVLMConfig,
+        DotsOCRConfig,
+        Dots3Config,
+        NemotronH_Nano_VL_V2_Config,
+        NemotronH_Nano_Omni_Reasoning_V3_Config,
+        NemotronHConfig,
+        NemotronHPuzzleConfig,
+        NanbeigeConfig,
+        DeepseekVLV2Config,
+        Qwen3_5Config,
+        Qwen3_5MoeConfig,
+        Qwen3_5TextConfig,
+        Qwen3_5MoeTextConfig,
+        InternS2PreviewConfig,
+        InternS2MobiusConfig,
+        InternS2MobiusTextConfig,
+        JetNemotronConfig,
+        JetVLMConfig,
+        KimiK25Config,
+        Step3p5Config,
+        Step3p7Config,
+        MiniCPMHybridConfig,
+        MiniCPMV4_6Config,
+        MiniCPMV4_6VisionConfig,
+        InklingModelConfig,
+        InklingAudioConfig,
+        InklingVisionConfig,
+        InklingMMConfig,
+        MiniMaxM3VLConfig,
+        XllmConfig,
+    ]
+}
+
+
+# DeepSeek V3.2 / V4 reuse the V3 config schema. Subclass the upstream
+# transformers class with each model_type so AutoConfig.register passes its
+# consistency check (which requires class.model_type == registered key).
+# Default-value divergences (e.g. V4's topk_group) are handled in
+# model_config.py post-load.
+try:
+    from transformers import DeepseekV3Config as _HFDeepseekV3Config
+
+    class _DeepseekV32ConfigAlias(_HFDeepseekV3Config):
+        model_type = "deepseek_v32"
+
+    class _DeepseekV4ConfigAlias(_HFDeepseekV3Config):
+        model_type = "deepseek_v4"
+
+    _CONFIG_REGISTRY["deepseek_v32"] = _DeepseekV32ConfigAlias
+    _CONFIG_REGISTRY["deepseek_v4"] = _DeepseekV4ConfigAlias
+
+    # For kimi_k25_eagle3
+    class _KimiK2ConfigAlias(_HFDeepseekV3Config):
+        model_type = "kimi_k2"
+
+    _CONFIG_REGISTRY["kimi_k2"] = _KimiK2ConfigAlias
+except ImportError:
+    pass
+
+# Newer transformers versions (>=5.10.2) expose MellumConfig directly,
+# but fallback to Qwen3MoeConfig for older versions.
+try:
+    import transformers as _hf_transformers
+
+    _HFMellumConfig = getattr(_hf_transformers, "MellumConfig", None)
+
+    if _HFMellumConfig is not None:
+        _CONFIG_REGISTRY["mellum"] = _HFMellumConfig
+    else:
+        from transformers import Qwen3MoeConfig as _HFQwen3MoeConfig
+
+        class _MellumConfigAlias(_HFQwen3MoeConfig):
+            model_type = "mellum"
+
+            def __post_init__(self, **kwargs):
+                # Qwen3MoeConfig.__post_init__ wipes sliding_window unless
+                # use_sliding_window=True. Mellum gates sliding attention
+                # per-layer via layer_types, so preserve sliding_window
+                # regardless of the legacy use_sliding_window flag.
+                sliding_window = getattr(self, "sliding_window", None)
+                super().__post_init__(**kwargs)
+                self.sliding_window = sliding_window
+
+        _CONFIG_REGISTRY["mellum"] = _MellumConfigAlias
+
+except ImportError:
+    pass
+
+
+try:
+    from transformers import Gemma4Config as _HFGemma4Config
+
+    class _Gemma4UnifiedConfigAlias(_HFGemma4Config):
+        model_type = "gemma4_unified"
+
+    _CONFIG_REGISTRY["gemma4_unified"] = _Gemma4UnifiedConfigAlias
+except ImportError:
+    pass
+
+for name, cls in _CONFIG_REGISTRY.items():
+    try:
+        AutoConfig.register(name, cls)
+    except ValueError as e:
+        err = str(e).lower()
+        if "already registered" not in err and "already used" not in err:
+            logger.warning("Failed to register config %s: %s", name, e)
+
+# Cosmos3 (understanding tower) reuses the Qwen3-VL config schema. Register it
+# with AutoConfig only (not `_CONFIG_REGISTRY`), so the nested `text_config` is
+# flattened onto the top-level config in `get_config` — the same path the base
+# Qwen3-VL config relies on. Adding it to `_CONFIG_REGISTRY` would trigger a
+# `from_pretrained` reload that drops that flattening.
+try:
+    AutoConfig.register(Cosmos3Config.model_type, Cosmos3Config)
+except ValueError as e:
+    err = str(e).lower()
+    if "already registered" not in err and "already used" not in err:
+        logger.warning("Failed to register config %s: %s", Cosmos3Config.model_type, e)
+
+# Cosmos3-Edge native text support starts from the checkpoint root config, then
+# consumes ``text_config`` in ``sglang.srt.models.cosmos3_edge``. Keep it out of
+# `_CONFIG_REGISTRY` so the generic parser can flatten text attributes onto the
+# root config after `AutoConfig.from_pretrained`, matching other multimodal
+# configs that use a text sub-config.
+for _cosmos3_edge_config_cls in (
+    Cosmos3EdgeTextConfig,
+    Cosmos3EdgeVisionConfig,
+    Cosmos3EdgeProjectorConfig,
+    Cosmos3EdgeConfig,
+):
+    try:
+        AutoConfig.register(
+            _cosmos3_edge_config_cls.model_type, _cosmos3_edge_config_cls
+        )
+    except ValueError as e:
+        err = str(e).lower()
+        if "already registered" not in err and "already used" not in err:
+            logger.warning(
+                "Failed to register config %s: %s",
+                _cosmos3_edge_config_cls.model_type,
+                e,
+            )
+
+
+# ---------------------------------------------------------------------------
+# Download / path helpers
+# ---------------------------------------------------------------------------
+
+
+def download_from_hf(
+    model_path: str,
+    allow_patterns: Optional[Union[str, list]] = None,
+):
+    if os.path.exists(model_path):
+        return model_path
+
+    if not allow_patterns:
+        allow_patterns = ["*.json", "*.bin", "*.model"]
+
+    return snapshot_download(model_path, allow_patterns=allow_patterns)
+
+
+def resolve_runai_obj_uri(model_name_or_path: str) -> str:
+    if is_runai_obj_uri(model_name_or_path):
+        return ObjectStorageModel.get_path(model_name_or_path)
+    return model_name_or_path
+
+
+def _resolve_local_or_cached_file(model_name_or_path, filename, revision=None):
+    """Resolve a file from a local directory or HF hub cache (no network)."""
+    local_path = Path(model_name_or_path) / filename
+    if local_path.is_file():
+        return str(local_path)
+    from huggingface_hub import hf_hub_download
+
+    return hf_hub_download(
+        model_name_or_path, filename, revision=revision, local_files_only=True
+    )
+
+
+def _cached_file_exists(model_name_or_path, filename, revision=None) -> bool:
+    """Whether *filename* is available locally or in the HF cache (no network)."""
+    try:
+        _resolve_local_or_cached_file(model_name_or_path, filename, revision)
+        return True
+    except Exception:
+        return False
+
+
+def _remote_file_exists(repo_id, filename, revision=None) -> bool:
+    """Whether *filename* exists on the HF hub (HEAD request only, no download).
+
+    Returns False on any error (offline, gated, network, invalid id) so callers
+    fall back to their default path instead of crashing.
+    """
+    from huggingface_hub.constants import HF_HUB_OFFLINE
+
+    if HF_HUB_OFFLINE:
+        return False
+    try:
+        from huggingface_hub import HfApi
+
+        return HfApi().file_exists(repo_id, filename, revision=revision)
+    except Exception:
+        return False
+
+
+def check_gguf_file(model: Union[str, os.PathLike]) -> bool:
+    model = Path(model)
+    if not model.is_file():
+        return False
+    elif model.suffix == ".gguf":
+        return True
+
+    with open(model, "rb") as f:
+        header = f.read(4)
+    return header == b"GGUF"
+
+
+def resolve_hf_gguf_reference(
+    model: str, revision: Optional[str] = None
+) -> Optional[str]:
+    """Download a .gguf named by Hub reference and return its local path.
+
+    owner/repo/path/inside/repo.gguf   -> exactly that file
+    owner/repo:QUANT_TYPE              -> the only matching quantization
+    owner/repo                         -> the only .gguf in the repo
+    """
+    from sglang.srt.utils import is_remote_url
+
+    if not model or os.path.exists(model) or is_remote_url(model):
+        return None
+
+    from huggingface_hub import hf_hub_download
+
+    if ":" in model:
+        repo_id, _, quant_type = model.rpartition(":")
+        if repo_id.count("/") != 1 or not quant_type:
+            return None
+
+        from huggingface_hub import HfApi
+
+        files = [
+            sibling.rfilename
+            for sibling in HfApi().repo_info(repo_id, revision=revision).siblings
+        ]
+        suffix = f"-{quant_type}.gguf"
+        candidates = [filename for filename in files if filename.endswith(suffix)]
+        if not candidates:
+            available = sorted(
+                filename for filename in files if filename.endswith(".gguf")
+            )
+            raise ValueError(
+                f"No file matching quant type {quant_type!r} in {repo_id}. "
+                f"Available GGUF files: {available}"
+            )
+        if len(candidates) > 1:
+            raise ValueError(
+                f"Quant type {quant_type!r} is ambiguous in {repo_id}: "
+                f"{sorted(candidates)}. Pass the full owner/repo/path/file.gguf "
+                "reference instead."
+            )
+        return hf_hub_download(repo_id, candidates[0], revision=revision)
+
+    parts = model.strip("/").split("/")
+    if len(parts) < 2:
+        return None
+
+    if len(parts) > 2 and model.endswith(".gguf"):
+        repo_id = "/".join(parts[:2])
+        filename = "/".join(parts[2:])
+        return hf_hub_download(repo_id, filename, revision=revision)
+
+    if len(parts) != 2:
+        return None
+
+    from huggingface_hub import HfApi
+
+    try:
+        files = [
+            s.rfilename for s in HfApi().repo_info(model, revision=revision).siblings
+        ]
+    except Exception:
+        return None
+    if any(f == "config.json" for f in files):
+        return None
+
+    candidates = [f for f in files if f.endswith(".gguf")]
+    if not candidates:
+        return None
+    if len(candidates) > 1:
+        listing = "\n  ".join(f"{model}/{f}" for f in sorted(candidates))
+        raise ValueError(
+            f"{model} contains {len(candidates)} .gguf files; name the one to "
+            f"serve:\n  {listing}"
+        )
+    return hf_hub_download(model, candidates[0], revision=revision)
+
+
+def gguf_sidecar_dir(
+    gguf_path: Union[str, os.PathLike], sentinel: str
+) -> Optional[Path]:
+    """Directory containing *sentinel* next to a .gguf file, if there is one."""
+    directory = Path(gguf_path).parent
+    return directory if (directory / sentinel).is_file() else None
+
+
+# ---------------------------------------------------------------------------
+# Rope / text config helpers
+# ---------------------------------------------------------------------------
+
+
+def get_rope_config(config):
+    """Get (rope_theta, rope_params) from config, supporting both v4 and v5.
+
+    Trust-remote-code configs or parent configs passed to sub-models may not
+    have the v5 ``rope_parameters`` property, so we fall back to the v4-style
+    ``config.rope_theta`` / ``config.rope_scaling`` attributes.
+
+    Returns:
+        (rope_theta, rope_params): In v5, rope_params is the full
+        rope_parameters dict (which subsumes rope_scaling and includes
+        rope_theta). In v4, rope_params is the rope_scaling dict or None.
+    """
+    rope_params = getattr(config, "rope_parameters", None)
+    if rope_params is not None:
+        rope_theta = rope_params.get("rope_theta", getattr(config, "rope_theta", 10000))
+        return rope_theta, rope_params
+    return getattr(config, "rope_theta", 10000), getattr(config, "rope_scaling", None)
+
+
+def _patch_text_config(parent_config: PretrainedConfig, text_config):
+    """Synchronize standard attributes between parent config and text sub-config.
+
+    In transformers v5, the "untangle config" refactor removed automatic
+    inheritance of top-level PretrainedConfig attributes (pad_token_id,
+    tie_word_embeddings, etc.) from sub-configs. Downstream code expects
+    these attributes to be present on both configs (some models pass the
+    parent directly to the language model, others pass the text sub-config),
+    so we propagate in both directions when an attribute is missing.
+    (See https://github.com/huggingface/transformers/pull/41541)
+    """
+    _ATTRS_TO_PROPAGATE = [
+        "pad_token_id",
+        "bos_token_id",
+        "eos_token_id",
+        "tie_word_embeddings",
+    ]
+    for attr in _ATTRS_TO_PROPAGATE:
+        parent_has = hasattr(parent_config, attr)
+        text_has = hasattr(text_config, attr)
+        if parent_has and not text_has:
+            setattr(text_config, attr, getattr(parent_config, attr))
+        elif text_has and not parent_has:
+            setattr(parent_config, attr, getattr(text_config, attr))
+    return text_config
+
+
+def get_hf_text_config(config: PretrainedConfig):
+    """Get the "sub" config relevant to llm for multi modal models.
+    No op for pure text models.
+    """
+    if config.architectures is not None:
+        class_name = config.architectures[0]
+        if class_name.startswith("Llava") and class_name.endswith("ForCausalLM"):
+            # We support non-hf version of llava models, so we do not want to
+            # read the wrong values from the unused default text_config.
+            # NOTE(HandH1998): We set `torch_dtype` of config to `torch.float16` for the weights, as
+            # `torch.float16` is default used for image features in `python/sglang/srt/models/llava.py`.
+            setattr(config, "dtype", torch.float16)
+            return config
+
+    text_config = None
+
+    # Some models (e.g. DeepSeek-OCR) store sub-configs as plain dicts.
+    # Convert to PretrainedConfig early so hasattr() checks and asserts work.
+    parent_dtype = getattr(config, "dtype", None)
+    for _attr in ("text_config", "llm_config", "language_config", "thinker_config"):
+        _sub = getattr(config, _attr, None)
+        if isinstance(_sub, dict):
+            _converted = PretrainedConfig(**_sub)
+            if getattr(_converted, "dtype", None) is None and parent_dtype is not None:
+                _converted.dtype = parent_dtype
+            setattr(config, _attr, _converted)
+        elif _sub is not None and parent_dtype is not None:
+            # transformers v5 multimodal configs (e.g. Mistral3Config) carry
+            # `dtype` only on the top-level config, leaving the sub-configs at
+            # None. Without this, _get_and_verify_dtype falls back to float32
+            # and then "auto" downcasts to float16, which overflows the Pixtral
+            # vision tower on real images and produces NaN features.
+            if getattr(_sub, "dtype", None) is None:
+                _sub.dtype = parent_dtype
+
+    # Priority: thinker_config > llm_config > language_config > text_config
+    if hasattr(config, "thinker_config"):
+        # qwen2.5 omni
+        thinker_config = config.thinker_config
+        if hasattr(thinker_config, "text_config"):
+            setattr(
+                thinker_config.text_config,
+                "dtype",
+                getattr(thinker_config, "dtype", None),
+            )
+            text_config = thinker_config.text_config
+        else:
+            text_config = thinker_config
+    elif hasattr(config, "llm_config"):
+        # PointsV1.5 Chat Model
+        assert hasattr(config.llm_config, "num_attention_heads")
+        text_config = config.llm_config
+    elif hasattr(config, "language_config"):
+        text_config = config.language_config
+    elif hasattr(config, "text_config"):
+        # The code operates under the assumption that text_config should have
+        # `num_attention_heads` (among others). Assert here to fail early
+        # if transformers config doesn't align with this assumption.
+        assert hasattr(config.text_config, "num_attention_heads")
+        text_config = config.text_config
+
+    # Ensure rope_scaling dicts have "type" for remote-code compat (v5).
+    normalize_rope_scaling_compat(config)
+
+    if text_config is not None:
+        return _patch_text_config(config, text_config)
+    return config
+
+
+# ---------------------------------------------------------------------------
+# Model-specific helpers
+# ---------------------------------------------------------------------------
+
+
+def _ensure_sub_configs(config: PretrainedConfig, *attr_names: str) -> None:
+    """Convert dict-valued sub-configs to proper AutoConfig objects in-place."""
+    for attr in attr_names:
+        sub = getattr(config, attr, None)
+        if sub is not None and isinstance(sub, dict):
+            setattr(config, attr, AutoConfig.for_model(**sub))
+
+
+def _is_deepseek_ocr_model(config: PretrainedConfig) -> bool:
+    # TODO: Remove this workaround once AutoConfig correctly identifies deepseek-ocr.
+    # Hugging Face's AutoConfig currently misidentifies it as deepseekvl2.
+    auto_map = getattr(config, "auto_map", None) or {}
+    return auto_map.get("AutoModel") == "modeling_deepseekocr.DeepseekOCRForCausalLM"
+
+
+def _is_deepseek_ocr2_model(config: PretrainedConfig) -> bool:
+    auto_map = getattr(config, "auto_map", None) or {}
+    return auto_map.get("AutoModel") == "modeling_deepseekocr2.DeepseekOCR2ForCausalLM"
+
+
+def _override_v_head_dim_if_zero(config: PretrainedConfig, patch: int = 128) -> None:
+    patched = False
+    for attr in ("text_config", "language_config"):
+        sub = getattr(config, attr, None)
+        if sub is None:
+            continue
+        if isinstance(sub, dict):
+            if sub.get("v_head_dim") == 0:
+                sub["v_head_dim"] = patch
+                patched = True
+        elif getattr(sub, "v_head_dim", None) == 0:
+            sub.v_head_dim = patch
+            patched = True
+    if patched:
+        logger.warning(
+            f"Overriding v_head_dim from 0 to {patch} to avoid potential issues."
+        )
+
+
+# ---------------------------------------------------------------------------
+# Context length / generation config / sparse attention
+# ---------------------------------------------------------------------------
+
+# Models don't use the same configuration key for determining the maximum
+# context length.  Store them here so we can sanely check them.
+# NOTE: The ordering here is important. Some models have two of these and we
+# have a preference for which value gets used.
+CONTEXT_LENGTH_KEYS = [
+    "max_sequence_length",
+    "seq_length",
+    "max_seq_len",
+    "model_max_length",
+    "max_position_embeddings",
+]
+
+
+def get_context_length(config):
+    """Get the context length of a model from a huggingface model configs."""
+    text_config = config
+    rope_scaling = getattr(text_config, "rope_scaling", None)
+    if rope_scaling:
+        rope_scaling_factor = rope_scaling.get("factor", 1)
+        if "original_max_position_embeddings" in rope_scaling:
+            rope_scaling_factor = 1
+        if rope_scaling.get("rope_type", None) == "llama3":
+            rope_scaling_factor = 1
+    else:
+        rope_scaling_factor = 1
+
+    for key in CONTEXT_LENGTH_KEYS:
+        val = getattr(text_config, key, None)
+        if val is not None:
+            return int(rope_scaling_factor * val)
+    return 2048
+
+
+@lru_cache_frozenset(maxsize=32)
+def get_generation_config(
+    model: str,
+    trust_remote_code: bool,
+    revision: Optional[str] = None,
+    **kwargs,
+):
+    if check_gguf_file(model):
+        sidecar = gguf_sidecar_dir(model, "generation_config.json")
+        if sidecar is not None:
+            model = str(sidecar)
+        else:
+            from .gguf_native import (
+                build_gguf_generation_config,
+                has_native_gguf_support,
+            )
+
+            if has_native_gguf_support(model):
+                return build_gguf_generation_config(model)
+
+    try:
+        return GenerationConfig.from_pretrained(
+            model, trust_remote_code=trust_remote_code, revision=revision, **kwargs
+        )
+    except (FileNotFoundError, OSError) as e:
+        # A missing generation_config.json is normal for many checkpoints and
+        # is surfaced by HF as a generic OSError (not FileNotFoundError). Treat
+        # it as benign — proceed without a generation config, at DEBUG level so
+        # normal startup logs stay quiet.
+        logger.debug(
+            "No generation config for %s: %s. Proceeding without it.",
+            model,
+            e,
+        )
+        return None
+
+
+# Qwen-1M related
+def get_sparse_attention_config(
+    model: str,
+    sparse_attention_config_filename: str = "sparse_attention_config.json",
+) -> Dict[str, Any]:
+    is_local = os.path.isdir(model)
+    if not is_local:
+        model = download_from_hf(model, allow_patterns=["*.json"])
+
+    config_file = os.path.join(model, sparse_attention_config_filename)
+    if not os.path.exists(config_file):
+        return {}
+
+    with open(config_file) as f:
+        config = json.load(f)
+    return config
+
+
+# ---------------------------------------------------------------------------
+# Tokenizer / processor helpers
+# ---------------------------------------------------------------------------
+
+
+# Some models don't have an available processor, e.g.: InternVL
+def get_tokenizer_from_processor(processor):
+    from transformers import PreTrainedTokenizerBase
+
+    if isinstance(processor, PreTrainedTokenizerBase):
+        return processor
+    return processor.tokenizer
+
+
+# Turn-final markers that some checkpoints ship without EOS metadata:
+# <|eom_id|> (Llama-3 tool use), <|content_model_end_sampling|> (Inkling,
+# whose bundled tokenizer config leaves eos_token unset), and
+# <|ifm|im_end|> (some K2 Horizon checkpoints, notably 0.9B, name only
+# <|endoftext|> as EOS).
+_ADDITIONAL_STOP_TOKEN_TEXTS = (
+    "<|eom_id|>",
+    "<|content_model_end_sampling|>",
+    "<|ifm|im_end|>",
+)
+
+
+def attach_additional_stop_token_ids(tokenizer):
+    added = tokenizer.get_added_vocab()
+    stop_ids = {added[text] for text in _ADDITIONAL_STOP_TOKEN_TEXTS if text in added}
+    tokenizer.additional_stop_token_ids = stop_ids or None
+
+# === agnes ===
+from sglang.srt.configs.agnes import AgnesConfig as _AgnesConfig
+
+_CONFIG_REGISTRY[_AgnesConfig.model_type] = _AgnesConfig

sglang_patch/v0.5.19/sglang/srt/configs/agnes.py

@@ -0,0 +1,57 @@
+# Agnes 3.0 Flash configuration for sglang.
+#
+# The HF checkpoint says model_type "agnes", names its layer types
+# agnes_delta_attention / agnes_global_attention and carries a parallel FFN
+# branch per layer.  The server runs it on its built-in hybrid
+# (delta-rule + global attention) implementation, so this class maps those
+# onto the fields that implementation reads; the checkpoint's tensor names
+# are translated while loading (see the model file patched by apply_patch.py).
+from sglang.srt.configs.qwen3_5 import Qwen3_5Config
+
+AGNES_DELTA = "agnes_delta_attention"
+AGNES_GLOBAL = "agnes_global_attention"
+
+
+class AgnesConfig(Qwen3_5Config):
+    model_type = "agnes"
+
+    def __init__(self, text_config=None, vision_config=None, **kwargs):
+        kwargs.pop("auto_map", None)  # the transformers remote code is not used in the server
+        if isinstance(text_config, dict):
+            text_config = dict(text_config)
+            text_config["model_type"] = "qwen3_5_text"
+            width = int(text_config.pop("parallel_ffn_intermediate_size", 0) or 0)
+            plan = text_config.pop("layer_types", None)
+            interval = text_config.pop("global_attention_interval", None)
+            if interval is None:
+                interval = text_config.pop("full_attention_interval", None)
+            if interval is None and plan:
+                interval = next(i + 1 for i, t in enumerate(plan) if t == AGNES_GLOBAL)
+            text_config["full_attention_interval"] = int(interval or 4)
+            main = int(text_config["intermediate_size"])
+            # the parallel branch is folded into the main MLP at load time
+            text_config["intermediate_size"] = main + width
+            text_config["agnes_main_intermediate_size"] = main
+            text_config["agnes_parallel_ffn_intermediate_size"] = width
+        if isinstance(vision_config, dict):
+            vision_config = dict(vision_config)
+            vision_config["model_type"] = "qwen3_5"
+        kwargs["architectures"] = ["Qwen3_5ForConditionalGeneration"]
+        super().__init__(text_config=text_config, vision_config=vision_config, **kwargs)
+        # every downstream check sees the built-in hybrid architecture
+        self.model_type = "qwen3_5"
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
+        # The weight loader needs the checkpoint directory to pick up the parallel
+        # branch tensors.  The path is written into the config *dict* before the
+        # object is built: the config reaches the worker processes through a
+        # to_dict round trip, which keeps fields that came in through __init__
+        # and drops attributes set afterwards (from_pretrained's own kwargs only
+        # override known fields, so they cannot carry it either).
+        path = str(pretrained_model_name_or_path)
+        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+        config_dict["agnes_model_path"] = path
+        if isinstance(config_dict.get("text_config"), dict):
+            config_dict["text_config"]["agnes_model_path"] = path
+        return cls.from_dict(config_dict, **kwargs)

sglang_patch/v0.5.19/sglang/srt/utils/hf_transformers/common.py

@@ -0,0 +1,670 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Shared helpers used by config, tokenizer, and processor modules."""
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Type, Union
+
+import torch
+from huggingface_hub import snapshot_download
+
+from sglang.srt.configs import (
+    AfmoeConfig,
+    BailingHybridConfig,
+    ChatGLMConfig,
+    DbrxConfig,
+    DeepseekVL2Config,
+    Dots3Config,
+    DotsOCRConfig,
+    DotsVLMConfig,
+    ExaoneConfig,
+    FalconH1Config,
+    GraniteMoeHybridConfig,
+    InklingAudioConfig,
+    InklingMMConfig,
+    InklingModelConfig,
+    InklingVisionConfig,
+    InternS2MobiusConfig,
+    InternS2MobiusTextConfig,
+    InternS2PreviewConfig,
+    JetNemotronConfig,
+    JetVLMConfig,
+    KimiK3Config,
+    KimiK25Config,
+    KimiLinearConfig,
+    KimiVLConfig,
+    LagunaConfig,
+    LocateAnythingConfig,
+    LongcatFlashConfig,
+    MiniCPMHybridConfig,
+    MiniCPMV4_6Config,
+    MiniCPMV4_6VisionConfig,
+    MiniMaxM3VLConfig,
+    MultiModalityConfig,
+    MuseGlimmerAssistantConfig,
+    MuseGlimmerConfig,
+    NemotronH_Nano_Omni_Reasoning_V3_Config,
+    NemotronH_Nano_VL_V2_Config,
+    NemotronHConfig,
+    NemotronHPuzzleConfig,
+    Olmo3Config,
+    Qwen3_5Config,
+    Qwen3_5MoeConfig,
+    Qwen3_5MoeTextConfig,
+    Qwen3_5TextConfig,
+    Qwen3NextConfig,
+    Spark2_5Config,
+    Step3p5Config,
+    Step3p7Config,
+    Step3VLConfig,
+)
+from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config
+from sglang.srt.configs.internvl import InternVLChatConfig
+from sglang.srt.utils import get_bool_env_var, logger, lru_cache_frozenset
+from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri
+
+from ..hf_transformers_patches import normalize_rope_scaling_compat
+
+if get_bool_env_var("SGLANG_USE_MODELSCOPE"):
+    from modelscope import AutoConfig, GenerationConfig
+else:
+    from transformers import AutoConfig, GenerationConfig
+
+from transformers import PretrainedConfig
+
+# ---------------------------------------------------------------------------
+# Config registry
+# ---------------------------------------------------------------------------
+
+_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
+    cls.model_type: cls
+    for cls in [
+        AfmoeConfig,
+        BailingHybridConfig,
+        ChatGLMConfig,
+        DbrxConfig,
+        ExaoneConfig,
+        DeepseekVL2Config,
+        MultiModalityConfig,
+        KimiVLConfig,
+        LocateAnythingConfig,
+        InternVLChatConfig,
+        LagunaConfig,
+        Spark2_5Config,
+        Step3VLConfig,
+        LongcatFlashConfig,
+        Olmo3Config,
+        MuseGlimmerConfig,
+        MuseGlimmerAssistantConfig,
+        KimiK3Config,
+        KimiLinearConfig,
+        Qwen3NextConfig,
+        FalconH1Config,
+        GraniteMoeHybridConfig,
+        DotsVLMConfig,
+        DotsOCRConfig,
+        Dots3Config,
+        NemotronH_Nano_VL_V2_Config,
+        NemotronH_Nano_Omni_Reasoning_V3_Config,
+        NemotronHConfig,
+        NemotronHPuzzleConfig,
+        DeepseekVLV2Config,
+        Qwen3_5Config,
+        Qwen3_5MoeConfig,
+        Qwen3_5TextConfig,
+        Qwen3_5MoeTextConfig,
+        InternS2PreviewConfig,
+        InternS2MobiusConfig,
+        InternS2MobiusTextConfig,
+        JetNemotronConfig,
+        JetVLMConfig,
+        KimiK25Config,
+        Step3p5Config,
+        Step3p7Config,
+        MiniCPMHybridConfig,
+        MiniCPMV4_6Config,
+        MiniCPMV4_6VisionConfig,
+        InklingModelConfig,
+        InklingAudioConfig,
+        InklingVisionConfig,
+        InklingMMConfig,
+        MiniMaxM3VLConfig,
+    ]
+}
+
+
+# DeepSeek V3.2 / V4 reuse the V3 config schema. Subclass the upstream
+# transformers class with each model_type so AutoConfig.register passes its
+# consistency check (which requires class.model_type == registered key).
+# Default-value divergences (e.g. V4's topk_group) are handled in
+# model_config.py post-load.
+try:
+    from transformers import DeepseekV3Config as _HFDeepseekV3Config
+
+    class _DeepseekV32ConfigAlias(_HFDeepseekV3Config):
+        model_type = "deepseek_v32"
+
+    class _DeepseekV4ConfigAlias(_HFDeepseekV3Config):
+        model_type = "deepseek_v4"
+
+    _CONFIG_REGISTRY["deepseek_v32"] = _DeepseekV32ConfigAlias
+    _CONFIG_REGISTRY["deepseek_v4"] = _DeepseekV4ConfigAlias
+
+    # For kimi_k25_eagle3
+    class _KimiK2ConfigAlias(_HFDeepseekV3Config):
+        model_type = "kimi_k2"
+
+    _CONFIG_REGISTRY["kimi_k2"] = _KimiK2ConfigAlias
+except ImportError:
+    pass
+
+# Newer transformers versions (>=5.10.2) expose MellumConfig directly,
+# but fallback to Qwen3MoeConfig for older versions.
+try:
+    import transformers as _hf_transformers
+
+    _HFMellumConfig = getattr(_hf_transformers, "MellumConfig", None)
+
+    if _HFMellumConfig is not None:
+        _CONFIG_REGISTRY["mellum"] = _HFMellumConfig
+    else:
+        from transformers import Qwen3MoeConfig as _HFQwen3MoeConfig
+
+        class _MellumConfigAlias(_HFQwen3MoeConfig):
+            model_type = "mellum"
+
+            def __post_init__(self, **kwargs):
+                # Qwen3MoeConfig.__post_init__ wipes sliding_window unless
+                # use_sliding_window=True. Mellum gates sliding attention
+                # per-layer via layer_types, so preserve sliding_window
+                # regardless of the legacy use_sliding_window flag.
+                sliding_window = getattr(self, "sliding_window", None)
+                super().__post_init__(**kwargs)
+                self.sliding_window = sliding_window
+
+        _CONFIG_REGISTRY["mellum"] = _MellumConfigAlias
+
+except ImportError:
+    pass
+
+
+try:
+    from transformers import Gemma4Config as _HFGemma4Config
+
+    class _Gemma4UnifiedConfigAlias(_HFGemma4Config):
+        model_type = "gemma4_unified"
+
+    _CONFIG_REGISTRY["gemma4_unified"] = _Gemma4UnifiedConfigAlias
+except ImportError:
+    pass
+
+for name, cls in _CONFIG_REGISTRY.items():
+    try:
+        AutoConfig.register(name, cls)
+    except ValueError as e:
+        err = str(e).lower()
+        if "already registered" not in err and "already used" not in err:
+            logger.warning("Failed to register config %s: %s", name, e)
+
+
+# ---------------------------------------------------------------------------
+# Download / path helpers
+# ---------------------------------------------------------------------------
+
+
+def download_from_hf(
+    model_path: str,
+    allow_patterns: Optional[Union[str, list]] = None,
+):
+    if os.path.exists(model_path):
+        return model_path
+
+    if not allow_patterns:
+        allow_patterns = ["*.json", "*.bin", "*.model"]
+
+    return snapshot_download(model_path, allow_patterns=allow_patterns)
+
+
+def resolve_runai_obj_uri(model_name_or_path: str) -> str:
+    if is_runai_obj_uri(model_name_or_path):
+        return ObjectStorageModel.get_path(model_name_or_path)
+    return model_name_or_path
+
+
+def _resolve_local_or_cached_file(model_name_or_path, filename, revision=None):
+    """Resolve a file from a local directory or HF hub cache (no network)."""
+    local_path = Path(model_name_or_path) / filename
+    if local_path.is_file():
+        return str(local_path)
+    from huggingface_hub import hf_hub_download
+
+    return hf_hub_download(
+        model_name_or_path, filename, revision=revision, local_files_only=True
+    )
+
+
+def _cached_file_exists(model_name_or_path, filename, revision=None) -> bool:
+    """Whether *filename* is available locally or in the HF cache (no network)."""
+    try:
+        _resolve_local_or_cached_file(model_name_or_path, filename, revision)
+        return True
+    except Exception:
+        return False
+
+
+def _remote_file_exists(repo_id, filename, revision=None) -> bool:
+    """Whether *filename* exists on the HF hub (HEAD request only, no download).
+
+    Returns False on any error (offline, gated, network, invalid id) so callers
+    fall back to their default path instead of crashing.
+    """
+    from huggingface_hub.constants import HF_HUB_OFFLINE
+
+    if HF_HUB_OFFLINE:
+        return False
+    try:
+        from huggingface_hub import HfApi
+
+        return HfApi().file_exists(repo_id, filename, revision=revision)
+    except Exception:
+        return False
+
+
+def check_gguf_file(model: Union[str, os.PathLike]) -> bool:
+    model = Path(model)
+    if not model.is_file():
+        return False
+    elif model.suffix == ".gguf":
+        return True
+
+    with open(model, "rb") as f:
+        header = f.read(4)
+    return header == b"GGUF"
+
+
+def resolve_hf_gguf_reference(
+    model: str, revision: Optional[str] = None
+) -> Optional[str]:
+    """Download a .gguf named by Hub reference and return its local path.
+
+    owner/repo/path/inside/repo.gguf   -> exactly that file
+    owner/repo:QUANT_TYPE              -> the only matching quantization
+    owner/repo                         -> the only .gguf in the repo
+    """
+    from sglang.srt.utils import is_remote_url
+
+    if not model or os.path.exists(model) or is_remote_url(model):
+        return None
+
+    from huggingface_hub import hf_hub_download
+
+    if ":" in model:
+        repo_id, _, quant_type = model.rpartition(":")
+        if repo_id.count("/") != 1 or not quant_type:
+            return None
+
+        from huggingface_hub import HfApi
+
+        files = [
+            sibling.rfilename
+            for sibling in HfApi().repo_info(repo_id, revision=revision).siblings
+        ]
+        suffix = f"-{quant_type}.gguf"
+        candidates = [filename for filename in files if filename.endswith(suffix)]
+        if not candidates:
+            available = sorted(
+                filename for filename in files if filename.endswith(".gguf")
+            )
+            raise ValueError(
+                f"No file matching quant type {quant_type!r} in {repo_id}. "
+                f"Available GGUF files: {available}"
+            )
+        if len(candidates) > 1:
+            raise ValueError(
+                f"Quant type {quant_type!r} is ambiguous in {repo_id}: "
+                f"{sorted(candidates)}. Pass the full owner/repo/path/file.gguf "
+                "reference instead."
+            )
+        return hf_hub_download(repo_id, candidates[0], revision=revision)
+
+    parts = model.strip("/").split("/")
+    if len(parts) < 2:
+        return None
+
+    if len(parts) > 2 and model.endswith(".gguf"):
+        repo_id = "/".join(parts[:2])
+        filename = "/".join(parts[2:])
+        return hf_hub_download(repo_id, filename, revision=revision)
+
+    if len(parts) != 2:
+        return None
+
+    from huggingface_hub import HfApi
+
+    try:
+        files = [
+            s.rfilename for s in HfApi().repo_info(model, revision=revision).siblings
+        ]
+    except Exception:
+        return None
+    if any(f == "config.json" for f in files):
+        return None
+
+    candidates = [f for f in files if f.endswith(".gguf")]
+    if not candidates:
+        return None
+    if len(candidates) > 1:
+        listing = "\n  ".join(f"{model}/{f}" for f in sorted(candidates))
+        raise ValueError(
+            f"{model} contains {len(candidates)} .gguf files; name the one to "
+            f"serve:\n  {listing}"
+        )
+    return hf_hub_download(model, candidates[0], revision=revision)
+
+
+def gguf_sidecar_dir(
+    gguf_path: Union[str, os.PathLike], sentinel: str
+) -> Optional[Path]:
+    """Directory containing *sentinel* next to a .gguf file, if there is one."""
+    directory = Path(gguf_path).parent
+    return directory if (directory / sentinel).is_file() else None
+
+
+# ---------------------------------------------------------------------------
+# Rope / text config helpers
+# ---------------------------------------------------------------------------
+
+
+def get_rope_config(config):
+    """Get (rope_theta, rope_params) from config, supporting both v4 and v5.
+
+    Trust-remote-code configs or parent configs passed to sub-models may not
+    have the v5 ``rope_parameters`` property, so we fall back to the v4-style
+    ``config.rope_theta`` / ``config.rope_scaling`` attributes.
+
+    Returns:
+        (rope_theta, rope_params): In v5, rope_params is the full
+        rope_parameters dict (which subsumes rope_scaling and includes
+        rope_theta). In v4, rope_params is the rope_scaling dict or None.
+    """
+    rope_params = getattr(config, "rope_parameters", None)
+    if rope_params is not None:
+        rope_theta = rope_params.get("rope_theta", getattr(config, "rope_theta", 10000))
+        return rope_theta, rope_params
+    return getattr(config, "rope_theta", 10000), getattr(config, "rope_scaling", None)
+
+
+def _patch_text_config(parent_config: PretrainedConfig, text_config):
+    """Synchronize standard attributes between parent config and text sub-config.
+
+    In transformers v5, the "untangle config" refactor removed automatic
+    inheritance of top-level PretrainedConfig attributes (pad_token_id,
+    tie_word_embeddings, etc.) from sub-configs. Downstream code expects
+    these attributes to be present on both configs (some models pass the
+    parent directly to the language model, others pass the text sub-config),
+    so we propagate in both directions when an attribute is missing.
+    (See https://github.com/huggingface/transformers/pull/41541)
+    """
+    _ATTRS_TO_PROPAGATE = [
+        "pad_token_id",
+        "bos_token_id",
+        "eos_token_id",
+        "tie_word_embeddings",
+    ]
+    for attr in _ATTRS_TO_PROPAGATE:
+        parent_has = hasattr(parent_config, attr)
+        text_has = hasattr(text_config, attr)
+        if parent_has and not text_has:
+            setattr(text_config, attr, getattr(parent_config, attr))
+        elif text_has and not parent_has:
+            setattr(parent_config, attr, getattr(text_config, attr))
+    return text_config
+
+
+def get_hf_text_config(config: PretrainedConfig):
+    """Get the "sub" config relevant to llm for multi modal models.
+    No op for pure text models.
+    """
+    if config.architectures is not None:
+        class_name = config.architectures[0]
+        if class_name.startswith("Llava") and class_name.endswith("ForCausalLM"):
+            # We support non-hf version of llava models, so we do not want to
+            # read the wrong values from the unused default text_config.
+            # NOTE(HandH1998): We set `torch_dtype` of config to `torch.float16` for the weights, as
+            # `torch.float16` is default used for image features in `python/sglang/srt/models/llava.py`.
+            setattr(config, "dtype", torch.float16)
+            return config
+
+    text_config = None
+
+    # Some models (e.g. DeepSeek-OCR) store sub-configs as plain dicts.
+    # Convert to PretrainedConfig early so hasattr() checks and asserts work.
+    parent_dtype = getattr(config, "dtype", None)
+    for _attr in ("text_config", "llm_config", "language_config", "thinker_config"):
+        _sub = getattr(config, _attr, None)
+        if isinstance(_sub, dict):
+            _converted = PretrainedConfig(**_sub)
+            if getattr(_converted, "dtype", None) is None and parent_dtype is not None:
+                _converted.dtype = parent_dtype
+            setattr(config, _attr, _converted)
+        elif _sub is not None and parent_dtype is not None:
+            # transformers v5 multimodal configs (e.g. Mistral3Config) carry
+            # `dtype` only on the top-level config, leaving the sub-configs at
+            # None. Without this, _get_and_verify_dtype falls back to float32
+            # and then "auto" downcasts to float16, which overflows the Pixtral
+            # vision tower on real images and produces NaN features.
+            if getattr(_sub, "dtype", None) is None:
+                _sub.dtype = parent_dtype
+
+    # Priority: thinker_config > llm_config > language_config > text_config
+    if hasattr(config, "thinker_config"):
+        # qwen2.5 omni
+        thinker_config = config.thinker_config
+        if hasattr(thinker_config, "text_config"):
+            setattr(
+                thinker_config.text_config,
+                "dtype",
+                getattr(thinker_config, "dtype", None),
+            )
+            text_config = thinker_config.text_config
+        else:
+            text_config = thinker_config
+    elif hasattr(config, "llm_config"):
+        # PointsV1.5 Chat Model
+        assert hasattr(config.llm_config, "num_attention_heads")
+        text_config = config.llm_config
+    elif hasattr(config, "language_config"):
+        text_config = config.language_config
+    elif hasattr(config, "text_config"):
+        # The code operates under the assumption that text_config should have
+        # `num_attention_heads` (among others). Assert here to fail early
+        # if transformers config doesn't align with this assumption.
+        assert hasattr(config.text_config, "num_attention_heads")
+        text_config = config.text_config
+
+    # Ensure rope_scaling dicts have "type" for remote-code compat (v5).
+    normalize_rope_scaling_compat(config)
+
+    if text_config is not None:
+        return _patch_text_config(config, text_config)
+    return config
+
+
+# ---------------------------------------------------------------------------
+# Model-specific helpers
+# ---------------------------------------------------------------------------
+
+
+def _ensure_sub_configs(config: PretrainedConfig, *attr_names: str) -> None:
+    """Convert dict-valued sub-configs to proper AutoConfig objects in-place."""
+    for attr in attr_names:
+        sub = getattr(config, attr, None)
+        if sub is not None and isinstance(sub, dict):
+            setattr(config, attr, AutoConfig.for_model(**sub))
+
+
+def _is_deepseek_ocr_model(config: PretrainedConfig) -> bool:
+    # TODO: Remove this workaround once AutoConfig correctly identifies deepseek-ocr.
+    # Hugging Face's AutoConfig currently misidentifies it as deepseekvl2.
+    auto_map = getattr(config, "auto_map", None) or {}
+    return auto_map.get("AutoModel") == "modeling_deepseekocr.DeepseekOCRForCausalLM"
+
+
+def _is_deepseek_ocr2_model(config: PretrainedConfig) -> bool:
+    auto_map = getattr(config, "auto_map", None) or {}
+    return auto_map.get("AutoModel") == "modeling_deepseekocr2.DeepseekOCR2ForCausalLM"
+
+
+def _override_v_head_dim_if_zero(config: PretrainedConfig, patch: int = 128) -> None:
+    patched = False
+    for attr in ("text_config", "language_config"):
+        sub = getattr(config, attr, None)
+        if sub is None:
+            continue
+        if isinstance(sub, dict):
+            if sub.get("v_head_dim") == 0:
+                sub["v_head_dim"] = patch
+                patched = True
+        elif getattr(sub, "v_head_dim", None) == 0:
+            sub.v_head_dim = patch
+            patched = True
+    if patched:
+        logger.warning(
+            f"Overriding v_head_dim from 0 to {patch} to avoid potential issues."
+        )
+
+
+# ---------------------------------------------------------------------------
+# Context length / generation config / sparse attention
+# ---------------------------------------------------------------------------
+
+# Models don't use the same configuration key for determining the maximum
+# context length.  Store them here so we can sanely check them.
+# NOTE: The ordering here is important. Some models have two of these and we
+# have a preference for which value gets used.
+CONTEXT_LENGTH_KEYS = [
+    "max_sequence_length",
+    "seq_length",
+    "max_seq_len",
+    "model_max_length",
+    "max_position_embeddings",
+]
+
+
+def get_context_length(config):
+    """Get the context length of a model from a huggingface model configs."""
+    text_config = config
+    rope_scaling = getattr(text_config, "rope_scaling", None)
+    if rope_scaling:
+        rope_scaling_factor = rope_scaling.get("factor", 1)
+        if "original_max_position_embeddings" in rope_scaling:
+            rope_scaling_factor = 1
+        if rope_scaling.get("rope_type", None) == "llama3":
+            rope_scaling_factor = 1
+    else:
+        rope_scaling_factor = 1
+
+    for key in CONTEXT_LENGTH_KEYS:
+        val = getattr(text_config, key, None)
+        if val is not None:
+            return int(rope_scaling_factor * val)
+    return 2048
+
+
+@lru_cache_frozenset(maxsize=32)
+def get_generation_config(
+    model: str,
+    trust_remote_code: bool,
+    revision: Optional[str] = None,
+    **kwargs,
+):
+    if check_gguf_file(model):
+        sidecar = gguf_sidecar_dir(model, "generation_config.json")
+        if sidecar is not None:
+            model = str(sidecar)
+        else:
+            from .gguf_native import (
+                build_gguf_generation_config,
+                has_native_gguf_support,
+            )
+
+            if has_native_gguf_support(model):
+                return build_gguf_generation_config(model)
+
+    try:
+        return GenerationConfig.from_pretrained(
+            model, trust_remote_code=trust_remote_code, revision=revision, **kwargs
+        )
+    except (FileNotFoundError, OSError) as e:
+        # A missing generation_config.json is normal for many checkpoints and
+        # is surfaced by HF as a generic OSError (not FileNotFoundError). Treat
+        # it as benign — proceed without a generation config, at DEBUG level so
+        # normal startup logs stay quiet.
+        logger.debug(
+            "No generation config for %s: %s. Proceeding without it.",
+            model,
+            e,
+        )
+        return None
+
+
+# Qwen-1M related
+def get_sparse_attention_config(
+    model: str,
+    sparse_attention_config_filename: str = "sparse_attention_config.json",
+) -> Dict[str, Any]:
+    is_local = os.path.isdir(model)
+    if not is_local:
+        model = download_from_hf(model, allow_patterns=["*.json"])
+
+    config_file = os.path.join(model, sparse_attention_config_filename)
+    if not os.path.exists(config_file):
+        return {}
+
+    with open(config_file) as f:
+        config = json.load(f)
+    return config
+
+
+# ---------------------------------------------------------------------------
+# Tokenizer / processor helpers
+# ---------------------------------------------------------------------------
+
+
+# Some models don't have an available processor, e.g.: InternVL
+def get_tokenizer_from_processor(processor):
+    from transformers import PreTrainedTokenizerBase
+
+    if isinstance(processor, PreTrainedTokenizerBase):
+        return processor
+    return processor.tokenizer
+
+
+# Turn-final markers that some checkpoints ship without EOS metadata:
+# <|eom_id|> (Llama-3 tool use) and <|content_model_end_sampling|> (Inkling,
+# whose bundled tokenizer config leaves eos_token unset).
+_ADDITIONAL_STOP_TOKEN_TEXTS = ("<|eom_id|>", "<|content_model_end_sampling|>")
+
+
+def attach_additional_stop_token_ids(tokenizer):
+    added = tokenizer.get_added_vocab()
+    stop_ids = {added[text] for text in _ADDITIONAL_STOP_TOKEN_TEXTS if text in added}
+    tokenizer.additional_stop_token_ids = stop_ids or None
+
+# === agnes ===
+from sglang.srt.configs.agnes import AgnesConfig as _AgnesConfig
+
+_CONFIG_REGISTRY[_AgnesConfig.model_type] = _AgnesConfig

tokenizer_config.json

@@ -0,0 +1,413 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "248044": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248045": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248046": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248047": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248048": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248049": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248050": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248051": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248052": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248053": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248054": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248055": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248056": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248057": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248058": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248059": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248060": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248061": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248062": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248063": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248064": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248065": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248066": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248067": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248068": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248069": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "248070": {
+      "content": "<|audio_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248071": {
+      "content": "<|audio_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248072": {
+      "content": "<tts_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248073": {
+      "content": "<tts_text_bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248074": {
+      "content": "<tts_text_eod>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248075": {
+      "content": "<tts_text_bos_single>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248076": {
+      "content": "<|audio_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248077": {
+      "content": "<|agnes_bos|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "248078": {
+      "content": "<|agnes_eos|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "248079": {
+      "content": "<|agnes_system|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "248080": {
+      "content": "<|agnes_user|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "248081": {
+      "content": "<|agnes_assistant|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "248082": {
+      "content": "<|agnes_tool|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "248083": {
+      "content": "<|agnes_think_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "248084": {
+      "content": "<|agnes_think_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "248085": {
+      "content": "<|agnes_reserved_0|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "248086": {
+      "content": "<|agnes_reserved_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "248087": {
+      "content": "<|agnes_reserved_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "248088": {
+      "content": "<|agnes_reserved_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>",
+    "<|agnes_bos|>",
+    "<|agnes_eos|>",
+    "<|agnes_system|>",
+    "<|agnes_user|>",
+    "<|agnes_assistant|>",
+    "<|agnes_tool|>",
+    "<|agnes_think_start|>",
+    "<|agnes_think_end|>",
+    "<|agnes_reserved_0|>",
+    "<|agnes_reserved_1|>",
+    "<|agnes_reserved_2|>",
+    "<|agnes_reserved_3|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n    {%- if content is string %}\n        {{- content }}\n    {%- elif content is iterable and content is not mapping %}\n        {%- for item in content %}\n            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n                {%- if is_system_content %}\n                    {{- raise_exception('System message cannot contain images.') }}\n                {%- endif %}\n                {%- if do_vision_count %}\n                    {%- set image_count.value = image_count.value + 1 %}\n                {%- endif %}\n                {%- if add_vision_id %}\n                    {{- 'Picture ' ~ image_count.value ~ ': ' }}\n                {%- endif %}\n                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n            {%- elif 'video' in item or item.type == 'video' %}\n                {%- if is_system_content %}\n                    {{- raise_exception('System message cannot contain videos.') }}\n                {%- endif %}\n                {%- if do_vision_count %}\n                    {%- set video_count.value = video_count.value + 1 %}\n                {%- endif %}\n                {%- if add_vision_id %}\n                    {{- 'Video ' ~ video_count.value ~ ': ' }}\n                {%- endif %}\n                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n            {%- elif 'text' in item %}\n                {{- item.text }}\n            {%- else %}\n                {{- raise_exception('Unexpected item type in content.') }}\n            {%- endif %}\n        {%- endfor %}\n    {%- elif content is none or content is undefined %}\n        {{- '' }}\n    {%- else %}\n        {{- raise_exception('Unexpected content type.') }}\n    {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n    {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- set reasoning_instructions = '' %}\n{%- if enable_thinking is undefined or enable_thinking is true %}\n    {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}\n    {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}\n        {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}\n    {%- endif %}\n    {%- if resolved_reasoning_effort == 'xhigh' %}\n        {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}\n    {%- elif resolved_reasoning_effort == 'low' %}\n        {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}\n    {%- endif %}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if reasoning_instructions %}\n        {{- reasoning_instructions + '\\n\\n' }}\n    {%- endif %}\n    {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\" }}\n    {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n    {%- if messages[0].role == 'system' %}\n        {%- set content = render_content(messages[0].content, false, true)|trim %}\n        {%- if content %}\n            {{- '\\n\\n' + content }}\n        {%- endif %}\n    {%- endif %}\n    {{- '<|im_end|>\\n' }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {%- set content = render_content(messages[0].content, false, true)|trim %}\n        {%- if content %}\n            {{- '<|im_start|>system\\n' + (reasoning_instructions + '\\n\\n' if reasoning_instructions else '')  + content + '<|im_end|>\\n' }}\n        {%- elif reasoning_instructions %}\n            {{- '<|im_start|>system\\n' + reasoning_instructions + '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- elif reasoning_instructions %}\n        {{- '<|im_start|>system\\n' + reasoning_instructions + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n    {%- set index = (messages|length - 1) - loop.index0 %}\n    {%- if ns.multi_step_tool and message.role == \"user\" %}\n        {%- set content = render_content(message.content, false)|trim %}\n        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n            {%- set ns.multi_step_tool = false %}\n            {%- set ns.last_query_index = index %}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n    {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n    {%- set content = render_content(message.content, true)|trim %}\n    {%- if message.role == \"system\" %}\n        {%- if not loop.first %}\n            {{- raise_exception('System message must be at the beginning.') }}\n        {%- endif %}\n    {%- elif message.role == \"user\" %}\n        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {%- set reasoning_content = '' %}\n        {%- if message.reasoning_content is string %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- endif %}\n        {%- set reasoning_content = reasoning_content|trim %}\n        {%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}\n            {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n        {%- else %}\n            {{- '<|im_start|>' + message.role + '\\n' + content }}\n        {%- endif %}\n        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if tool_call.function is defined %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {%- if loop.first %}\n                    {%- if content|trim %}\n                        {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                    {%- else %}\n                        {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                    {%- endif %}\n                {%- else %}\n                    {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                {%- endif %}\n                {%- if tool_call.arguments is defined and tool_call.arguments != '' %}\n                    {%- for args_name, args_value in tool_call.arguments|items %}\n                        {{- '<parameter=' + args_name + '>\\n' }}\n                        {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}\n                        {{- args_value }}\n                        {{- '\\n</parameter>\\n' }}\n                    {%- endfor %}\n                {%- endif %}\n                {{- '</function>\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- content }}\n        {{- '\\n</tool_response>' }}\n        {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n            {{- '<|im_end|>\\n' }}\n        {%- elif loop.last %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- else %}\n        {{- raise_exception('Unexpected message role.') }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n    {%- if enable_thinking is defined and enable_thinking is false %}\n        {{- '<think>\\n\\n</think>\\n\\n' }}\n    {%- else %}\n        {{- '<think>\\n' }}\n    {%- endif %}\n{%- endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 262144,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": null,
+  "add_bos_token": false,
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "extra_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  }
+}

video_preprocessor_config.json

@@ -0,0 +1,27 @@
+{
+  "video_processor_type": "AgnesVideoProcessor",
+  "auto_map": {
+    "AutoVideoProcessor": "video_processing_agnes.AgnesVideoProcessor"
+  },
+  "size": {
+    "shortest_edge": 131072,
+    "longest_edge": 786432
+  },
+  "patch_size": 16,
+  "temporal_patch_size": 2,
+  "merge_size": 2,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "fps": 2,
+  "min_frames": 4,
+  "max_frames": 768,
+  "do_sample_frames": true
+}

video_processing_agnes.py

@@ -0,0 +1,197 @@
+# Copyright 2026 Agnes AI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Video processor for Agnes 3.0 Flash: frame sampling and dynamic-resolution patching."""
+
+import math
+
+import numpy as np
+import torch
+
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_utils import ChannelDimension, PILImageResampling, SizeDict, get_image_size
+from transformers.processing_utils import Unpack, VideosKwargs
+from transformers.utils import TensorType, add_start_docstrings, is_torchvision_available, logging
+from transformers.video_processing_utils import BASE_VIDEO_PROCESSOR_DOCSTRING, BaseVideoProcessor
+from transformers.video_utils import VideoMetadata, group_videos_by_shape, reorder_videos
+
+
+if is_torchvision_available():
+    from torchvision.transforms.v2 import functional as tvF
+
+logger = logging.get_logger(__name__)
+
+
+def fit_video_to_grid(
+    num_frames: int,
+    height: int,
+    width: int,
+    temporal_factor: int = 2,
+    factor: int = 32,
+    min_pixels: int = 128 * 128,
+    max_pixels: int = 16 * 16 * 2 * 2 * 2 * 6144,
+):
+    """Spatial size for a clip: multiples of `factor`, with the frame count
+    rounded up to `temporal_factor` and the total voxel count kept inside
+    [min_pixels, max_pixels]."""
+    if height < factor or width < factor:
+        raise ValueError(f"height:{height} or width:{width} must be larger than factor:{factor}")
+    elif max(height, width) / min(height, width) > 200:
+        raise ValueError(f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}")
+    h = round(height / factor) * factor
+    w = round(width / factor) * factor
+    t = math.ceil(num_frames / temporal_factor) * temporal_factor
+    if t * h * w > max_pixels:
+        scale = math.sqrt((num_frames * height * width) / max_pixels)
+        h = max(factor, math.floor(height / scale / factor) * factor)
+        w = max(factor, math.floor(width / scale / factor) * factor)
+    elif t * h * w < min_pixels:
+        scale = math.sqrt(min_pixels / (num_frames * height * width))
+        h = math.ceil(height * scale / factor) * factor
+        w = math.ceil(width * scale / factor) * factor
+    return h, w
+
+
+class AgnesVideoProcessorInitKwargs(VideosKwargs, total=False):
+    patch_size: int
+    temporal_patch_size: int
+    merge_size: int
+    min_frames: int
+    max_frames: int
+
+
+@add_start_docstrings(
+    "Video processor for Agnes 3.0 Flash; resizes each clip to a patch grid that fits its own resolution.",
+    BASE_VIDEO_PROCESSOR_DOCSTRING,
+    """
+        patch_size (`int`, *optional*, defaults to 16):
+            Spatial patch size of the vision tower.
+        temporal_patch_size (`int`, *optional*, defaults to 2):
+            Temporal patch size of the vision tower.
+        merge_size (`int`, *optional*, defaults to 2):
+            Side of the patch square merged into one language-model token.
+    """,
+)
+class AgnesVideoProcessor(BaseVideoProcessor):
+    resample = PILImageResampling.BICUBIC
+    size = {"shortest_edge": 128 * 32 * 32, "longest_edge": 32 * 32 * 768}
+    image_mean = [0.5, 0.5, 0.5]
+    image_std = [0.5, 0.5, 0.5]
+    do_resize = True
+    do_rescale = True
+    do_normalize = True
+    do_convert_rgb = True
+    patch_size = 16
+    temporal_patch_size = 2
+    merge_size = 2
+    fps = 2
+    min_frames = 4
+    max_frames = 768
+    do_sample_frames = True
+    valid_kwargs = AgnesVideoProcessorInitKwargs
+    model_input_names = ["pixel_values_videos", "video_grid_thw"]
+
+    def __init__(self, **kwargs: Unpack[AgnesVideoProcessorInitKwargs]):
+        super().__init__(**kwargs)
+
+    def _standardize_kwargs(self, **kwargs) -> dict:
+        kwargs = super()._standardize_kwargs(**kwargs)
+        size = kwargs.get("size", self.size)
+        if not size.shortest_edge or not size.longest_edge:
+            raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.")
+        return kwargs
+
+    def sample_frames(self, metadata: VideoMetadata, num_frames: int | None = None, fps: int | float | None = None, **kwargs):
+        """Frame indices to keep: `fps` frames per second of source video when
+        metadata is available, clamped to [min_frames, max_frames]; `num_frames`
+        overrides that.  Indices are spread uniformly over the clip."""
+        if fps is not None and num_frames is not None:
+            raise ValueError("`num_frames` and `fps` are mutually exclusive arguments, please use only one!")
+        total = metadata.total_num_frames
+        fps = fps if fps is not None else self.fps
+        if num_frames is None and fps is not None:
+            if metadata.fps is None:
+                metadata.fps = 24
+                logger.warning_once(
+                    "Asked to sample `fps` frames per second but no video metadata was provided which is required when sampling with `fps`. "
+                    "Defaulting to `fps=24`. Please provide `video_metadata` for more accurate results."
+                )
+            num_frames = int(total / metadata.fps * fps)
+            num_frames = min(max(num_frames, self.min_frames), self.max_frames, total)
+        if num_frames is None:
+            num_frames = min(max(total, self.min_frames), self.max_frames)
+        return np.linspace(0, total - 1, num_frames).round().astype(int)
+
+    def _preprocess(
+        self,
+        videos: list[torch.Tensor],
+        do_convert_rgb: bool = True,
+        do_resize: bool = True,
+        size: SizeDict | None = None,
+        resample: "PILImageResampling | tvF.InterpolationMode | int | None" = PILImageResampling.BICUBIC,
+        do_rescale: bool = True,
+        rescale_factor: float = 1 / 255.0,
+        do_normalize: bool = True,
+        image_mean: float | list[float] | None = None,
+        image_std: float | list[float] | None = None,
+        patch_size: int | None = None,
+        temporal_patch_size: int | None = None,
+        merge_size: int | None = None,
+        return_tensors: str | TensorType | None = None,
+        **kwargs,
+    ):
+        # 1. resize, batched per input shape
+        by_shape, order = group_videos_by_shape(videos)
+        resized = {}
+        for shape, batch in by_shape.items():
+            if do_convert_rgb:
+                batch = self.convert_to_rgb(batch)
+            n, t, c, h, w = batch.shape
+            if do_resize:
+                new_h, new_w = fit_video_to_grid(
+                    num_frames=t, height=h, width=w, temporal_factor=temporal_patch_size,
+                    factor=patch_size * merge_size, min_pixels=size.shortest_edge, max_pixels=size.longest_edge,
+                )
+                batch = self.resize(batch.view(n * t, c, h, w), size=SizeDict(height=new_h, width=new_w), resample=resample)
+                batch = batch.view(n, t, c, new_h, new_w)
+            resized[shape] = batch
+        videos = reorder_videos(resized, order)
+
+        # 2. normalise, pad the frame count to the temporal patch, cut into patches
+        by_shape, order = group_videos_by_shape(videos)
+        flat = {}
+        grids = {}
+        for shape, batch in by_shape.items():
+            new_h, new_w = get_image_size(batch[0], channel_dim=ChannelDimension.FIRST)
+            px = self.rescale_and_normalize(batch, do_rescale, rescale_factor, do_normalize, image_mean, image_std)
+            t = px.shape[1]
+            if pad := -t % temporal_patch_size:
+                px = torch.cat((px, px[:, -1:].expand(-1, pad, -1, -1, -1)), dim=1)
+            n, gt, c = px.shape[:3]
+            gt = gt // temporal_patch_size
+            gh, gw = new_h // patch_size, new_w // patch_size
+            px = px.view(
+                n, gt, temporal_patch_size, c, gh // merge_size, merge_size, patch_size, gw // merge_size, merge_size, patch_size
+            )
+            px = px.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
+            flat[shape] = px.reshape(n, gt * gh * gw, c * temporal_patch_size * patch_size * patch_size)
+            grids[shape] = [[gt, gh, gw]] * n
+
+        pixel_values_videos = torch.cat(reorder_videos(flat, order), dim=0)
+        video_grid_thw = torch.tensor(reorder_videos(grids, order))
+        return BatchFeature(
+            data={"pixel_values_videos": pixel_values_videos, "video_grid_thw": video_grid_thw}, tensor_type=return_tensors
+        )
+
+
+__all__ = ["AgnesVideoProcessor"]