Upload folder using huggingface_hub

.meta.json

@@ -0,0 +1,17 @@
+{
+  "source_model": {
+    "name": "XHToken/Spark-X2.6-4B",
+    "commit_id": "66e3ae7bee54e0a523d692cb8ef841f35409807b"
+  },
+  "env": {
+    "python": "3.12.13",
+    "packages": {
+      "huggingface_hub": "2.3.0",
+      "safetensors": "0.8.0",
+      "tokenizers": "0.23.3",
+      "torch": "2.13.0",
+      "transformers": "5.19.0"
+    },
+    "cuda": "13.0"
+  }
+}

README.md

@@ -0,0 +1,157 @@
+---
+library_name: transformers
+base_model:
+- XHToken/Spark-X2.6-4B
+---
+
+This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [XHToken/Spark-X2.6-4B](https://huggingface.co/XHToken/Spark-X2.6-4B).
+
+Structural comparison with `XHToken/Spark-X2.6-4B`:
+
+| Feature | Source | Tiny |
+|---|---:|---:|
+| Vocabulary / tied embeddings | 131,072 / yes | same |
+| Layers | 36 | 12; three 3-sliding + 1-full cycles |
+| Hidden size / head dimension | 2,560 / 256 | 16 / 32 |
+| Query / KV heads | 16 / 4 | 1 / 1 |
+| MLP width | 10,240 | 64 |
+| Sliding window | 512 | same |
+| Per-head output gate | sigmoid | same |
+| Stored precision | BF16 | BF16 |
+
+The source implementation invokes eager attention directly. The tiny definition
+retains that source behavior rather than falsely advertising an optimized
+attention backend.
+
+| File path | Size |
+|------|------|
+| model.safetensors | 4.3MB |
+
+
+### Example usage:
+
+- Transformers (✅ verified)
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_id = "tiny-random/spark-x2.6"
+device = "cuda" if torch.cuda.is_available() else "cpu"
+dtype = torch.bfloat16 if device == "cuda" else torch.float32
+# The pinned Spark implementation calls eager_attention_forward directly;
+# it exposes no SDPA/FlashAttention dispatch path to select instead.
+model = AutoModelForCausalLM.from_pretrained(model_id, dtype=dtype, trust_remote_code=True).to(device).eval()
+tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
+inputs = tokenizer("Hello", return_tensors="pt").to(device)
+output = model.generate(**inputs, do_sample=False, min_new_tokens=10, max_new_tokens=10)
+print(tokenizer.decode(output[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True))
+```
+
+### Codes to create this repo (✅ verified):
+
+<details><summary>Click to expand</summary>
+
+```python
+import json
+from pathlib import Path
+from shutil import copyfile
+
+import torch
+from huggingface_hub import hf_hub_download, list_repo_files
+from transformers import AutoConfig, AutoModelForCausalLM, set_seed
+
+source_model_id = "XHToken/Spark-X2.6-4B"
+save_folder = "/tmp/tiny-random/spark-x26"
+source_revision = "66e3ae7bee54e0a523d692cb8ef841f35409807b"
+set_seed(42)
+folder = Path(save_folder)
+folder.mkdir(parents=True, exist_ok=True)
+raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
+raw.update(
+    hidden_size=16, head_dim=32, num_attention_heads=1, num_key_value_heads=1,
+    intermediate_size=64, num_hidden_layers=12,
+    layer_types=(["sliding_attention"] * 3 + ["full_attention"]) * 3,
+)
+(folder / "config.json").write_text(json.dumps(raw, indent=2), encoding="utf-8")
+for filename in ("configuration_spark.py", "modeling_spark.py"):
+    source = Path(hf_hub_download(source_model_id, filename, revision=source_revision)).read_text(encoding="utf-8")
+    if filename == "modeling_spark.py":
+        # Spark's pinned remote implementation targets Transformers 4.57.
+        # These are API compatibility adaptations for Transformers 5.19;
+        # the attention, state-dict names, and tied-embedding semantics stay unchanged.
+        source = source.replace(
+            '_tied_weights_keys = ["lm_head.weight"]',
+            '_tied_weights_keys = {"lm_head.weight": "model.embedding.weight"}',
+        )
+        source = source.replace(
+            '"input_embeds": inputs_embeds,\n                "attention_mask"',
+            '"inputs_embeds": inputs_embeds,\n                "attention_mask"',
+        )
+        source = source.replace(
+            '                "cache_position": cache_position,\n                "past_key_values"',
+            '                "past_key_values"',
+        )
+    (folder / filename).write_text(source, encoding="utf-8")
+config = AutoConfig.from_pretrained(folder, trust_remote_code=True)
+model = AutoModelForCausalLM.from_config(config, dtype=torch.bfloat16, trust_remote_code=True)
+with torch.no_grad():
+    for name, parameter in sorted(model.named_parameters()):
+        if name.endswith("weight") and "norm" in name:
+            parameter.fill_(1)
+        else:
+            torch.nn.init.normal_(parameter, mean=0, std=0.02)
+model.save_pretrained(folder, safe_serialization=True)
+assets = {"chat_template.jinja", "generation_config.json", "merges.txt", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json", "vocab.json"}
+for filename in sorted(set(list_repo_files(source_model_id, revision=source_revision)) & assets):
+    copyfile(hf_hub_download(source_model_id, filename, revision=source_revision), folder / filename)
+```
+
+</details>
+
+### Printing the model:
+
+<details><summary>Click to expand</summary>
+
+```text
+Spark2_5ForCausalLM(
+  (model): Spark2_5Model(
+    (embedding): Embedding(131072, 16, padding_idx=2)
+    (layers): ModuleList(
+      (0-11): 12 x Spark2_5DecoderLayer(
+        (self_attn): Spark2_5Attention(
+          (q_k_v_proj): Linear(in_features=16, out_features=96, bias=False)
+          (g_proj): Linear(in_features=16, out_features=1, bias=False)
+          (out_proj): Linear(in_features=32, out_features=16, bias=False)
+        )
+        (mlp): Spark2_5MLP(
+          (gate_proj): Linear(in_features=16, out_features=64, bias=False)
+          (up_proj): Linear(in_features=16, out_features=64, bias=False)
+          (down_proj): Linear(in_features=64, out_features=16, bias=False)
+          (act_fn): GELUActivation()
+        )
+        (input_layernorm): Spark2_5RMSNorm((16,), eps=1e-06)
+        (post_attention_layernorm): Spark2_5RMSNorm((16,), eps=1e-06)
+      )
+    )
+    (norm): Spark2_5RMSNorm((16,), eps=1e-06)
+  )
+  (lm_head): Linear(in_features=16, out_features=131072, bias=False)
+)
+```
+
+</details>
+
+### Test environment:
+
+- huggingface_hub: 2.3.0
+- safetensors: 0.8.0
+- tokenizers: 0.23.3
+- torch: 2.13.0
+- transformers: 5.19.0
+
+### Change log:
+
+- 2026-10-23: Initial version.
+
+This repo is generated by GPT-6.1-Sol

chat_template.jinja

@@ -0,0 +1,121 @@
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- set enable_thinking = enable_thinking | default(true) %}
+{%- set reasoning_instructions = '' %}
+{%- if enable_thinking is true %}
+    {%- set resolved_reasoning_effort = reasoning_effort|default('high') %}
+    {%- if resolved_reasoning_effort not in ('high', 'low') %}
+        {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are high (default) and low.') }}
+    {%- endif %}
+    {%- if resolved_reasoning_effort == 'high' %}
+        {%- set reasoning_instructions = '\n`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low` and `high`.\nNow the system is invoked with `thinking_effort=high`.' %}
+    {%- elif resolved_reasoning_effort == 'low' %}
+        {%- set reasoning_instructions = '\n`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low` and `high`.\nNow the system is invoked with `thinking_effort=low`.' %}
+    {%- endif %}
+{%- endif %}
+
+{#- Render a string or a list of text blocks. -#}
+{%- macro render_content(content, context_name) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for block in content %}
+            {%- if block.type == 'text' %}
+                {{- block.text }}
+            {%- else %}
+                {{- raise_exception('Unsupported ' ~ context_name ~ ' content block type: ' ~ (block.type | string)) }}
+            {%- endif %}
+        {%- endfor %}
+    {%- else %}
+        {{- raise_exception(context_name ~ ' content must be a string or a list of text blocks') }}
+    {%- endif %}
+{%- endmacro %}
+
+{#- Default system prompt -#}
+{%- set default_system = "you are a helpful assistant." + reasoning_instructions %}
+
+{#- The first message-level system is placed in the initial system block. -#}
+{%- set ns = namespace(initial_system='') %}
+{%- if messages[0].role == "system" %}
+    {%- set ns.initial_system = render_content(messages[0].content, 'system') %}
+{%- endif %}
+
+{#- System block -#}
+{{- '<｜start▁of▁sentence｜><|System|>' + '\n' + default_system }}
+{%- if tools %}
+    {{- '\n\n' + '## Tools' + '\n' + 'You have access to the following functions:' + '\n' + '<tools>' }}
+    {%- for tool in tools %}
+        {{- '\n' + (tool.function | tojson(separators=(',', ':'))) }}
+    {%- endfor %}
+    {{- '\n' + '</tools>' }}
+{%- endif %}
+{%- if ns.initial_system %}
+    {{- '\n\n' + ns.initial_system }}
+{%- endif %}
+{{- '<｜end▁of▁sentence｜>'}}
+
+{#- Conversation turns -#}
+{%- for message in messages %}
+    {%- if message.role == "system" %}
+        {#- The first system message was consumed by the initial block. -#}
+        {%- if not loop.first %}
+            {{- '<｜start▁of▁sentence｜><|System|>\n' + render_content(message.content, 'system') + '<｜end▁of▁sentence｜>' }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<｜start▁of▁sentence｜><|User|>' + render_content(message.content, 'user') + '<｜end▁of▁sentence｜>' }}
+    {%- elif message.role == "assistant" %}
+        {%- set assistant_content = render_content(message.content, 'assistant') %}
+        {%- if message.reasoning_content is defined and message.reasoning_content %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- set reasoning_content = '' %}
+        {%- endif %}
+        {{- '<｜start▁of▁sentence｜><|Bot|>'}}
+        {%- if reasoning_content %}
+            {{- '<think>' + reasoning_content + '</think>'}}
+        {%- else %}
+            {{- '</think>' }}
+        {%- endif %}
+        {%- if assistant_content %}
+            {{- assistant_content }}
+        {%- endif %}
+        {%- if message.tool_calls is defined and message.tool_calls is not none %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function.arguments is not mapping %}
+                    {{- raise_exception('tool_call.function.arguments must be a dictionary; normalize JSON strings before apply_chat_template') }}
+                {%- endif %}
+                {%- set args = tool_call.function.arguments %}
+                {{- '<tool_call>' + tool_call.function.name }}
+                {%- for k, v in args.items() %}
+                    {{- '<arg_key>' ~ k ~ '</arg_key><arg_value>' ~ (v if v is string else v | tojson(separators=(', ', ': '))) ~ '</arg_value>' }}
+                {%- endfor %}
+                {{- '</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<｜end▁of▁sentence｜>' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem is undefined or loop.previtem.role != "tool" %}
+            {{- '<｜start▁of▁sentence｜><|Tool|>' }}
+        {%- endif %}
+        {{- '<tool_response>' ~ message.content ~ '</tool_response>' }}
+        {%- if loop.nextitem is undefined or loop.nextitem.role != "tool" %}
+            {{- '<｜end▁of▁sentence｜>' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unsupported message role: ' ~ message.role) }}
+    {%- endif %}
+{%- endfor %}
+
+{#- Generation prompt -#}
+{%- if add_generation_prompt %}
+    {{- '<｜start▁of▁sentence｜><|Bot|>' }}
+    {%- if enable_thinking is defined and enable_thinking %}
+        {{- '<think>' }}
+    {%- endif %}
+    {%- if enable_thinking is defined and not enable_thinking %}
+        {{- '</think>' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,61 @@
+{
+  "architectures": [
+    "Spark2_5ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_spark.Spark2_5Config",
+    "AutoModel": "modeling_spark.Spark2_5Model",
+    "AutoModelForCausalLM": "modeling_spark.Spark2_5ForCausalLM"
+  },
+  "bos_token_id": 0,
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "gate_attn_act_mode": "sigmoid",
+  "head_dim": 32,
+  "headwise_attn_output_gate": true,
+  "hidden_act": "gelu",
+  "hidden_size": 16,
+  "initializer_range": 0.01976,
+  "intermediate_size": 64,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 1048576,
+  "mlp_bias": false,
+  "model_type": "spark2_5",
+  "num_attention_heads": 1,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 1,
+  "pad_token_id": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "full_attention": {
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 5000000,
+      "rope_type": "default"
+    },
+    "sliding_attention": {
+      "partial_rotary_factor": 1.0,
+      "rope_theta": 10000,
+      "rope_type": "default"
+    }
+  },
+  "sliding_window": 512,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.19.0",
+  "use_cache": true,
+  "vocab_size": 131072
+}

configuration_spark.py

@@ -0,0 +1,118 @@
+# coding=utf-8
+# Copyright 2026 The XHToken team and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from transformers import PretrainedConfig
+
+
+class Spark2_5Config(PretrainedConfig):
+    model_type = "spark2_5"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_k_v_proj": "colwise",
+        "layers.*.self_attn.g_proj": "colwise",
+        "layers.*.self_attn.out_proj": "rowwise",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj": "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+    }
+    base_model_pp_plan = {
+        "embedding": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+
+    def __init__(
+        self,
+        vocab_size=32000,
+        hidden_size=4096,
+        intermediate_size=11008,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=None,
+        hidden_act="gelu",
+        max_position_embeddings=2048,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=1,
+        eos_token_id=2,
+        tie_word_embeddings=False,
+        rope_parameters=None,
+        attention_bias=False,
+        attention_dropout=0.0,
+        mlp_bias=False,
+        head_dim=None,
+        headwise_attn_output_gate=False,
+        gate_attn_act_mode="sigmoid",
+        sliding_window=None,
+        layer_types=None,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+        if num_attention_heads % num_key_value_heads != 0:
+            raise ValueError(
+                f"num_attention_heads ({num_attention_heads}) must be divisible by num_key_value_heads ({num_key_value_heads})"
+            )
+        self.num_key_value_heads = num_key_value_heads
+
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+        self.mlp_bias = mlp_bias
+        self.head_dim = head_dim if head_dim is not None else self.hidden_size // self.num_attention_heads
+        self.headwise_attn_output_gate = headwise_attn_output_gate
+        self.gate_attn_act_mode = gate_attn_act_mode
+        self.sliding_window = sliding_window
+        self.rope_parameters = rope_parameters
+
+        if layer_types is None:
+            layer_types = ["full_attention"] * num_hidden_layers
+        if len(layer_types) != num_hidden_layers:
+            raise ValueError(
+                f"layer_types length ({len(layer_types)}) must match num_hidden_layers ({num_hidden_layers})"
+            )
+        self.layer_types = layer_types
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+    def get_rope_theta(self, layer_type):
+        params = self.rope_parameters.get(layer_type, {})
+        return params.get("rope_theta", 10000)
+
+    def get_partial_rotary_factor(self, layer_type):
+        params = self.rope_parameters.get(layer_type, {})
+        return params.get("partial_rotary_factor", 1.0)
+
+
+__all__ = ["Spark2_5Config"]

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 2,
+  "max_tokens": 1048576,
+  "temperature": 1.0,
+  "top_p": 0.95, 
+  "top_k": -1,
+  "repetition_penalty": 1.0,
+  "presence_penalty":0,
+  "frequency_penalty":0,
+  "do_sample": true,
+  "transformers_version": "4.57.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:254aecb0c2a75d91b96d368caed3996123e13e782ab142dc83bb02b18a2ff473
+size 4328808

modeling_spark.py

@@ -0,0 +1,482 @@
+import math
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.generation import GenerationMixin
+from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPast,
+    CausalLMOutputWithPast,
+)
+from transformers.modeling_utils import PreTrainedModel
+from transformers.processing_utils import Unpack
+from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
+from transformers.utils import TransformersKwargs, can_return_tuple, logging
+
+from .configuration_spark import Spark2_5Config
+
+logger = logging.get_logger(__name__)
+
+_CONFIG_FOR_DOC = "Spark2_5Config"
+
+def rotate_half(x):
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def compute_rope_cos_sin(positions, head_dim, rope_theta, partial_rotary_factor=1.0, device="cpu"):
+    rope_head_dim = int(head_dim * partial_rotary_factor)
+    inv_freq = 1.0 / (rope_theta ** (torch.arange(0, rope_head_dim, 2, dtype=torch.int64).to(device="cpu", dtype=torch.float) / rope_head_dim))
+    inv_freq = inv_freq.to(device)
+    t = positions.to(device=device, dtype=torch.float32)
+    freqs = torch.outer(t, inv_freq)
+    freqs = torch.cat([freqs, freqs], dim=-1)
+    cos = freqs.cos()
+    sin = freqs.sin()
+    return cos, sin
+
+
+def apply_rotary_pos_emb(x, cos, sin):
+    rope_head_dim = cos.shape[-1]
+    x_f32 = x.float()
+    if x_f32.shape[-1] > rope_head_dim:
+        x_rot = x_f32[..., :rope_head_dim]
+        x_pass = x_f32[..., rope_head_dim:]
+        c = cos.unsqueeze(0).unsqueeze(0)
+        s = sin.unsqueeze(0).unsqueeze(0)
+        x_rot = x_rot * c + rotate_half(x_rot) * s
+        result = torch.cat([x_rot, x_pass], dim=-1)
+    else:
+        c = cos.unsqueeze(0).unsqueeze(0)
+        s = sin.unsqueeze(0).unsqueeze(0)
+        result = x_f32 * c + rotate_half(x_f32) * s
+    return result.to(x.dtype)
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+def eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: torch.Tensor | None = None,
+    scaling: float | None = None,
+    dropout: float = 0.0,
+    **kwargs: Unpack[TransformersKwargs],
+):
+    key = repeat_kv(key, module.num_key_value_groups)
+    value = repeat_kv(value, module.num_key_value_groups)
+
+    if scaling is None:
+        scaling = 1.0 / math.sqrt(query.shape[-1])
+
+    attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling
+    if attention_mask is not None:
+        causal_mask = attention_mask[:, :, :, : key.shape[-2]]
+        attn_weights = attn_weights + causal_mask
+
+    attn_weights = attn_weights - attn_weights.max(dim=-1, keepdim=True).values
+    attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+    attn_output = torch.matmul(attn_weights, value)
+    return attn_output, attn_weights
+
+
+class Spark2_5RMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return (self.weight.float() * hidden_states).to(input_dtype)
+
+    def extra_repr(self):
+        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+ALL_LAYERNORM_LAYERS.append(Spark2_5RMSNorm)
+
+
+class Spark2_5MLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias)
+        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias)
+        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.mlp_bias)
+
+        if config.hidden_act != "gelu":
+            raise ValueError(f"只支持hidden_act='gelu'，当前传入:{config.hidden_act}")
+        
+        self.act_fn = ACT2FN[config.hidden_act]
+
+    def forward(self, x):
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+
+
+class Spark2_5Attention(nn.Module):
+    def __init__(self, config: Spark2_5Config, layer_idx: int | None = None):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.attention_dropout = config.attention_dropout
+        self.hidden_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = config.head_dim
+        self.num_key_value_heads = config.num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.scaling = 1.0 / math.sqrt(self.head_dim)
+        self.headwise_attn_output_gate = config.headwise_attn_output_gate
+        self.gate_attn_act_mode = config.gate_attn_act_mode
+        self.q_dim = self.num_heads * self.head_dim
+        self.kv_dim = self.num_key_value_heads * self.head_dim
+
+        qkv_out_dim = self.q_dim + 2 * self.kv_dim
+        self.q_k_v_proj = nn.Linear(self.hidden_size, qkv_out_dim, bias=config.attention_bias)
+        self.g_proj = nn.Linear(self.hidden_size, self.num_heads, bias=config.attention_bias) if self.headwise_attn_output_gate else None
+        self.out_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.attention_bias)
+        self.sliding_window = None
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        attention_mask: torch.Tensor | None = None,
+        past_key_values: Cache | None = None,
+        cache_position: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        input_shape = hidden_states.shape[:-1]
+        bsz, seq_len = input_shape
+
+        qkv = self.q_k_v_proj(hidden_states)
+        q = qkv[..., :self.q_dim]
+        k = qkv[..., self.q_dim:self.q_dim + self.kv_dim]
+        v = qkv[..., self.q_dim + self.kv_dim:]
+        gate_score = self.g_proj(hidden_states) if self.g_proj is not None else None
+
+        q = q.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
+        k = k.view(bsz, seq_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        v = v.view(bsz, seq_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        if gate_score is not None:
+            gate_score = gate_score.view(bsz, seq_len, self.num_heads, 1).transpose(1, 2)
+
+        cos, sin = position_embeddings
+        q = apply_rotary_pos_emb(q, cos, sin)
+        k = apply_rotary_pos_emb(k, cos, sin)
+
+
+        if past_key_values is not None:
+            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
+            k, v = past_key_values.update(k, v, self.layer_idx, cache_kwargs)
+
+        attn_output, attn_weights = eager_attention_forward(
+            self, q, k, v,
+            attention_mask=attention_mask,
+            scaling=self.scaling,
+            dropout=self.attention_dropout if self.training else 0.0,
+        )
+
+        if gate_score is not None:
+            if self.gate_attn_act_mode == "sigmoid":
+                gate = torch.sigmoid(gate_score.float())
+            elif self.gate_attn_act_mode == "silu":
+                gate = F.silu(gate_score.float())
+            else:
+                raise ValueError(f"Unsupported gate_attn_act_mode: {self.gate_attn_act_mode}")
+            gate = gate.to(attn_output.dtype)
+            attn_output = attn_output * gate
+
+        attn_output = attn_output.transpose(1, 2).contiguous().view(bsz, seq_len, -1)
+        attn_output = self.out_proj(attn_output)
+
+        return attn_output, attn_weights
+
+
+class Spark2_5DecoderLayer(nn.Module):
+    def __init__(self, config: Spark2_5Config, layer_idx: int):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+
+        self.self_attn = Spark2_5Attention(config=config, layer_idx=layer_idx)
+        self.mlp = Spark2_5MLP(config)
+        self.input_layernorm = Spark2_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = Spark2_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+        self.layer_type = config.layer_types[layer_idx] if layer_idx < len(config.layer_types) else "full_attention"
+        if self.layer_type == "sliding_attention" and config.sliding_window is not None:
+            self.self_attn.sliding_window = config.sliding_window
+        else:
+            self.self_attn.sliding_window = None
+        self.self_attn.partial_rotary_factor = config.get_partial_rotary_factor(self.layer_type)
+    
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        attention_mask: torch.Tensor | None = None,
+        past_key_values: Cache | None = None,
+        cache_position: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs]
+    ) -> torch.Tensor:
+
+        residual = hidden_states
+        hidden_states = self.input_layernorm(hidden_states)
+        hidden_states = hidden_states.to(self.mlp.gate_proj.weight.dtype)
+
+        hidden_states, _ = self.self_attn(
+            hidden_states=hidden_states,
+            position_embeddings=position_embeddings,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            cache_position=cache_position,
+            position_ids=position_ids,
+        )
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = hidden_states.to(self.mlp.gate_proj.weight.dtype)
+
+
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+
+        return hidden_states
+
+
+class Spark2_5PreTrainedModel(PreTrainedModel):
+    config_class = Spark2_5Config
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Spark2_5DecoderLayer"]  # noqa: RUF012
+    _skip_keys_device_placement = ["past_key_values"]  # noqa: RUF012
+
+    def _init_weights(self, module):
+        std = self.config.initializer_range
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+
+class Spark2_5Model(Spark2_5PreTrainedModel):
+    def __init__(self, config: Spark2_5Config):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+
+        self.embedding = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList(
+            [Spark2_5DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+        )
+        self.norm = Spark2_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.gradient_checkpointing = False
+        self.has_sliding_layers = "sliding_attention" in config.layer_types
+
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embedding
+
+    def set_input_embeddings(self, value):
+        self.embedding = value
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | list[torch.FloatTensor] | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        use_cache: bool | None = None,
+        cache_position: torch.LongTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> BaseModelOutputWithPast:
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError(
+                "You cannot specify both input_ids and inputs_embeds at the same time, and must specify either one"
+            )
+
+        if self.gradient_checkpointing and self.training and use_cache:
+            logger.warning_once(
+                "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`."
+            )
+            use_cache = False
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embedding(input_ids)
+
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache(config=self.config)
+
+        if cache_position is None:
+            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+            cache_position = torch.arange(
+                past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
+            )
+
+        if position_ids is None:
+            position_ids = cache_position.unsqueeze(0)
+
+        if not isinstance(attention_mask, dict):
+            mask_kwargs = {
+                "config": self.config,
+                "inputs_embeds": inputs_embeds,
+                "attention_mask": attention_mask,
+                "past_key_values": past_key_values,
+                "position_ids": position_ids,
+            }
+            causal_mask_mapping = {
+                "full_attention": create_causal_mask(**mask_kwargs),
+            }
+            if self.has_sliding_layers:
+                causal_mask_mapping["sliding_attention"] = create_sliding_window_causal_mask(**mask_kwargs)
+        else:
+            causal_mask_mapping = attention_mask
+
+        hidden_states = inputs_embeds.float()
+
+        device = hidden_states.device
+        dtype = self.embedding.weight.dtype
+
+        head_dim = self.config.head_dim
+        rope_cache = {}
+        for lt in set(self.config.layer_types):
+            rope_theta = self.config.get_rope_theta(lt)
+            prf = self.config.get_partial_rotary_factor(lt)
+            cos, sin = compute_rope_cos_sin(cache_position, head_dim, rope_theta, partial_rotary_factor=prf, device=device)
+            rope_cache[lt] = (cos, sin)
+
+        for decoder_layer in self.layers:
+            layer_type = decoder_layer.layer_type
+            position_embeddings = rope_cache.get(layer_type, rope_cache.get("full_attention"))
+            layer_attention_mask = causal_mask_mapping.get(layer_type, causal_mask_mapping.get("full_attention"))
+
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    decoder_layer.__call__,
+                    hidden_states,
+                    position_embeddings,
+                    layer_attention_mask,
+                )
+                hidden_states = layer_outputs[0] if isinstance(layer_outputs, tuple) else layer_outputs
+            else:
+                hidden_states = decoder_layer(
+                    hidden_states,
+                    position_embeddings=position_embeddings,
+                    attention_mask=layer_attention_mask,
+                    past_key_values=past_key_values,
+                    cache_position=cache_position,
+                    position_ids=position_ids,
+                )
+
+        hidden_states = self.norm(hidden_states)
+        hidden_states = hidden_states.to(dtype)
+
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=past_key_values if use_cache else None,
+        )
+
+class Spark2_5ForCausalLM(Spark2_5PreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {"lm_head.weight": "model.embedding.weight"}  # noqa: RUF012
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = Spark2_5Model(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embedding
+
+    def set_input_embeddings(self, value):
+        self.model.embedding = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def set_decoder(self, decoder):
+        self.model = decoder
+
+    def get_decoder(self):
+        return self.model
+
+    @can_return_tuple
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | list[torch.FloatTensor] | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        use_cache: bool | None = None,
+        cache_position: torch.LongTensor | None = None,
+        logits_to_keep: int = 0,
+        token_type_ids: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> CausalLMOutputWithPast:
+        outputs: BaseModelOutputWithPast = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            cache_position=cache_position,
+        )
+
+        hidden_states = outputs.last_hidden_state
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        hidden_states = hidden_states[:, slice_indices, :]
+
+        if self.config.tie_word_embeddings:
+            embed_weight = self.model.embedding.weight
+            logits = F.linear(hidden_states, embed_weight)
+        else:
+            logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+__all__ = ["Spark2_5Config", "Spark2_5ForCausalLM", "Spark2_5Model"]

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<｜start▁of▁sentence｜>",
+  "eos_token": "<｜end▁of▁sentence｜>",
+  "unk_token": "<unk>",
+  "pad_token": "<｜▁pad▁｜>"
+}

tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<｜start▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "legacy": true,
+  "model_max_length": 131072,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<｜▁pad▁｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sp_model_kwargs": {},
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "chat_template": "{%- if not messages %}{{- raise_exception('No messages provided.') }}{%- endif %}{%- set enable_thinking = enable_thinking | default(true) %}{%- set reasoning_instructions = '' %}{%- if enable_thinking is true %}{%- set resolved_reasoning_effort = reasoning_effort|default('high') %}{%- if resolved_reasoning_effort not in ('high', 'low') %}{{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are high (default) and low.') }}{%- endif %}{%- if resolved_reasoning_effort == 'high' %}{%- set reasoning_instructions = '\n`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low` and `high`.\nNow the system is invoked with `thinking_effort=high`.' %}{%- elif resolved_reasoning_effort == 'low' %}{%- set reasoning_instructions = '\n`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low` and `high`.\nNow the system is invoked with `thinking_effort=low`.' %}{%- endif %}{%- endif %}{#- Render a string or a list of text blocks. -#}{%- macro render_content(content, context_name) %}{%- if content is string %}{{- content }}{%- elif content is none or content is undefined %}{{- '' }}{%- elif content is iterable and content is not mapping %}{%- for block in content %}{%- if block.type == 'text' %}{{- block.text }}{%- else %}{{- raise_exception('Unsupported ' ~ context_name ~ ' content block type: ' ~ (block.type | string)) }}{%- endif %}{%- endfor %}{%- else %}{{- raise_exception(context_name ~ ' content must be a string or a list of text blocks') }}{%- endif %}{%- endmacro %}{#- Default system prompt -#}{%- set default_system = \"you are a helpful assistant.\" + reasoning_instructions %}{#- The first message-level system is placed in the initial system block. -#}{%- set ns = namespace(initial_system='') %}{%- if messages[0].role == \"system\" %}{%- set ns.initial_system = render_content(messages[0].content, 'system') %}{%- endif %}{#- System block -#}{{- '<｜start▁of▁sentence｜><|System|>' + '\n' + default_system }}{%- if tools %}{{- '\n\n' + '## Tools' + '\n' + 'You have access to the following functions:' + '\n' + '<tools>' }}{%- for tool in tools %}{{- '\n' + (tool.function | tojson(separators=(',', ':'))) }}{%- endfor %}{{- '\n' + '</tools>' }}{%- endif %}{%- if ns.initial_system %}{{- '\n\n' + ns.initial_system }}{%- endif %}{{- '<｜end▁of▁sentence｜>'}}{#- Conversation turns -#}{%- for message in messages %}{%- if message.role == \"system\" %}{#- The first system message was consumed by the initial block. -#}{%- if not loop.first %}{{- '<｜start▁of▁sentence｜><|System|>\n' + render_content(message.content, 'system') + '<｜end▁of▁sentence｜>' }}{%- endif %}{%- elif message.role == \"user\" %}{{- '<｜start▁of▁sentence｜><|User|>' + render_content(message.content, 'user') + '<｜end▁of▁sentence｜>' }}{%- elif message.role == \"assistant\" %}{%- set assistant_content = render_content(message.content, 'assistant') %}{%- if message.reasoning_content is defined and message.reasoning_content %}{%- set reasoning_content = message.reasoning_content %}{%- else %}{%- set reasoning_content = '' %}{%- endif %}{{- '<｜start▁of▁sentence｜><|Bot|>'}}{%- if reasoning_content %}{{- '<think>' + reasoning_content + '</think>'}}{%- else %}{{- '</think>' }}{%- endif %}{%- if assistant_content %}{{- assistant_content }}{%- endif %}{%- if message.tool_calls is defined and message.tool_calls is not none %}{%- for tool_call in message.tool_calls %}{%- if tool_call.function.arguments is not mapping %}{{- raise_exception('tool_call.function.arguments must be a dictionary; normalize JSON strings before apply_chat_template') }}{%- endif %}{%- set args = tool_call.function.arguments %}{{- '<tool_call>' + tool_call.function.name }}{%- for k, v in args.items() %}{{- '<arg_key>' ~ k ~ '</arg_key><arg_value>' ~ (v if v is string else v | tojson(separators=(', ', ': '))) ~ '</arg_value>' }}{%- endfor %}{{- '</tool_call>' }}{%- endfor %}{%- endif %}{{- '<｜end▁of▁sentence｜>' }}{%- elif message.role == \"tool\" %}{%- if loop.previtem is undefined or loop.previtem.role != \"tool\" %}{{- '<｜start▁of▁sentence｜><|Tool|>' }}{%- endif %}{{- '<tool_response>' ~ message.content ~ '</tool_response>' }}{%- if loop.nextitem is undefined or loop.nextitem.role != \"tool\" %}{{- '<｜end▁of▁sentence｜>' }}{%- endif %}{%- else %}{{- raise_exception('Unsupported message role: ' ~ message.role) }}{%- endif %}{%- endfor %}{#- Generation prompt -#}{%- if add_generation_prompt %}{{- '<｜start▁of▁sentence｜><|Bot|>' }}{%- if enable_thinking is defined and enable_thinking %}{{- '<think>' }}{%- endif %}{%- if enable_thinking is defined and not enable_thinking %}{{- '</think>' }}{%- endif %}{%- endif %}"
+}