Download modeling_maxwell_moe.py from maxwell-model/Maxwell-V1-Flash: direct link, hf CLI and curl.
- Browser
- Download file 93.9 kB
-
https://huggingface.co/maxwell-model/Maxwell-V1-Flash/resolve/main/modeling_maxwell_moe.py
- Command line
-
hf download hf://maxwell-model/Maxwell-V1-Flash/modeling_maxwell_moe.py
-
curl -L -o modeling_maxwell_moe.py https://huggingface.co/maxwell-model/Maxwell-V1-Flash/resolve/main/modeling_maxwell_moe.py
93.9 kB
# coding=utf-8
# Copyright 2026 the Maxwell team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""PyTorch MaxwellMoE model — w35a3 variant.
Supports: mHC, QK LayerNorm, expert bias, sigmoid routing, learned HC head,
mixed dense/MoE layers, shared experts.
"""
import math
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union
import torch
import torch.nn.functional as F
import torch.utils.checkpoint
from torch import nn
from transformers.activations import ACT2FN
from transformers.cache_utils import Cache, DynamicCache, StaticCache
from transformers.generation import GenerationMixin
from transformers.modeling_attn_mask_utils import AttentionMaskConverter
from transformers.modeling_outputs import (
MoeModelOutputWithPast,
ModelOutput,
)
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
from transformers.modeling_utils import PreTrainedModel
from transformers.utils import (
add_start_docstrings,
add_start_docstrings_to_model_forward,
is_flash_attn_2_available,
logging,
replace_return_docstrings,
)
try:
from transformers.utils import is_flash_attn_greater_or_equal_2_10
except ImportError:
try:
from transformers.utils import is_flash_attn_greater_or_equal
def is_flash_attn_greater_or_equal_2_10():
return is_flash_attn_greater_or_equal("2.1.0")
except ImportError:
def is_flash_attn_greater_or_equal_2_10():
return True
from .configuration_maxwell_moe import MaxwellMoeConfig
if is_flash_attn_2_available():
from transformers.modeling_flash_attention_utils import _flash_attention_forward
logger = logging.get_logger(__name__)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
class MaxwellMoeRMSNorm(nn.Module):
def __init__(self, hidden_size, eps=1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, hidden_states):
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
# Apply the weight in fp32 and round once, as TE RMSNorm does. Rounding
# before the multiply is visible at large norm weights (q/k norms of the
# last layer reach ~14) where attention logits run into the thousands.
return (self.weight.float() * hidden_states).to(input_dtype)
class MaxwellMoeRotaryEmbedding(nn.Module):
def __init__(self, dim=None, max_position_embeddings=2048, base=10000,
device=None, scaling_factor=1.0, rope_type="default",
config: Optional[MaxwellMoeConfig] = None):
super().__init__()
self.rope_kwargs = {}
if config is None:
self.rope_kwargs = {
"rope_type": rope_type, "factor": scaling_factor,
"dim": dim, "base": base,
"max_position_embeddings": max_position_embeddings,
}
self.rope_type = rope_type
self.max_seq_len_cached = max_position_embeddings
self.original_max_seq_len = max_position_embeddings
else:
if config.rope_scaling is not None:
self.rope_type = config.rope_scaling.get(
"rope_type", config.rope_scaling.get("type"))
else:
self.rope_type = "default"
self.max_seq_len_cached = config.max_position_embeddings
self.original_max_seq_len = config.max_position_embeddings
self.config = config
self._rope_base = base
self._rope_dim = dim
if self.rope_type == "default":
if config is not None:
self._rope_base = config.rope_theta
self._rope_dim = (config.attention_head_size
if config.attention_head_size
else config.hidden_size // config.num_attention_heads)
else:
self._rope_base = self.rope_kwargs.get("base", 10000)
self._rope_dim = self.rope_kwargs.get("dim", 64)
inv_freq = self._compute_default_inv_freq(device)
self.attention_scaling = 1.0
else:
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
inv_freq, self.attention_scaling = self.rope_init_fn(
self.config, device, **self.rope_kwargs)
self.register_buffer("inv_freq", inv_freq, persistent=False)
self.original_inv_freq = self.inv_freq
def _compute_default_inv_freq(self, device=None):
return 1.0 / (self._rope_base ** (
torch.arange(0, self._rope_dim, 2, dtype=torch.int64).float().to(device)
/ self._rope_dim))
def _inv_freq_is_valid(self):
if self.inv_freq.numel() == 0:
return False
expected = self._compute_default_inv_freq(self.inv_freq.device)
if self.inv_freq.shape != expected.shape:
return False
if not torch.isfinite(self.inv_freq).all():
return False
if (self.inv_freq <= 0).any():
return False
return torch.allclose(self.inv_freq.float(), expected.float(), rtol=1e-5, atol=1e-8)
def _reset_inv_freq(self, device):
inv_freq = self._compute_default_inv_freq(device)
self.register_buffer("inv_freq", inv_freq, persistent=False)
self.original_inv_freq = self.inv_freq
def _dynamic_frequency_update(self, position_ids, device):
seq_len = torch.max(position_ids) + 1
if seq_len > self.max_seq_len_cached:
if hasattr(self, 'rope_init_fn'):
inv_freq, self.attention_scaling = self.rope_init_fn(
self.config, device, seq_len=seq_len, **self.rope_kwargs)
self.register_buffer("inv_freq", inv_freq, persistent=False)
self.max_seq_len_cached = seq_len
if (seq_len < self.original_max_seq_len
and self.max_seq_len_cached > self.original_max_seq_len):
self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)
self.max_seq_len_cached = self.original_max_seq_len
@torch.no_grad()
def forward(self, x, position_ids):
if "dynamic" in self.rope_type:
self._dynamic_frequency_update(position_ids, device=x.device)
if self.rope_type == "default" and not self._inv_freq_is_valid():
self._reset_inv_freq(x.device)
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(
position_ids.shape[0], -1, 1)
position_ids_expanded = position_ids[:, None, :].float()
device_type = x.device.type
device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
with torch.autocast(device_type=device_type, enabled=False):
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
emb = torch.cat((freqs, freqs), dim=-1)
cos = emb.cos() * self.attention_scaling
sin = emb.sin() * self.attention_scaling
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
def rotate_half(x):
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2 :]
return torch.cat((-x2, x1), dim=-1)
def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
cos = cos.unsqueeze(unsqueeze_dim)
sin = sin.unsqueeze(unsqueeze_dim)
q_embed = (q * cos) + (rotate_half(q) * sin)
k_embed = (k * cos) + (rotate_half(k) * sin)
return q_embed, k_embed
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
batch, num_key_value_heads, slen, head_dim = hidden_states.shape
if n_rep == 1:
return hidden_states
hidden_states = hidden_states[:, :, None, :, :].expand(
batch, num_key_value_heads, n_rep, slen, head_dim)
return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
# ---------------------------------------------------------------------------
# MLP
# ---------------------------------------------------------------------------
class MaxwellMoeMLP(nn.Module):
def __init__(self, config, intermediate_size=None):
super().__init__()
self.hidden_size = config.hidden_size
self.intermediate_size = intermediate_size
self.w1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
self.w3 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
self.w2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
self.act_fn = ACT2FN[config.hidden_act]
def forward(self, x):
return self.w2(self.act_fn(self.w1(x)) * self.w3(x))
# ---------------------------------------------------------------------------
# Attention with QK LayerNorm
# ---------------------------------------------------------------------------
class MaxwellMoeAttention(nn.Module):
def __init__(self, config: MaxwellMoeConfig, layer_idx: Optional[int] = None):
super().__init__()
self.config = config
self.layer_idx = layer_idx
self.hidden_size = config.hidden_size
self.num_heads = config.num_attention_heads
self.head_dim = (config.attention_head_size
if config.attention_head_size
else self.hidden_size // self.num_heads)
self.num_key_value_heads = config.num_key_value_heads
self.num_key_value_groups = self.num_heads // self.num_key_value_heads
self.max_position_embeddings = config.max_position_embeddings
self.rope_theta = config.rope_theta
self.is_causal = True
self.attention_dropout = config.attention_dropout
self.qk_layernorm = getattr(config, 'qk_layernorm', False)
self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
if self.qk_layernorm:
self.q_layernorm = MaxwellMoeRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_layernorm = MaxwellMoeRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.rotary_emb = MaxwellMoeRotaryEmbedding(config=self.config)
def forward(
self, hidden_states, attention_mask=None, position_ids=None,
past_key_value=None, output_attentions=False, use_cache=False,
cache_position=None, position_embeddings=None,
):
bsz, q_len, _ = hidden_states.size()
query_states = self.q_proj(hidden_states)
key_states = self.k_proj(hidden_states)
value_states = self.v_proj(hidden_states)
query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim)
key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim)
if self.qk_layernorm:
query_states = self.q_layernorm(query_states)
key_states = self.k_layernorm(key_states)
query_states = query_states.transpose(1, 2)
key_states = key_states.transpose(1, 2)
value_states = value_states.view(
bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
if position_embeddings is None:
cos, sin = self.rotary_emb(value_states, position_ids)
else:
cos, sin = position_embeddings
query_states, key_states = apply_rotary_pos_emb(
query_states, key_states, cos, sin, position_ids)
if past_key_value is not None:
cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
key_states, value_states = past_key_value.update(
key_states, value_states, self.layer_idx, cache_kwargs)
key_states = repeat_kv(key_states, self.num_key_value_groups)
value_states = repeat_kv(value_states, self.num_key_value_groups)
attn_weights = torch.matmul(
query_states.float(), key_states.float().transpose(2, 3)) / math.sqrt(self.head_dim)
if attention_mask is not None:
causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
attn_weights = attn_weights + causal_mask
attn_weights = nn.functional.softmax(
attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
attn_weights = nn.functional.dropout(
attn_weights, p=self.attention_dropout, training=self.training)
attn_output = torch.matmul(attn_weights, value_states)
attn_output = attn_output.transpose(1, 2).contiguous()
attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.head_dim)
attn_output = self.o_proj(attn_output)
if not output_attentions:
attn_weights = None
return attn_output, attn_weights, past_key_value
class MaxwellMoeBlockSparseGQAIndexer(nn.Module):
"""Per-group index branch used by the Block Sparse GQA model."""
def __init__(self, config: MaxwellMoeConfig):
super().__init__()
self.num_groups = config.block_sparse_gqa_indexer_n_heads
self.index_head_dim = config.block_sparse_gqa_indexer_head_dim
self.index_softmax_scale = self.index_head_dim ** -0.5
self.linear_q_proj = nn.Linear(
config.hidden_size,
self.num_groups * self.index_head_dim,
bias=False,
)
self.linear_k_proj = nn.Linear(
config.hidden_size,
self.index_head_dim,
bias=False,
)
self.q_layernorm = (
MaxwellMoeRMSNorm(
self.index_head_dim,
eps=config.rms_norm_eps,
)
if config.qk_layernorm
else None
)
self.k_layernorm = (
MaxwellMoeRMSNorm(
self.index_head_dim,
eps=config.rms_norm_eps,
)
if config.qk_layernorm
else None
)
def project_qk(self, hidden_states, cos, sin):
"""Project detached hidden states and reuse the main branch RoPE."""
hidden_states = hidden_states.detach()
batch_size, query_len, _ = hidden_states.shape
query = self.linear_q_proj(hidden_states).view(
batch_size,
query_len,
self.num_groups,
self.index_head_dim,
)
key = self.linear_k_proj(hidden_states).view(
batch_size,
query_len,
1,
self.index_head_dim,
)
if self.q_layernorm is not None:
query = self.q_layernorm(query)
if self.k_layernorm is not None:
key = self.k_layernorm(key)
query = query.transpose(1, 2)
key = key.transpose(1, 2)
query, key = apply_rotary_pos_emb(query, key, cos, sin)
return query, key.squeeze(1)
class MaxwellMoeBlockSparseGQACache(DynamicCache):
"""Dynamic main K/V cache plus the post-RoPE shared indexer K."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.indexer_key_states = []
def _ensure_indexer_layer(self, layer_idx):
while len(self.indexer_key_states) <= layer_idx:
self.indexer_key_states.append(None)
def update_indexer_key_state(self, key_states, layer_idx):
self._ensure_indexer_layer(layer_idx)
previous = self.indexer_key_states[layer_idx]
if previous is None:
updated = key_states
else:
updated = torch.cat(
(
previous.to(
device=key_states.device,
dtype=key_states.dtype,
),
key_states,
),
dim=1,
)
self.indexer_key_states[layer_idx] = updated
return updated
def get_indexer_seq_length(self, layer_idx=0):
if (
layer_idx >= len(self.indexer_key_states)
or self.indexer_key_states[layer_idx] is None
):
return 0
return int(self.indexer_key_states[layer_idx].shape[1])
def reorder_cache(self, beam_idx):
result = super().reorder_cache(beam_idx)
for layer_idx, state in enumerate(self.indexer_key_states):
if state is not None:
self.indexer_key_states[layer_idx] = state.index_select(
0, beam_idx.to(state.device)
)
return result
def batch_repeat_interleave(self, repeats):
result = super().batch_repeat_interleave(repeats)
for layer_idx, state in enumerate(self.indexer_key_states):
if state is not None:
self.indexer_key_states[layer_idx] = state.repeat_interleave(
repeats, dim=0
)
return result
def batch_select_indices(self, indices):
result = super().batch_select_indices(indices)
for layer_idx, state in enumerate(self.indexer_key_states):
if state is not None:
self.indexer_key_states[layer_idx] = state.index_select(
0, indices.to(state.device)
)
return result
def crop(self, max_length):
result = super().crop(max_length)
retained_length = self.get_seq_length()
for layer_idx, state in enumerate(self.indexer_key_states):
if state is not None:
self.indexer_key_states[layer_idx] = state[:, :retained_length]
return result
def to_legacy_cache(self):
raise ValueError(
"block_sparse_gqa cannot export a legacy cache because that format "
"cannot retain indexer K"
)
@classmethod
def from_legacy_cache(cls, past_key_values=None):
if past_key_values:
raise ValueError(
"block_sparse_gqa cannot restore a non-empty legacy cache "
"because historical indexer K is absent"
)
return cls()
class MaxwellMoeBlockSparseGQAAttention(MaxwellMoeAttention):
"""Eager block-sparse attention used by every backbone decoder layer.
Combines block-aligned prefix/local support with per-group block-max
remote selection. The MTP decoder uses its separate attention class.
"""
def __init__(self, config: MaxwellMoeConfig, layer_idx: Optional[int] = None):
super().__init__(config, layer_idx)
if layer_idx is None:
raise ValueError("block_sparse_gqa attention requires layer_idx")
self.indexer = MaxwellMoeBlockSparseGQAIndexer(config)
self.block_size = config.block_sparse_gqa_block_size
self.prefix_window_size = (
config.block_sparse_gqa_prefix_window_size
)
self.local_window_size = config.block_sparse_gqa_local_window_size
self.num_remote_blocks = config.block_sparse_gqa_num_remote_blocks
self.num_prefix_blocks = self.prefix_window_size // self.block_size
self.num_local_blocks = self.local_window_size // self.block_size
self.query_chunk_size = config.block_sparse_gqa_query_chunk_size
self.softmax_scale = self.head_dim ** -0.5
@staticmethod
def _query_positions(
*, batch_size, query_len, key_len, device, cache_position
):
if cache_position is None:
positions = torch.arange(
key_len - query_len,
key_len,
device=device,
dtype=torch.long,
)
else:
positions = cache_position.to(
device=device, dtype=torch.long
).reshape(-1)
if positions.numel() != query_len:
raise ValueError(
f"cache_position has {positions.numel()} entries for "
f"query_len={query_len}"
)
return positions.unsqueeze(0).expand(batch_size, -1)
@staticmethod
def _allowed_keys(*, query_positions, key_len, attention_mask):
key_positions = torch.arange(key_len, device=query_positions.device)
allowed = (
key_positions.view(1, 1, key_len)
<= query_positions.unsqueeze(-1)
)
if attention_mask is None:
return allowed
if attention_mask.dim() != 4 or attention_mask.dtype == torch.bool:
raise ValueError(
"block_sparse_gqa attention expects the model-level 4D "
"additive causal mask"
)
mask = attention_mask[
:, 0, -query_positions.shape[-1]:, :key_len
]
return allowed & (mask >= 0)
def _select_blocks(self, index_query, index_key, query_positions, allowed):
"""Select an independent block-max remote route for every GQA group."""
batch_size, num_groups, query_len, _ = index_query.shape
key_len = index_key.shape[1]
num_blocks = (key_len + self.block_size - 1) // self.block_size
selected_count = min(max(self.num_remote_blocks, 1), num_blocks)
block_starts = torch.arange(
num_blocks,
device=index_query.device,
) * self.block_size
block_ends = block_starts + self.block_size
selected_chunks = []
for query_start in range(0, query_len, self.query_chunk_size):
query_end = min(
query_start + self.query_chunk_size,
query_len,
)
chunk_len = query_end - query_start
query_chunk = index_query[
:, :, query_start:query_end, :
]
per_group_block_scores = torch.full(
(
batch_size,
num_groups,
chunk_len,
num_blocks,
),
float("-inf"),
dtype=torch.float32,
device=index_query.device,
)
# Match the Megatron max-route oracle while bounding the eager
# token-score temporary for 256K contexts.
key_blocks_per_chunk = max(
1,
8192 // self.block_size,
)
for block_start in range(
0,
num_blocks,
key_blocks_per_chunk,
):
block_end = min(
block_start + key_blocks_per_chunk,
num_blocks,
)
key_start = block_start * self.block_size
key_end = min(
block_end * self.block_size,
key_len,
)
token_scores = torch.einsum(
"bgqd,bkd->bgqk",
query_chunk.float(),
index_key[:, key_start:key_end].float(),
)
token_scores.mul_(self.indexer.index_softmax_scale)
token_scores.masked_fill_(
~allowed[
:,
None,
query_start:query_end,
key_start:key_end,
],
float("-inf"),
)
block_count = block_end - block_start
padding = (
block_count * self.block_size
- (key_end - key_start)
)
if padding:
token_scores = F.pad(
token_scores,
(0, padding),
value=float("-inf"),
)
per_group_block_scores[
...,
block_start:block_end,
] = token_scores.view(
batch_size,
num_groups,
chunk_len,
block_count,
self.block_size,
).amax(dim=-1)
route_scores = per_group_block_scores.permute(
0, 2, 1, 3
).contiguous()
current_block = torch.div(
query_positions[:, query_start:query_end],
self.block_size,
rounding_mode="floor",
)
local_start_block = (
current_block - self.num_local_blocks + 1
).clamp(min=0)
local_start = local_start_block * self.block_size
eligible = (
block_starts.view(1, 1, -1)
>= self.prefix_window_size
) & (
block_ends.view(1, 1, -1)
<= local_start.unsqueeze(-1)
)
route_scores.masked_fill_(
~eligible.unsqueeze(2),
float("-inf"),
)
values, indices = route_scores.topk(
selected_count,
dim=-1,
)
indices = torch.where(
torch.isfinite(values),
indices,
-1,
)
if selected_count < self.num_remote_blocks:
indices = F.pad(
indices,
(0, self.num_remote_blocks - selected_count),
value=-1,
)
selected_chunks.append(indices)
return torch.cat(selected_chunks, dim=1).contiguous()
def _build_block_aligned_route(self, remote_blocks, query_positions):
"""Add the same fixed prefix/local blocks to every group route."""
if remote_blocks.ndim != 4:
raise ValueError(
"per-group remote blocks must be [batch, query, groups, remote]"
)
batch_size, query_len, num_groups, _ = remote_blocks.shape
current_block = torch.div(
query_positions,
self.block_size,
rounding_mode="floor",
).unsqueeze(-1).unsqueeze(-1)
prefix = torch.arange(
self.num_prefix_blocks,
device=remote_blocks.device,
dtype=remote_blocks.dtype,
).view(1, 1, 1, -1)
prefix = prefix.expand(
batch_size, query_len, num_groups, -1
)
prefix = torch.where(prefix <= current_block, prefix, -1)
local_offsets = torch.arange(
1 - self.num_local_blocks,
1,
device=remote_blocks.device,
dtype=remote_blocks.dtype,
).view(1, 1, 1, -1)
local = current_block.to(remote_blocks.dtype) + local_offsets
local = local.expand(
batch_size, query_len, num_groups, -1
)
local = torch.where(local >= self.num_prefix_blocks, local, -1)
return torch.cat(
(prefix, remote_blocks, local),
dim=-1,
).contiguous()
def _sparse_attention(
self,
query_states,
key_states,
value_states,
selected_blocks,
allowed,
):
batch_size, num_query_heads, query_len, head_dim = (
query_states.shape
)
num_groups = key_states.shape[1]
key_len = key_states.shape[2]
if selected_blocks.ndim != 4 or selected_blocks.shape[:3] != (
batch_size,
query_len,
num_groups,
):
raise ValueError(
"selected blocks must be [batch, query, groups, blocks]"
)
heads_per_group = num_query_heads // num_groups
selected_tokens = selected_blocks.shape[-1] * self.block_size
block_offsets = torch.arange(
self.block_size,
device=query_states.device,
)
output = torch.empty(
batch_size,
num_query_heads,
query_len,
head_dim,
dtype=value_states.dtype,
device=value_states.device,
)
for query_start in range(0, query_len, self.query_chunk_size):
query_end = min(
query_start + self.query_chunk_size,
query_len,
)
chunk_len = query_end - query_start
selected_chunk = selected_blocks[
:, query_start:query_end
].permute(0, 2, 1, 3)
token_positions = (
selected_chunk.unsqueeze(-1)
* self.block_size
+ block_offsets
).reshape(
batch_size,
num_groups,
chunk_len,
selected_tokens,
)
in_range = (
(token_positions >= 0)
& (token_positions < key_len)
)
gather_positions = token_positions.clamp(
min=0,
max=key_len - 1
)
allowed_chunk = allowed[
:, None, query_start:query_end
].expand(
batch_size,
num_groups,
chunk_len,
key_len,
)
valid = in_range & torch.gather(
allowed_chunk,
3,
gather_positions,
)
gather_index = gather_positions[..., None].expand(
batch_size,
num_groups,
chunk_len,
selected_tokens,
head_dim,
)
selected_key = torch.gather(
key_states.unsqueeze(2).expand(
batch_size,
num_groups,
chunk_len,
key_len,
head_dim,
),
3,
gather_index,
)
selected_value = torch.gather(
value_states.unsqueeze(2).expand(
batch_size,
num_groups,
chunk_len,
key_len,
head_dim,
),
3,
gather_index,
)
grouped_query = query_states[
:, :, query_start:query_end
].reshape(
batch_size,
num_groups,
heads_per_group,
chunk_len,
head_dim,
)
# Accumulate scores in fp32 like the Triton kernels; a bf16 einsum
# output quantizes logits of ~1e3 to steps of several units.
scores = torch.einsum(
"bghqd,bgqkd->bghqk",
grouped_query.float(),
selected_key.float(),
)
scores.mul_(self.softmax_scale)
scores.masked_fill_(
~valid[:, :, None],
float("-inf"),
)
probabilities = torch.softmax(
scores,
dim=-1,
dtype=torch.float32,
)
probabilities = torch.nan_to_num(
probabilities,
nan=0.0,
).to(selected_value.dtype)
context = torch.einsum(
"bghqk,bgqkd->bghqd",
probabilities,
selected_value,
)
output[
:, :, query_start:query_end
] = context.reshape(
batch_size,
num_query_heads,
chunk_len,
head_dim,
)
return output
def forward(
self, hidden_states, attention_mask=None, position_ids=None,
past_key_value=None, output_attentions=False, use_cache=False,
cache_position=None, position_embeddings=None,
):
if output_attentions:
raise ValueError(
"block_sparse_gqa does not materialize dense attention weights"
)
batch_size, query_len, _ = hidden_states.shape
query_states = self.q_proj(hidden_states).view(
batch_size,
query_len,
self.num_heads,
self.head_dim,
)
key_states = self.k_proj(hidden_states).view(
batch_size,
query_len,
self.num_key_value_heads,
self.head_dim,
)
value_states = self.v_proj(hidden_states).view(
batch_size,
query_len,
self.num_key_value_heads,
self.head_dim,
)
if self.qk_layernorm:
query_states = self.q_layernorm(query_states)
key_states = self.k_layernorm(key_states)
query_states = query_states.transpose(1, 2)
key_states = key_states.transpose(1, 2)
value_states = value_states.transpose(1, 2)
if position_embeddings is None:
cos, sin = self.rotary_emb(value_states, position_ids)
else:
cos, sin = position_embeddings
query_states, key_states = apply_rotary_pos_emb(
query_states,
key_states,
cos,
sin,
position_ids,
)
index_query, index_key = self.indexer.project_qk(
hidden_states,
cos,
sin,
)
if past_key_value is not None:
if not isinstance(
past_key_value,
MaxwellMoeBlockSparseGQACache,
):
raise TypeError(
"block_sparse_gqa requires "
"MaxwellMoeBlockSparseGQACache"
)
cache_kwargs = {
"sin": sin,
"cos": cos,
"cache_position": cache_position,
}
key_states, value_states = past_key_value.update(
key_states,
value_states,
self.layer_idx,
cache_kwargs,
)
index_key = past_key_value.update_indexer_key_state(
index_key,
self.layer_idx,
)
if key_states.shape[-2] != index_key.shape[1]:
raise RuntimeError(
"main KV and indexer-K cache lengths diverged: "
f"{key_states.shape[-2]} != {index_key.shape[1]}"
)
query_positions = self._query_positions(
batch_size=batch_size,
query_len=query_len,
key_len=key_states.shape[-2],
device=query_states.device,
cache_position=cache_position,
)
allowed = self._allowed_keys(
query_positions=query_positions,
key_len=key_states.shape[-2],
attention_mask=attention_mask,
)
selected_blocks = self._select_blocks(
index_query,
index_key,
query_positions,
allowed,
)
route_blocks = self._build_block_aligned_route(
selected_blocks,
query_positions,
)
attention_output = self._sparse_attention(
query_states,
key_states,
value_states,
route_blocks,
allowed,
)
attention_output = attention_output.transpose(
1, 2
).contiguous().reshape(
batch_size,
query_len,
self.num_heads * self.head_dim,
)
return self.o_proj(attention_output), None, past_key_value
class MaxwellMoeFlashAttention2(MaxwellMoeAttention):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
def forward(
self, hidden_states, attention_mask=None, position_ids=None,
past_key_value=None, output_attentions=False, use_cache=False,
cache_position=None, position_embeddings=None,
):
bsz, q_len, _ = hidden_states.size()
query_states = self.q_proj(hidden_states)
key_states = self.k_proj(hidden_states)
value_states = self.v_proj(hidden_states)
query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim)
key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim)
if self.qk_layernorm:
query_states = self.q_layernorm(query_states)
key_states = self.k_layernorm(key_states)
query_states = query_states.transpose(1, 2)
key_states = key_states.transpose(1, 2)
value_states = value_states.view(
bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
if position_embeddings is None:
cos, sin = self.rotary_emb(value_states, position_ids)
else:
cos, sin = position_embeddings
query_states, key_states = apply_rotary_pos_emb(
query_states, key_states, cos, sin)
if past_key_value is not None:
cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
key_states, value_states = past_key_value.update(
key_states, value_states, self.layer_idx, cache_kwargs)
key_states = repeat_kv(key_states, self.num_key_value_groups)
value_states = repeat_kv(value_states, self.num_key_value_groups)
dropout_rate = 0.0 if not self.training else self.attention_dropout
input_dtype = query_states.dtype
if input_dtype == torch.float32:
if torch.is_autocast_enabled():
target_dtype = torch.get_autocast_gpu_dtype()
elif hasattr(self.config, "_pre_quantization_dtype"):
target_dtype = self.config._pre_quantization_dtype
else:
target_dtype = self.q_proj.weight.dtype
query_states = query_states.to(target_dtype)
key_states = key_states.to(target_dtype)
value_states = value_states.to(target_dtype)
query_states = query_states.transpose(1, 2)
key_states = key_states.transpose(1, 2)
value_states = value_states.transpose(1, 2)
attn_output = _flash_attention_forward(
query_states, key_states, value_states, attention_mask, q_len,
position_ids=position_ids, dropout=dropout_rate,
sliding_window=None, is_causal=self.is_causal,
use_top_left_mask=self._flash_attn_uses_top_left_mask,
)
attn_output = attn_output.reshape(
bsz, q_len, self.num_heads * self.head_dim).contiguous()
attn_output = self.o_proj(attn_output)
if not output_attentions:
attn_weights = None
return attn_output, attn_weights, past_key_value
class MaxwellMoeSdpaAttention(MaxwellMoeAttention):
def forward(
self, hidden_states, attention_mask=None, position_ids=None,
past_key_value=None, output_attentions=False, use_cache=False,
cache_position=None, position_embeddings=None,
):
if output_attentions:
return super().forward(
hidden_states=hidden_states, attention_mask=attention_mask,
position_ids=position_ids, past_key_value=past_key_value,
output_attentions=output_attentions, use_cache=use_cache,
)
bsz, q_len, _ = hidden_states.size()
query_states = self.q_proj(hidden_states)
key_states = self.k_proj(hidden_states)
value_states = self.v_proj(hidden_states)
query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim)
key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim)
if self.qk_layernorm:
query_states = self.q_layernorm(query_states)
key_states = self.k_layernorm(key_states)
query_states = query_states.transpose(1, 2)
key_states = key_states.transpose(1, 2)
value_states = value_states.view(
bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
if position_embeddings is None:
cos, sin = self.rotary_emb(value_states, position_ids)
else:
cos, sin = position_embeddings
query_states, key_states = apply_rotary_pos_emb(
query_states, key_states, cos, sin)
if past_key_value is not None:
cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
key_states, value_states = past_key_value.update(
key_states, value_states, self.layer_idx, cache_kwargs)
key_states = repeat_kv(key_states, self.num_key_value_groups)
value_states = repeat_kv(value_states, self.num_key_value_groups)
causal_mask = attention_mask
if attention_mask is not None:
causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
if query_states.device.type == "cuda" and attention_mask is not None:
query_states = query_states.contiguous()
key_states = key_states.contiguous()
value_states = value_states.contiguous()
is_causal = True if causal_mask is None and q_len > 1 else False
attn_output = torch.nn.functional.scaled_dot_product_attention(
query_states, key_states, value_states,
attn_mask=causal_mask,
dropout_p=self.attention_dropout if self.training else 0.0,
is_causal=is_causal,
)
attn_output = attn_output.transpose(1, 2).contiguous()
attn_output = attn_output.view(bsz, q_len, self.num_heads * self.head_dim)
attn_output = self.o_proj(attn_output)
return attn_output, None, past_key_value
class MaxwellMoeTeAttention(MaxwellMoeAttention):
"""Attention using Transformer Engine's DotProductAttention kernel."""
def __init__(self, config, layer_idx=None):
super().__init__(config, layer_idx)
try:
import transformer_engine.pytorch as te
self.te_attn = te.DotProductAttention(
num_attention_heads=self.num_heads,
kv_channels=self.head_dim,
num_gqa_groups=self.num_key_value_heads,
attention_dropout=self.attention_dropout,
qkv_format="bshd",
attn_mask_type="causal",
softmax_scale=1.0 / math.sqrt(self.head_dim),
layer_number=layer_idx or 1,
)
except ImportError:
raise ImportError("transformer_engine is required for attn_implementation='te'")
def forward(
self, hidden_states, attention_mask=None, position_ids=None,
past_key_value=None, output_attentions=False, use_cache=False,
cache_position=None, position_embeddings=None,
):
bsz, q_len, _ = hidden_states.size()
query_states = self.q_proj(hidden_states)
key_states = self.k_proj(hidden_states)
value_states = self.v_proj(hidden_states)
query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim)
key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim)
value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim)
if self.qk_layernorm:
query_states = self.q_layernorm(query_states)
key_states = self.k_layernorm(key_states)
if position_embeddings is None:
cos, sin = self.rotary_emb(value_states, position_ids)
else:
cos, sin = position_embeddings
# RoPE expects [bsz, heads, seq, dim]
q_t = query_states.transpose(1, 2)
k_t = key_states.transpose(1, 2)
q_t, k_t = apply_rotary_pos_emb(q_t, k_t, cos, sin)
query_states = q_t.transpose(1, 2).contiguous()
key_states = k_t.transpose(1, 2).contiguous()
if past_key_value is not None:
k_for_cache = key_states.transpose(1, 2)
v_for_cache = value_states.transpose(1, 2)
cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
k_for_cache, v_for_cache = past_key_value.update(
k_for_cache, v_for_cache, self.layer_idx, cache_kwargs)
key_states = k_for_cache.transpose(1, 2).contiguous()
value_states = v_for_cache.transpose(1, 2).contiguous()
# TE DotProductAttention expects [bsz, seq, heads, dim] for bshd format
attn_output = self.te_attn(query_states, key_states, value_states)
attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.head_dim)
attn_output = self.o_proj(attn_output)
return attn_output, None, past_key_value
MAXWELLMOE_ATTENTION_CLASSES = {
"eager": MaxwellMoeAttention,
"flash_attention_2": MaxwellMoeFlashAttention2,
"sdpa": MaxwellMoeSdpaAttention,
"te": MaxwellMoeTeAttention,
}
# ---------------------------------------------------------------------------
# mHC (Manifold-Constrained Hyper-Connections)
# ---------------------------------------------------------------------------
def _sinkhorn_iterations(M: torch.Tensor, num_iterations: int, eps: float = 0.0):
"""TileKernels-aligned Sinkhorn-Knopp projection.
Mirrors the TileKernels mhc_sinkhorn_kernel forward pass:
1. M = softmax(logits, dim=-1) + eps
2. M = M / (M.sum(dim=-2) + eps) # col norm
3. repeat-1 times:
M = M / (M.sum(dim=-1) + eps) # row norm
M = M / (M.sum(dim=-2) + eps) # col norm
Input M is the softmax output from _compute_mappings.
Megatron --use-tile-mhc passes eps=0.0 (tile_mhc_kernels.py:169-170).
DeepSeek V4 HuggingFace uses eps=1e-6 (config.hc_eps).
"""
# softmax + eps (softmax already applied in _compute_mappings)
M = M + eps
# comb = comb / (comb.sum(-2) + eps)
M = M / (M.sum(dim=-2, keepdim=True) + eps)
for _ in range(num_iterations - 1):
# comb = comb / (comb.sum(-1) + eps)
M = M / (M.sum(dim=-1, keepdim=True) + eps)
# comb = comb / (comb.sum(-2) + eps)
M = M / (M.sum(dim=-2, keepdim=True) + eps)
return M
class HyperConnectionModule(nn.Module):
def __init__(self, config: MaxwellMoeConfig):
super().__init__()
self.n = config.num_residual_streams
self.hidden_size = config.hidden_size
self.sinkhorn_iterations = config.mhc_sinkhorn_iterations
self.eps = 1e-6
mix_dim = self.n * self.n + 2 * self.n
self.mapping_proj = nn.Linear(self.n * self.hidden_size, mix_dim, bias=False)
self.alpha_pre = nn.Parameter(torch.full((1,), config.mhc_init_gating_factor))
self.alpha_post = nn.Parameter(torch.full((1,), config.mhc_init_gating_factor))
self.alpha_res = nn.Parameter(torch.full((1,), config.mhc_init_gating_factor))
self.bias = nn.Parameter(torch.zeros(mix_dim))
def _compute_mappings(self, x):
b, s, nC = x.shape
n = self.n
norm = x.float().norm(dim=-1, keepdim=True)
r = 1.0 / (norm / math.sqrt(nC) + self.eps)
proj = F.linear(x.float(), self.mapping_proj.weight.float())
alpha = torch.cat([
self.alpha_pre.expand(n),
self.alpha_post.expand(n),
self.alpha_res.expand(n * n),
], dim=-1).float()
h = r.float() * proj * alpha + self.bias.float()
h_pre = (h[..., :n].sigmoid() + self.eps).to(x.dtype)
h_post = (h[..., n:2*n].sigmoid() * 2).to(x.dtype)
h_res_logits = h[..., 2*n:]
h_res = torch.softmax(h_res_logits.view(b, s, n, n), dim=-1)
h_res = _sinkhorn_iterations(h_res, self.sinkhorn_iterations, eps=self.eps)
h_res = h_res.to(x.dtype)
return h_pre, h_post, h_res
def aggregate(self, x, h_pre):
b, s, _ = x.shape
C = self.hidden_size
x_streams = x.view(b, s, self.n, C).float()
return (x_streams * h_pre.float().unsqueeze(-1)).sum(dim=2).to(x.dtype)
def expand_and_mix(self, layer_output, residual, h_post, h_res):
b, s, C = layer_output.shape
n = self.n
res_streams = residual.view(b, s, n, C).float()
h_res_batched = h_res.view(b * s, n, n).float()
res_batched = res_streams.view(b * s, n, C)
mixed = torch.bmm(h_res_batched.transpose(-1, -2), res_batched).view(b, s, n, C)
expanded = h_post.float().unsqueeze(-1) * layer_output.float().unsqueeze(2)
return (expanded + mixed).to(residual.dtype).view(b, s, n * C)
@staticmethod
def input_expand(x, n):
b, s, C = x.shape
return x.unsqueeze(2).expand(b, s, n, C).contiguous().view(b, s, n * C)
@staticmethod
def output_contract(x, n):
b, s, nC = x.shape
C = nC // n
return x.view(b, s, n, C).mean(dim=2)
class HCHead(nn.Module):
"""Learned n-stream to 1-stream contraction replacing simple averaging."""
def __init__(self, config: MaxwellMoeConfig):
super().__init__()
self.n = config.num_residual_streams
self.hidden_size = config.hidden_size
self.eps = 1e-6
self.hc_head_fn = nn.Parameter(
torch.zeros(self.n, self.n * self.hidden_size))
self.hc_head_base = nn.Parameter(torch.zeros(self.n))
self.hc_head_scale = nn.Parameter(torch.ones(1))
def forward(self, x):
"""
Args:
x: [b, s, n*C] n-stream hidden states
Returns:
[b, s, C] contracted single-stream
"""
b, s, nC = x.shape
n = self.n
C = self.hidden_size
dtype = x.dtype
x_flat = x.float()
rsqrt = torch.rsqrt(x_flat.square().mean(-1, keepdim=True) + self.eps)
proj = F.linear(x_flat, self.hc_head_fn.float())
mixes = proj * rsqrt
weights = torch.sigmoid(mixes * self.hc_head_scale.float() + self.hc_head_base.float()) + self.eps
x_streams = x.view(b, s, n, C).float()
out = (x_streams * weights.unsqueeze(-1)).sum(dim=2)
return out.to(dtype)
# ---------------------------------------------------------------------------
# MoE Block with expert bias and sigmoid routing
# ---------------------------------------------------------------------------
class MaxwellMoeSparseMoeBlock(nn.Module):
def __init__(self, config):
super().__init__()
self.num_experts = config.num_experts
self.top_k = config.num_experts_per_tok
self.use_shared_expert_gate = config.use_shared_expert_gate
self.router_score_function = getattr(config, 'router_score_function', 'softmax')
self.router_dtype = getattr(config, 'moe_router_dtype', 'fp32')
self.gate = nn.Linear(config.hidden_size, config.num_experts, bias=False)
if getattr(config, 'moe_router_enable_expert_bias', False):
self.expert_bias = nn.Parameter(torch.zeros(config.num_experts), requires_grad=False)
self.experts = nn.ModuleList(
[MaxwellMoeMLP(config, intermediate_size=config.moe_intermediate_size)
for _ in range(self.num_experts)])
if config.shared_expert_intermediate_size is not None:
self.num_shared_experts = 1
self.shared_expert = MaxwellMoeMLP(
config, intermediate_size=config.shared_expert_intermediate_size)
if self.use_shared_expert_gate:
self.shared_expert_gate = nn.Linear(config.hidden_size, 1, bias=False)
else:
self.num_shared_experts = 0
def _router_dtype(self, input_dtype):
if self.router_dtype == 'fp32':
return torch.float32
if self.router_dtype == 'fp64':
return torch.float64
return input_dtype
@staticmethod
def _autocast_device_type(tensor):
device_type = tensor.device.type
return device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
def _compute_router_logits(self, hidden_states):
router_dtype = self._router_dtype(hidden_states.dtype)
device_type = self._autocast_device_type(hidden_states)
with torch.autocast(device_type=device_type, enabled=False):
return F.linear(hidden_states.to(router_dtype), self.gate.weight.to(router_dtype))
def _maintain_float32_expert_bias(self):
if hasattr(self, 'expert_bias') and self.expert_bias.dtype != torch.float32:
self.expert_bias.data = self.expert_bias.data.to(torch.float32)
def forward(self, hidden_states):
self._maintain_float32_expert_bias()
batch_size, sequence_length, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
router_logits = self._compute_router_logits(hidden_states)
with torch.autocast(device_type=self._autocast_device_type(router_logits), enabled=False):
if self.router_score_function == 'sigmoid':
scores = torch.sigmoid(router_logits.float())
if hasattr(self, 'expert_bias'):
scores_for_routing = scores + self.expert_bias.float()
_, selected_experts = torch.topk(scores_for_routing, self.top_k, dim=-1)
topk_scores = torch.gather(scores, dim=1, index=selected_experts)
else:
topk_scores, selected_experts = torch.topk(scores, self.top_k, dim=-1)
routing_weights = (
topk_scores / (topk_scores.sum(dim=-1, keepdim=True) + 1e-20)
).type_as(router_logits)
else:
topk_routing_weights, selected_experts = torch.topk(
router_logits, self.top_k, dim=-1)
routing_weights = torch.softmax(
topk_routing_weights, dim=1, dtype=torch.float).type_as(router_logits)
final_hidden_states = torch.zeros(
(batch_size * sequence_length, hidden_dim),
dtype=hidden_states.dtype, device=hidden_states.device,
)
expert_mask = torch.nn.functional.one_hot(
selected_experts, num_classes=self.num_experts).permute(2, 1, 0)
for expert_idx in range(self.num_experts):
expert_layer = self.experts[expert_idx]
idx, top_x = torch.where(expert_mask[expert_idx])
if top_x.numel() == 0:
continue
current_state = hidden_states[None, top_x].reshape(-1, hidden_dim)
current_hidden_states = (
expert_layer(current_state) * routing_weights[top_x, idx, None])
final_hidden_states.index_add_(
0, top_x, current_hidden_states.to(hidden_states.dtype))
if self.num_shared_experts:
shared_expert_output = self.shared_expert(hidden_states)
if self.use_shared_expert_gate:
shared_expert_output = (
F.sigmoid(self.shared_expert_gate(hidden_states))
* shared_expert_output)
final_hidden_states = final_hidden_states + shared_expert_output
final_hidden_states = final_hidden_states.reshape(
batch_size, sequence_length, hidden_dim)
return final_hidden_states, router_logits
# ---------------------------------------------------------------------------
# Decoder Layer
# ---------------------------------------------------------------------------
class MaxwellMoeDecoderLayer(nn.Module):
def __init__(self, config: MaxwellMoeConfig, layer_idx: int):
super().__init__()
self.hidden_size = config.hidden_size
self.enable_hyper_connections = config.enable_hyper_connections
# Never let Transformers' backend auto-selection replace the
# architecture-specific shared block route with plain dense SDPA.
self.self_attn = MaxwellMoeBlockSparseGQAAttention(
config, layer_idx
)
is_dense = layer_idx in config.mlp_only_layers
if is_dense or config.num_experts == 0:
self.block_sparse_moe = MaxwellMoeMLP(
config, intermediate_size=config.intermediate_size)
else:
self.block_sparse_moe = MaxwellMoeSparseMoeBlock(config)
self.input_layernorm = MaxwellMoeRMSNorm(
config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = MaxwellMoeRMSNorm(
config.hidden_size, eps=config.rms_norm_eps)
if self.enable_hyper_connections:
self.self_attn_hyper_connection = HyperConnectionModule(config)
self.mlp_hyper_connection = HyperConnectionModule(config)
def forward(
self, hidden_states, attention_mask=None, position_ids=None,
past_key_value=None, output_attentions=False,
output_router_logits=False, use_cache=False,
cache_position=None, position_embeddings=None, **kwargs,
):
if self.enable_hyper_connections:
return self._forward_with_hc(
hidden_states, attention_mask, position_ids, past_key_value,
output_attentions, output_router_logits, use_cache,
cache_position, position_embeddings, **kwargs)
else:
return self._forward_standard(
hidden_states, attention_mask, position_ids, past_key_value,
output_attentions, output_router_logits, use_cache,
cache_position, position_embeddings, **kwargs)
def _forward_standard(
self, hidden_states, attention_mask, position_ids, past_key_value,
output_attentions, output_router_logits, use_cache,
cache_position, position_embeddings, **kwargs,
):
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states, self_attn_weights, present_key_value = self.self_attn(
hidden_states=hidden_states, attention_mask=attention_mask,
position_ids=position_ids, past_key_value=past_key_value,
output_attentions=output_attentions, use_cache=use_cache,
cache_position=cache_position, position_embeddings=position_embeddings,
)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.block_sparse_moe(hidden_states)
if isinstance(hidden_states, tuple):
hidden_states, router_logits = hidden_states
else:
router_logits = None
hidden_states = residual + hidden_states
outputs = (hidden_states,)
if output_attentions:
outputs += (self_attn_weights,)
if use_cache:
outputs += (present_key_value,)
if output_router_logits:
outputs += (router_logits,)
return outputs
def _forward_with_hc(
self, hidden_states, attention_mask, position_ids, past_key_value,
output_attentions, output_router_logits, use_cache,
cache_position, position_embeddings, **kwargs,
):
# === Self-Attention with mHC ===
residual = hidden_states
hc = self.self_attn_hyper_connection
h_pre, h_post, h_res = hc._compute_mappings(hidden_states)
aggregated = hc.aggregate(hidden_states, h_pre)
aggregated = self.input_layernorm(aggregated)
attn_output, self_attn_weights, present_key_value = self.self_attn(
hidden_states=aggregated, attention_mask=attention_mask,
position_ids=position_ids, past_key_value=past_key_value,
output_attentions=output_attentions, use_cache=use_cache,
cache_position=cache_position, position_embeddings=position_embeddings,
)
hidden_states = hc.expand_and_mix(attn_output, residual, h_post, h_res)
# === MLP with mHC ===
residual = hidden_states
hc_mlp = self.mlp_hyper_connection
h_pre, h_post, h_res = hc_mlp._compute_mappings(hidden_states)
aggregated = hc_mlp.aggregate(hidden_states, h_pre)
aggregated = self.post_attention_layernorm(aggregated)
mlp_output = self.block_sparse_moe(aggregated)
if isinstance(mlp_output, tuple):
mlp_output, router_logits = mlp_output
else:
router_logits = None
hidden_states = hc_mlp.expand_and_mix(mlp_output, residual, h_post, h_res)
outputs = (hidden_states,)
if output_attentions:
outputs += (self_attn_weights,)
if use_cache:
outputs += (present_key_value,)
if output_router_logits:
outputs += (router_logits,)
return outputs
# ---------------------------------------------------------------------------
# Model
# ---------------------------------------------------------------------------
MAXWELLMOE_START_DOCSTRING = ""
MAXWELLMOE_INPUTS_DOCSTRING = ""
class MaxwellMoePreTrainedModel(PreTrainedModel):
config_class = MaxwellMoeConfig
base_model_prefix = "model"
supports_gradient_checkpointing = True
_no_split_modules = ["MaxwellMoeDecoderLayer"]
_skip_keys_device_placement = "past_key_values"
_supports_flash_attn_2 = False
_supports_sdpa = False
_supports_cache_class = True
def _init_weights(self, module):
std = self.config.initializer_range
if isinstance(module, nn.Linear):
module.weight.data.normal_(mean=0.0, std=std)
if module.bias is not None:
module.bias.data.zero_()
elif isinstance(module, nn.Embedding):
module.weight.data.normal_(mean=0.0, std=std)
if module.padding_idx is not None:
module.weight.data[module.padding_idx].zero_()
class MaxwellMoeModel(MaxwellMoePreTrainedModel):
def __init__(self, config: MaxwellMoeConfig):
super().__init__(config)
self.padding_idx = config.pad_token_id
self.vocab_size = config.vocab_size
self.enable_hyper_connections = config.enable_hyper_connections
self.num_residual_streams = config.num_residual_streams
self.embed_tokens = nn.Embedding(
config.vocab_size, config.hidden_size, self.padding_idx)
self.layers = nn.ModuleList(
[MaxwellMoeDecoderLayer(config, layer_idx)
for layer_idx in range(config.num_hidden_layers)])
self._attn_implementation = config._attn_implementation
self.norm = MaxwellMoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.rotary_emb = MaxwellMoeRotaryEmbedding(config=config)
if self.enable_hyper_connections:
self.hc_head = HCHead(config)
self.gradient_checkpointing = False
self.post_init()
def get_input_embeddings(self):
return self.embed_tokens
def set_input_embeddings(self, value):
self.embed_tokens = value
@staticmethod
def _normalize_block_sparse_gqa_attention_mask(attention_mask):
"""Accept only masks that do not alter physical sparse coordinates.
Block Sparse GQA defines its route in physical KV-cache coordinates.
Padding and packed/custom masks need a
logical-to-physical token map that this bounded HF oracle does not
implement, so reject them instead of silently changing the route.
"""
if attention_mask is None:
return None
if not isinstance(attention_mask, torch.Tensor):
raise TypeError(
"block_sparse_gqa attention_mask must be a torch.Tensor or None"
)
if attention_mask.dim() == 2:
if not torch.all(attention_mask == 1).item():
raise ValueError(
"block_sparse_gqa HF does not support padding: a 2D "
"attention_mask must contain only ones"
)
# Canonicalize the no-op mask so the model constructs its causal
# additive mask in one place.
return None
if attention_mask.dim() == 4:
if attention_mask.dtype == torch.bool:
raise ValueError(
"block_sparse_gqa HF does not support 4D boolean masks or "
"packed inputs"
)
if not torch.all(attention_mask == 0).item():
raise ValueError(
"block_sparse_gqa HF does not support nontrivial 4D "
"additive masks, padding, or packed inputs"
)
# An all-zero additive mask is a no-op. Canonicalize it to None
# so it cannot accidentally replace the internal causal mask.
return None
raise ValueError(
"block_sparse_gqa HF supports only an all-ones 2D mask, an "
"all-zero 4D additive mask, or no attention mask"
)
@staticmethod
def _validate_block_sparse_gqa_cache(past_key_values):
"""Require main K/V and indexer K to remain synchronized."""
if past_key_values is None:
return
if not isinstance(past_key_values, MaxwellMoeBlockSparseGQACache):
raise TypeError(
"block_sparse_gqa requires MaxwellMoeBlockSparseGQACache; "
"ordinary, legacy, static, sliding, and quantized caches omit "
"historical indexer K"
)
main_length = past_key_values.get_seq_length()
for layer_idx, state in enumerate(
past_key_values.indexer_key_states
):
if state is not None and int(state.shape[1]) != main_length:
raise ValueError(
"block_sparse_gqa cache is inconsistent at layer "
f"{layer_idx}: main length={main_length}, "
f"indexer length={state.shape[1]}"
)
def forward(
self,
input_ids=None, attention_mask=None, position_ids=None,
past_key_values=None, inputs_embeds=None, use_cache=None,
output_attentions=None, output_hidden_states=None,
output_router_logits=None, return_dict=None,
cache_position=None, output_multistream=False,
):
output_attentions = (output_attentions if output_attentions is not None
else self.config.output_attentions)
output_router_logits = (output_router_logits if output_router_logits is not None
else self.config.output_router_logits)
output_hidden_states = (output_hidden_states if output_hidden_states is not None
else self.config.output_hidden_states)
use_cache = use_cache if use_cache is not None else self.config.use_cache
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
if (input_ids is None) ^ (inputs_embeds is not None):
raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
if self.gradient_checkpointing and self.training:
raise ValueError(
"block_sparse_gqa HF is an inference reference and does not "
"support gradient checkpointing"
)
if use_cache:
if past_key_values is None:
past_key_values = MaxwellMoeBlockSparseGQACache()
elif not isinstance(
past_key_values, MaxwellMoeBlockSparseGQACache
):
existing_length = (
past_key_values.get_seq_length()
if isinstance(past_key_values, Cache)
else len(past_key_values)
if isinstance(past_key_values, (tuple, list))
else None
)
if existing_length not in (0, None):
raise ValueError(
"block_sparse_gqa cannot restore a non-empty cache "
"without historical indexer K"
)
if existing_length is None:
raise TypeError(
"block_sparse_gqa received an unsupported cache type"
)
past_key_values = MaxwellMoeBlockSparseGQACache()
self._validate_block_sparse_gqa_cache(past_key_values)
attention_mask = self._normalize_block_sparse_gqa_attention_mask(
attention_mask
)
if inputs_embeds is None:
inputs_embeds = self.embed_tokens(input_ids)
if cache_position is None:
past_seen_tokens = (past_key_values.get_seq_length()
if past_key_values is not None else 0)
cache_position = torch.arange(
past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1],
device=inputs_embeds.device)
if position_ids is None:
position_ids = cache_position.unsqueeze(0)
causal_mask = self._update_causal_mask(
attention_mask, inputs_embeds, cache_position,
past_key_values, output_attentions)
hidden_states = inputs_embeds
if self.enable_hyper_connections:
hidden_states = HyperConnectionModule.input_expand(
hidden_states, self.num_residual_streams)
position_embeddings = self.rotary_emb(hidden_states, position_ids)
all_hidden_states = () if output_hidden_states else None
all_self_attns = () if output_attentions else None
all_router_logits = () if output_router_logits else None
next_decoder_cache = None
for decoder_layer in self.layers:
if output_hidden_states:
all_hidden_states += (hidden_states,)
if self.gradient_checkpointing and self.training:
layer_outputs = self._gradient_checkpointing_func(
decoder_layer.__call__,
hidden_states, causal_mask, position_ids, past_key_values,
output_attentions, output_router_logits, use_cache,
cache_position, position_embeddings,
)
else:
layer_outputs = decoder_layer(
hidden_states, attention_mask=causal_mask,
position_ids=position_ids, past_key_value=past_key_values,
output_attentions=output_attentions,
output_router_logits=output_router_logits,
use_cache=use_cache, cache_position=cache_position,
position_embeddings=position_embeddings,
)
hidden_states = layer_outputs[0]
if use_cache:
next_decoder_cache = layer_outputs[2 if output_attentions else 1]
if output_attentions:
all_self_attns += (layer_outputs[1],)
if output_router_logits and layer_outputs[-1] is not None:
all_router_logits += (layer_outputs[-1],)
multistream_hidden_states = None
if output_multistream and self.enable_hyper_connections:
multistream_hidden_states = hidden_states
if self.enable_hyper_connections:
hidden_states = self.hc_head(hidden_states)
hidden_states = self.norm(hidden_states)
if output_hidden_states:
all_hidden_states += (hidden_states,)
next_cache = next_decoder_cache if use_cache else None
if not return_dict:
result = tuple(v for v in [
hidden_states, next_cache, all_hidden_states,
all_self_attns, all_router_logits
] if v is not None)
if output_multistream:
return result, multistream_hidden_states, position_embeddings, causal_mask, position_ids, cache_position
return result
model_output = MoeModelOutputWithPast(
last_hidden_state=hidden_states,
past_key_values=next_cache,
hidden_states=all_hidden_states,
attentions=all_self_attns,
router_logits=all_router_logits,
)
if output_multistream:
return model_output, multistream_hidden_states, position_embeddings, causal_mask, position_ids, cache_position
return model_output
def _update_causal_mask(self, attention_mask, input_tensor, cache_position,
past_key_values, output_attentions):
if self.config._attn_implementation == "flash_attention_2":
if attention_mask is not None and 0.0 in attention_mask:
return attention_mask
return None
past_seen_tokens = (past_key_values.get_seq_length()
if past_key_values is not None else 0)
using_static_cache = isinstance(past_key_values, StaticCache)
if (self.config._attn_implementation == "sdpa"
and not using_static_cache
and not output_attentions):
if AttentionMaskConverter._ignore_causal_mask_sdpa(
attention_mask, inputs_embeds=input_tensor,
past_key_values_length=past_seen_tokens,
sliding_window=None,
is_training=self.training,
):
return None
dtype, device = input_tensor.dtype, input_tensor.device
min_dtype = torch.finfo(dtype).min
sequence_length = input_tensor.shape[1]
if using_static_cache:
target_length = past_key_values.get_max_cache_shape()
else:
target_length = (
attention_mask.shape[-1] if isinstance(attention_mask, torch.Tensor)
else past_seen_tokens + sequence_length + 1)
causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position(
attention_mask, sequence_length=sequence_length,
target_length=target_length, dtype=dtype, device=device,
cache_position=cache_position, batch_size=input_tensor.shape[0],
config=self.config, past_key_values=past_key_values,
)
if (self.config._attn_implementation == "sdpa"
and attention_mask is not None
and attention_mask.device.type == "cuda"
and not output_attentions):
causal_mask = AttentionMaskConverter._unmask_unattended(
causal_mask, min_dtype)
return causal_mask
@staticmethod
def _prepare_4d_causal_attention_mask_with_cache_position(
attention_mask, sequence_length, target_length, dtype, device,
cache_position, batch_size, config, past_key_values,
):
if attention_mask is not None and attention_mask.dim() == 4:
causal_mask = attention_mask
else:
min_dtype = torch.finfo(dtype).min
causal_mask = torch.full(
(sequence_length, target_length),
fill_value=min_dtype, dtype=dtype, device=device)
diagonal_attend_mask = (
torch.arange(target_length, device=device)
> cache_position.reshape(-1, 1))
causal_mask *= diagonal_attend_mask
causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
if attention_mask is not None:
causal_mask = causal_mask.clone()
if attention_mask.shape[-1] > target_length:
attention_mask = attention_mask[:, :target_length]
mask_length = attention_mask.shape[-1]
padding_mask = (
causal_mask[:, :, :, :mask_length]
+ attention_mask[:, None, None, :]) == 0
causal_mask[:, :, :, :mask_length] = causal_mask[
:, :, :, :mask_length].masked_fill(padding_mask, min_dtype)
return causal_mask
# ---------------------------------------------------------------------------
# MTP (Multi-Token Prediction)
# ---------------------------------------------------------------------------
#
# The MTP layer's inner transformer layer uses plain (non block-sparse) GQA
# attention -- confirmed against the actual mcore checkpoint tensors
# (mtp.layers.0.transformer_layer.self_attention.* has no indexer.*
# counterpart). So MaxwellMoeMTPDecoderLayer reuses this file's existing
# plain attention classes (MAXWELLMOE_ATTENTION_CLASSES) rather than the
# block-sparse MaxwellMoeDecoderLayer used by the main 45-layer decoder.
class MaxwellMoeMTPDecoderLayer(nn.Module):
def __init__(self, config: MaxwellMoeConfig, layer_idx: int):
super().__init__()
self.hidden_size = config.hidden_size
self.enable_hyper_connections = config.enable_hyper_connections
self.self_attn = MAXWELLMOE_ATTENTION_CLASSES[
config._attn_implementation](config, layer_idx)
is_dense = layer_idx in config.mlp_only_layers
if is_dense or config.num_experts == 0:
self.block_sparse_moe = MaxwellMoeMLP(
config, intermediate_size=config.intermediate_size)
else:
self.block_sparse_moe = MaxwellMoeSparseMoeBlock(config)
self.input_layernorm = MaxwellMoeRMSNorm(
config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = MaxwellMoeRMSNorm(
config.hidden_size, eps=config.rms_norm_eps)
if self.enable_hyper_connections:
self.self_attn_hyper_connection = HyperConnectionModule(config)
self.mlp_hyper_connection = HyperConnectionModule(config)
def forward(
self, hidden_states, attention_mask=None, position_ids=None,
past_key_value=None, output_attentions=False,
output_router_logits=False, use_cache=False,
cache_position=None, position_embeddings=None, **kwargs,
):
if self.enable_hyper_connections:
return self._forward_with_hc(
hidden_states, attention_mask, position_ids, past_key_value,
output_attentions, output_router_logits, use_cache,
cache_position, position_embeddings, **kwargs)
else:
return self._forward_standard(
hidden_states, attention_mask, position_ids, past_key_value,
output_attentions, output_router_logits, use_cache,
cache_position, position_embeddings, **kwargs)
def _forward_standard(
self, hidden_states, attention_mask, position_ids, past_key_value,
output_attentions, output_router_logits, use_cache,
cache_position, position_embeddings, **kwargs,
):
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states, self_attn_weights, present_key_value = self.self_attn(
hidden_states=hidden_states, attention_mask=attention_mask,
position_ids=position_ids, past_key_value=past_key_value,
output_attentions=output_attentions, use_cache=use_cache,
cache_position=cache_position, position_embeddings=position_embeddings,
)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.block_sparse_moe(hidden_states)
if isinstance(hidden_states, tuple):
hidden_states, router_logits = hidden_states
else:
router_logits = None
hidden_states = residual + hidden_states
outputs = (hidden_states,)
if output_attentions:
outputs += (self_attn_weights,)
if use_cache:
outputs += (present_key_value,)
if output_router_logits:
outputs += (router_logits,)
return outputs
def _forward_with_hc(
self, hidden_states, attention_mask, position_ids, past_key_value,
output_attentions, output_router_logits, use_cache,
cache_position, position_embeddings, **kwargs,
):
residual = hidden_states
hc = self.self_attn_hyper_connection
h_pre, h_post, h_res = hc._compute_mappings(hidden_states)
aggregated = hc.aggregate(hidden_states, h_pre)
aggregated = self.input_layernorm(aggregated)
attn_output, self_attn_weights, present_key_value = self.self_attn(
hidden_states=aggregated, attention_mask=attention_mask,
position_ids=position_ids, past_key_value=past_key_value,
output_attentions=output_attentions, use_cache=use_cache,
cache_position=cache_position, position_embeddings=position_embeddings,
)
hidden_states = hc.expand_and_mix(attn_output, residual, h_post, h_res)
residual = hidden_states
hc_mlp = self.mlp_hyper_connection
h_pre, h_post, h_res = hc_mlp._compute_mappings(hidden_states)
aggregated = hc_mlp.aggregate(hidden_states, h_pre)
aggregated = self.post_attention_layernorm(aggregated)
mlp_output = self.block_sparse_moe(aggregated)
if isinstance(mlp_output, tuple):
mlp_output, router_logits = mlp_output
else:
router_logits = None
hidden_states = hc_mlp.expand_and_mix(mlp_output, residual, h_post, h_res)
outputs = (hidden_states,)
if output_attentions:
outputs += (self_attn_weights,)
if use_cache:
outputs += (present_key_value,)
if output_router_logits:
outputs += (router_logits,)
return outputs
class MaxwellMoeMTPLayer(nn.Module):
"""Multi-Token Prediction layer.
Takes the multi-stream hidden states from the decoder, combines with
rolled (next-position) token embeddings via separate projections,
passes through a full MoE transformer layer, and contracts back to
a single-stream representation for the shared lm_head.
"""
def __init__(self, config: MaxwellMoeConfig, layer_idx: int):
super().__init__()
self.config = config
n = config.num_residual_streams
h = config.hidden_size
self.enorm = MaxwellMoeRMSNorm(h, eps=config.rms_norm_eps)
self.hnorm = MaxwellMoeRMSNorm(h, eps=config.rms_norm_eps)
self.e_proj = nn.Linear(h, h, bias=False)
self.h_proj = nn.Linear(h, h, bias=False)
mtp_layer_idx = config.num_hidden_layers + layer_idx
self.decoder_layer = MaxwellMoeMTPDecoderLayer(config, layer_idx=mtp_layer_idx)
self.hc_head = HCHead(config)
self.final_layernorm = MaxwellMoeRMSNorm(h, eps=config.rms_norm_eps)
def forward(
self,
multistream_hidden_states,
input_ids,
position_ids,
embed_tokens,
attention_mask=None,
position_embeddings=None,
cache_position=None,
):
"""
Args:
multistream_hidden_states: [b, s, n*h] multi-stream from decoder
input_ids: [b, s] current token IDs
position_ids: [b, s] or [1, s]
embed_tokens: embedding layer (shared with main model)
attention_mask: causal mask
position_embeddings: (cos, sin) from rotary embedding
Returns:
hidden_states: [b, s, h] contracted single-stream output
mtp_multistream: [b, s, n*h] multi-stream output (for chaining)
rolled_input_ids: [b, s] rolled input IDs
rolled_position_ids: [b, s] rolled position IDs
"""
b, s, nC = multistream_hidden_states.shape
n = self.config.num_residual_streams
h = self.config.hidden_size
rolled_input_ids = torch.roll(input_ids, shifts=-1, dims=-1)
rolled_input_ids[:, -1] = self.config.pad_token_id
rolled_position_ids = torch.roll(position_ids, shifts=-1, dims=-1)
rolled_position_ids[:, -1] = 0
decoder_input = embed_tokens(rolled_input_ids)
decoder_input = self.enorm(decoder_input)
hs_streams = multistream_hidden_states.view(b, s, n, h)
hs_streams = self.hnorm(hs_streams)
e_out = self.e_proj(decoder_input)
h_out = self.h_proj(hs_streams)
e_out = e_out.unsqueeze(2).expand(b, s, n, h)
hidden_states = (e_out + h_out).reshape(b, s, n * h)
layer_outputs = self.decoder_layer(
hidden_states,
attention_mask=attention_mask,
position_ids=position_ids,
position_embeddings=position_embeddings,
cache_position=cache_position,
)
mtp_multistream = layer_outputs[0]
hidden_states = self.hc_head(mtp_multistream)
hidden_states = self.final_layernorm(hidden_states)
return hidden_states, mtp_multistream, rolled_input_ids, rolled_position_ids
@dataclass
class MTPCausalLMOutput(ModelOutput):
loss: Optional[torch.FloatTensor] = None
logits: torch.FloatTensor = None
mtp_logits: Optional[torch.FloatTensor] = None
past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
router_logits: Optional[Tuple[torch.FloatTensor]] = None
class MaxwellMoeForCausalLM(MaxwellMoePreTrainedModel, GenerationMixin):
"""MaxwellMoE with Multi-Token Prediction head for speculative decoding."""
_tied_weights_keys = ["lm_head.weight"]
def __init__(self, config):
super().__init__(config)
self.model = MaxwellMoeModel(config)
self.vocab_size = config.vocab_size
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
self.router_aux_loss_coef = config.router_aux_loss_coef
self.num_experts = config.num_experts
self.num_experts_per_tok = config.num_experts_per_tok
self.mtp_num_layers = getattr(config, 'mtp_num_layers', 0)
if self.mtp_num_layers > 0:
self.mtp_layers = nn.ModuleList([
MaxwellMoeMTPLayer(config, layer_idx=i)
for i in range(self.mtp_num_layers)
])
else:
self.mtp_layers = nn.ModuleList()
self.post_init()
def get_input_embeddings(self):
return self.model.embed_tokens
def set_input_embeddings(self, value):
self.model.embed_tokens = value
def get_output_embeddings(self):
return self.lm_head
def set_output_embeddings(self, new_embeddings):
self.lm_head = new_embeddings
def set_decoder(self, decoder):
self.model = decoder
def get_decoder(self):
return self.model
def _fix_expert_bias_precision(self):
"""Restore decoder and MTP expert_bias to fp32 from safetensors (first forward).
from_pretrained(dtype=bf16) truncates expert_bias, and the per-expert
spread (range ~0.005) is below bf16's quantization step (~0.06), so
routing collapses without it.
"""
self._expert_bias_fp32_done = True
import json, os
from safetensors import safe_open
model_dir = self.config._name_or_path
idx_path = os.path.join(model_dir, "model.safetensors.index.json")
if not os.path.exists(idx_path):
return
with open(idx_path) as f:
weight_map = json.load(f)["weight_map"]
for layer_idx, layer in enumerate(self.model.layers):
moe = getattr(layer, 'block_sparse_moe', None)
if moe is None or not hasattr(moe, 'expert_bias'):
continue
key = f"model.layers.{layer_idx}.block_sparse_moe.expert_bias"
shard = weight_map.get(key)
if shard is None:
continue
with safe_open(os.path.join(model_dir, shard), framework="pt") as f:
moe.expert_bias.data = f.get_tensor(key).float().to(moe.expert_bias.device)
for mtp_idx, mtp_layer in enumerate(self.mtp_layers):
moe = getattr(mtp_layer.decoder_layer, 'block_sparse_moe', None)
if moe is None or not hasattr(moe, 'expert_bias'):
continue
key = f"mtp_layers.{mtp_idx}.decoder_layer.block_sparse_moe.expert_bias"
shard = weight_map.get(key)
if shard is None:
continue
with safe_open(os.path.join(model_dir, shard), framework="pt") as f:
moe.expert_bias.data = f.get_tensor(key).float().to(moe.expert_bias.device)
def forward(
self,
input_ids=None, attention_mask=None, position_ids=None,
past_key_values=None, inputs_embeds=None, labels=None,
use_cache=None, output_attentions=None, output_hidden_states=None,
output_router_logits=None, return_dict=None,
cache_position=None, num_logits_to_keep=0,
output_mtp_logits=False,
):
if not getattr(self, '_expert_bias_fp32_done', False):
self._fix_expert_bias_precision()
output_attentions = (output_attentions if output_attentions is not None
else self.config.output_attentions)
output_router_logits = (output_router_logits if output_router_logits is not None
else self.config.output_router_logits)
output_hidden_states = (output_hidden_states if output_hidden_states is not None
else self.config.output_hidden_states)
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
need_mtp = output_mtp_logits and len(self.mtp_layers) > 0
if need_mtp:
result = self.model(
input_ids=input_ids, attention_mask=attention_mask,
position_ids=position_ids, past_key_values=past_key_values,
inputs_embeds=inputs_embeds, use_cache=use_cache,
output_attentions=output_attentions,
output_hidden_states=output_hidden_states,
output_router_logits=output_router_logits,
return_dict=return_dict, cache_position=cache_position,
output_multistream=True,
)
if return_dict:
outputs, multistream_hidden_states, position_embeddings, causal_mask, resolved_position_ids, resolved_cache_position = result
else:
base_result, multistream_hidden_states, position_embeddings, causal_mask, resolved_position_ids, resolved_cache_position = result
outputs = base_result
else:
outputs = self.model(
input_ids=input_ids, attention_mask=attention_mask,
position_ids=position_ids, past_key_values=past_key_values,
inputs_embeds=inputs_embeds, use_cache=use_cache,
output_attentions=output_attentions,
output_hidden_states=output_hidden_states,
output_router_logits=output_router_logits,
return_dict=return_dict, cache_position=cache_position,
)
hidden_states = outputs[0] if not return_dict else outputs.last_hidden_state
lm_input = hidden_states[:, -num_logits_to_keep:, :]
logits = torch.nn.functional.linear(lm_input.float(), self.lm_head.weight.float(), self.lm_head.bias if self.lm_head.bias is not None else None)
mtp_logits = None
if need_mtp and multistream_hidden_states is not None:
cur_input_ids = input_ids
cur_position_ids = resolved_position_ids
cur_multistream = multistream_hidden_states
for mtp_layer in self.mtp_layers:
mtp_hidden, cur_multistream, cur_input_ids, cur_position_ids = mtp_layer(
multistream_hidden_states=cur_multistream,
input_ids=cur_input_ids,
position_ids=cur_position_ids,
embed_tokens=self.model.embed_tokens,
attention_mask=causal_mask,
position_embeddings=position_embeddings,
cache_position=resolved_cache_position,
)
mtp_logits = torch.nn.functional.linear(mtp_hidden.float(), self.lm_head.weight.float(), self.lm_head.bias if self.lm_head.bias is not None else None)
loss = None
if labels is not None:
loss = self.loss_function(logits, labels, self.vocab_size)
if not return_dict:
output = (logits,) + outputs[1:]
if mtp_logits is not None:
output = output + (mtp_logits,)
return (loss,) + output if loss is not None else output
return MTPCausalLMOutput(
loss=loss,
logits=logits,
mtp_logits=mtp_logits,
past_key_values=outputs.past_key_values,
hidden_states=outputs.hidden_states,
attentions=outputs.attentions,
router_logits=outputs.router_logits,
)
# coding=utf-8
# Copyright 2026 the Maxwell team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""PyTorch MaxwellMoE model — w35a3 variant.
Supports: mHC, QK LayerNorm, expert bias, sigmoid routing, learned HC head,
mixed dense/MoE layers, shared experts.
"""
import math
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union
import torch
import torch.nn.functional as F
import torch.utils.checkpoint
from torch import nn
from transformers.activations import ACT2FN
from transformers.cache_utils import Cache, DynamicCache, StaticCache
from transformers.generation import GenerationMixin
from transformers.modeling_attn_mask_utils import AttentionMaskConverter
from transformers.modeling_outputs import (
MoeModelOutputWithPast,
ModelOutput,
)
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
from transformers.modeling_utils import PreTrainedModel
from transformers.utils import (
add_start_docstrings,
add_start_docstrings_to_model_forward,
is_flash_attn_2_available,
logging,
replace_return_docstrings,
)
try:
from transformers.utils import is_flash_attn_greater_or_equal_2_10
except ImportError:
try:
from transformers.utils import is_flash_attn_greater_or_equal
def is_flash_attn_greater_or_equal_2_10():
return is_flash_attn_greater_or_equal("2.1.0")
except ImportError:
def is_flash_attn_greater_or_equal_2_10():
return True
from .configuration_maxwell_moe import MaxwellMoeConfig
if is_flash_attn_2_available():
from transformers.modeling_flash_attention_utils import _flash_attention_forward
logger = logging.get_logger(__name__)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
class MaxwellMoeRMSNorm(nn.Module):
def __init__(self, hidden_size, eps=1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, hidden_states):
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
# Apply the weight in fp32 and round once, as TE RMSNorm does. Rounding
# before the multiply is visible at large norm weights (q/k norms of the
# last layer reach ~14) where attention logits run into the thousands.
return (self.weight.float() * hidden_states).to(input_dtype)
class MaxwellMoeRotaryEmbedding(nn.Module):
def __init__(self, dim=None, max_position_embeddings=2048, base=10000,
device=None, scaling_factor=1.0, rope_type="default",
config: Optional[MaxwellMoeConfig] = None):
super().__init__()
self.rope_kwargs = {}
if config is None:
self.rope_kwargs = {
"rope_type": rope_type, "factor": scaling_factor,
"dim": dim, "base": base,
"max_position_embeddings": max_position_embeddings,
}