Maxwell-V1-Flash / modeling_maxwell_moe.py
Maxwell
Initial commit
5a278bd
Raw History Blame Contribute Delete
93.9 kB
# coding=utf-8 # Copyright 2026 the Maxwell team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """PyTorch MaxwellMoE model — w35a3 variant. Supports: mHC, QK LayerNorm, expert bias, sigmoid routing, learned HC head, mixed dense/MoE layers, shared experts. """ import math from dataclasses import dataclass from typing import List, Optional, Tuple, Union import torch import torch.nn.functional as F import torch.utils.checkpoint from torch import nn from transformers.activations import ACT2FN from transformers.cache_utils import Cache, DynamicCache, StaticCache from transformers.generation import GenerationMixin from transformers.modeling_attn_mask_utils import AttentionMaskConverter from transformers.modeling_outputs import ( MoeModelOutputWithPast, ModelOutput, ) from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS from transformers.modeling_utils import PreTrainedModel from transformers.utils import ( add_start_docstrings, add_start_docstrings_to_model_forward, is_flash_attn_2_available, logging, replace_return_docstrings, ) try: from transformers.utils import is_flash_attn_greater_or_equal_2_10 except ImportError: try: from transformers.utils import is_flash_attn_greater_or_equal def is_flash_attn_greater_or_equal_2_10(): return is_flash_attn_greater_or_equal("2.1.0") except ImportError: def is_flash_attn_greater_or_equal_2_10(): return True from .configuration_maxwell_moe import MaxwellMoeConfig if is_flash_attn_2_available(): from transformers.modeling_flash_attention_utils import _flash_attention_forward logger = logging.get_logger(__name__) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- class MaxwellMoeRMSNorm(nn.Module): def __init__(self, hidden_size, eps=1e-6): super().__init__() self.weight = nn.Parameter(torch.ones(hidden_size)) self.variance_epsilon = eps def forward(self, hidden_states): input_dtype = hidden_states.dtype hidden_states = hidden_states.to(torch.float32) variance = hidden_states.pow(2).mean(-1, keepdim=True) hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) # Apply the weight in fp32 and round once, as TE RMSNorm does. Rounding # before the multiply is visible at large norm weights (q/k norms of the # last layer reach ~14) where attention logits run into the thousands. return (self.weight.float() * hidden_states).to(input_dtype) class MaxwellMoeRotaryEmbedding(nn.Module): def __init__(self, dim=None, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0, rope_type="default", config: Optional[MaxwellMoeConfig] = None): super().__init__() self.rope_kwargs = {} if config is None: self.rope_kwargs = { "rope_type": rope_type, "factor": scaling_factor, "dim": dim, "base": base, "max_position_embeddings": max_position_embeddings, } self.rope_type = rope_type self.max_seq_len_cached = max_position_embeddings self.original_max_seq_len = max_position_embeddings else: if config.rope_scaling is not None: self.rope_type = config.rope_scaling.get( "rope_type", config.rope_scaling.get("type")) else: self.rope_type = "default" self.max_seq_len_cached = config.max_position_embeddings self.original_max_seq_len = config.max_position_embeddings self.config = config self._rope_base = base self._rope_dim = dim if self.rope_type == "default": if config is not None: self._rope_base = config.rope_theta self._rope_dim = (config.attention_head_size if config.attention_head_size else config.hidden_size // config.num_attention_heads) else: self._rope_base = self.rope_kwargs.get("base", 10000) self._rope_dim = self.rope_kwargs.get("dim", 64) inv_freq = self._compute_default_inv_freq(device) self.attention_scaling = 1.0 else: self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] inv_freq, self.attention_scaling = self.rope_init_fn( self.config, device, **self.rope_kwargs) self.register_buffer("inv_freq", inv_freq, persistent=False) self.original_inv_freq = self.inv_freq def _compute_default_inv_freq(self, device=None): return 1.0 / (self._rope_base ** ( torch.arange(0, self._rope_dim, 2, dtype=torch.int64).float().to(device) / self._rope_dim)) def _inv_freq_is_valid(self): if self.inv_freq.numel() == 0: return False expected = self._compute_default_inv_freq(self.inv_freq.device) if self.inv_freq.shape != expected.shape: return False if not torch.isfinite(self.inv_freq).all(): return False if (self.inv_freq <= 0).any(): return False return torch.allclose(self.inv_freq.float(), expected.float(), rtol=1e-5, atol=1e-8) def _reset_inv_freq(self, device): inv_freq = self._compute_default_inv_freq(device) self.register_buffer("inv_freq", inv_freq, persistent=False) self.original_inv_freq = self.inv_freq def _dynamic_frequency_update(self, position_ids, device): seq_len = torch.max(position_ids) + 1 if seq_len > self.max_seq_len_cached: if hasattr(self, 'rope_init_fn'): inv_freq, self.attention_scaling = self.rope_init_fn( self.config, device, seq_len=seq_len, **self.rope_kwargs) self.register_buffer("inv_freq", inv_freq, persistent=False) self.max_seq_len_cached = seq_len if (seq_len < self.original_max_seq_len and self.max_seq_len_cached > self.original_max_seq_len): self.register_buffer("inv_freq", self.original_inv_freq, persistent=False) self.max_seq_len_cached = self.original_max_seq_len @torch.no_grad() def forward(self, x, position_ids): if "dynamic" in self.rope_type: self._dynamic_frequency_update(position_ids, device=x.device) if self.rope_type == "default" and not self._inv_freq_is_valid(): self._reset_inv_freq(x.device) inv_freq_expanded = self.inv_freq[None, :, None].float().expand( position_ids.shape[0], -1, 1) position_ids_expanded = position_ids[:, None, :].float() device_type = x.device.type device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" with torch.autocast(device_type=device_type, enabled=False): freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) emb = torch.cat((freqs, freqs), dim=-1) cos = emb.cos() * self.attention_scaling sin = emb.sin() * self.attention_scaling return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) def rotate_half(x): x1 = x[..., : x.shape[-1] // 2] x2 = x[..., x.shape[-1] // 2 :] return torch.cat((-x2, x1), dim=-1) def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): cos = cos.unsqueeze(unsqueeze_dim) sin = sin.unsqueeze(unsqueeze_dim) q_embed = (q * cos) + (rotate_half(q) * sin) k_embed = (k * cos) + (rotate_half(k) * sin) return q_embed, k_embed def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: batch, num_key_value_heads, slen, head_dim = hidden_states.shape if n_rep == 1: return hidden_states hidden_states = hidden_states[:, :, None, :, :].expand( batch, num_key_value_heads, n_rep, slen, head_dim) return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) # --------------------------------------------------------------------------- # MLP # --------------------------------------------------------------------------- class MaxwellMoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() self.hidden_size = config.hidden_size self.intermediate_size = intermediate_size self.w1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.w3 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.w2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) self.act_fn = ACT2FN[config.hidden_act] def forward(self, x): return self.w2(self.act_fn(self.w1(x)) * self.w3(x)) # --------------------------------------------------------------------------- # Attention with QK LayerNorm # --------------------------------------------------------------------------- class MaxwellMoeAttention(nn.Module): def __init__(self, config: MaxwellMoeConfig, layer_idx: Optional[int] = None): super().__init__() self.config = config self.layer_idx = layer_idx self.hidden_size = config.hidden_size self.num_heads = config.num_attention_heads self.head_dim = (config.attention_head_size if config.attention_head_size else self.hidden_size // self.num_heads) self.num_key_value_heads = config.num_key_value_heads self.num_key_value_groups = self.num_heads // self.num_key_value_heads self.max_position_embeddings = config.max_position_embeddings self.rope_theta = config.rope_theta self.is_causal = True self.attention_dropout = config.attention_dropout self.qk_layernorm = getattr(config, 'qk_layernorm', False) self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) if self.qk_layernorm: self.q_layernorm = MaxwellMoeRMSNorm(self.head_dim, eps=config.rms_norm_eps) self.k_layernorm = MaxwellMoeRMSNorm(self.head_dim, eps=config.rms_norm_eps) self.rotary_emb = MaxwellMoeRotaryEmbedding(config=self.config) def forward( self, hidden_states, attention_mask=None, position_ids=None, past_key_value=None, output_attentions=False, use_cache=False, cache_position=None, position_embeddings=None, ): bsz, q_len, _ = hidden_states.size() query_states = self.q_proj(hidden_states) key_states = self.k_proj(hidden_states) value_states = self.v_proj(hidden_states) query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim) key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim) if self.qk_layernorm: query_states = self.q_layernorm(query_states) key_states = self.k_layernorm(key_states) query_states = query_states.transpose(1, 2) key_states = key_states.transpose(1, 2) value_states = value_states.view( bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) if position_embeddings is None: cos, sin = self.rotary_emb(value_states, position_ids) else: cos, sin = position_embeddings query_states, key_states = apply_rotary_pos_emb( query_states, key_states, cos, sin, position_ids) if past_key_value is not None: cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} key_states, value_states = past_key_value.update( key_states, value_states, self.layer_idx, cache_kwargs) key_states = repeat_kv(key_states, self.num_key_value_groups) value_states = repeat_kv(value_states, self.num_key_value_groups) attn_weights = torch.matmul( query_states.float(), key_states.float().transpose(2, 3)) / math.sqrt(self.head_dim) if attention_mask is not None: causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] attn_weights = attn_weights + causal_mask attn_weights = nn.functional.softmax( attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) attn_weights = nn.functional.dropout( attn_weights, p=self.attention_dropout, training=self.training) attn_output = torch.matmul(attn_weights, value_states) attn_output = attn_output.transpose(1, 2).contiguous() attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.head_dim) attn_output = self.o_proj(attn_output) if not output_attentions: attn_weights = None return attn_output, attn_weights, past_key_value class MaxwellMoeBlockSparseGQAIndexer(nn.Module): """Per-group index branch used by the Block Sparse GQA model.""" def __init__(self, config: MaxwellMoeConfig): super().__init__() self.num_groups = config.block_sparse_gqa_indexer_n_heads self.index_head_dim = config.block_sparse_gqa_indexer_head_dim self.index_softmax_scale = self.index_head_dim ** -0.5 self.linear_q_proj = nn.Linear( config.hidden_size, self.num_groups * self.index_head_dim, bias=False, ) self.linear_k_proj = nn.Linear( config.hidden_size, self.index_head_dim, bias=False, ) self.q_layernorm = ( MaxwellMoeRMSNorm( self.index_head_dim, eps=config.rms_norm_eps, ) if config.qk_layernorm else None ) self.k_layernorm = ( MaxwellMoeRMSNorm( self.index_head_dim, eps=config.rms_norm_eps, ) if config.qk_layernorm else None ) def project_qk(self, hidden_states, cos, sin): """Project detached hidden states and reuse the main branch RoPE.""" hidden_states = hidden_states.detach() batch_size, query_len, _ = hidden_states.shape query = self.linear_q_proj(hidden_states).view( batch_size, query_len, self.num_groups, self.index_head_dim, ) key = self.linear_k_proj(hidden_states).view( batch_size, query_len, 1, self.index_head_dim, ) if self.q_layernorm is not None: query = self.q_layernorm(query) if self.k_layernorm is not None: key = self.k_layernorm(key) query = query.transpose(1, 2) key = key.transpose(1, 2) query, key = apply_rotary_pos_emb(query, key, cos, sin) return query, key.squeeze(1) class MaxwellMoeBlockSparseGQACache(DynamicCache): """Dynamic main K/V cache plus the post-RoPE shared indexer K.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.indexer_key_states = [] def _ensure_indexer_layer(self, layer_idx): while len(self.indexer_key_states) <= layer_idx: self.indexer_key_states.append(None) def update_indexer_key_state(self, key_states, layer_idx): self._ensure_indexer_layer(layer_idx) previous = self.indexer_key_states[layer_idx] if previous is None: updated = key_states else: updated = torch.cat( ( previous.to( device=key_states.device, dtype=key_states.dtype, ), key_states, ), dim=1, ) self.indexer_key_states[layer_idx] = updated return updated def get_indexer_seq_length(self, layer_idx=0): if ( layer_idx >= len(self.indexer_key_states) or self.indexer_key_states[layer_idx] is None ): return 0 return int(self.indexer_key_states[layer_idx].shape[1]) def reorder_cache(self, beam_idx): result = super().reorder_cache(beam_idx) for layer_idx, state in enumerate(self.indexer_key_states): if state is not None: self.indexer_key_states[layer_idx] = state.index_select( 0, beam_idx.to(state.device) ) return result def batch_repeat_interleave(self, repeats): result = super().batch_repeat_interleave(repeats) for layer_idx, state in enumerate(self.indexer_key_states): if state is not None: self.indexer_key_states[layer_idx] = state.repeat_interleave( repeats, dim=0 ) return result def batch_select_indices(self, indices): result = super().batch_select_indices(indices) for layer_idx, state in enumerate(self.indexer_key_states): if state is not None: self.indexer_key_states[layer_idx] = state.index_select( 0, indices.to(state.device) ) return result def crop(self, max_length): result = super().crop(max_length) retained_length = self.get_seq_length() for layer_idx, state in enumerate(self.indexer_key_states): if state is not None: self.indexer_key_states[layer_idx] = state[:, :retained_length] return result def to_legacy_cache(self): raise ValueError( "block_sparse_gqa cannot export a legacy cache because that format " "cannot retain indexer K" ) @classmethod def from_legacy_cache(cls, past_key_values=None): if past_key_values: raise ValueError( "block_sparse_gqa cannot restore a non-empty legacy cache " "because historical indexer K is absent" ) return cls() class MaxwellMoeBlockSparseGQAAttention(MaxwellMoeAttention): """Eager block-sparse attention used by every backbone decoder layer. Combines block-aligned prefix/local support with per-group block-max remote selection. The MTP decoder uses its separate attention class. """ def __init__(self, config: MaxwellMoeConfig, layer_idx: Optional[int] = None): super().__init__(config, layer_idx) if layer_idx is None: raise ValueError("block_sparse_gqa attention requires layer_idx") self.indexer = MaxwellMoeBlockSparseGQAIndexer(config) self.block_size = config.block_sparse_gqa_block_size self.prefix_window_size = ( config.block_sparse_gqa_prefix_window_size ) self.local_window_size = config.block_sparse_gqa_local_window_size self.num_remote_blocks = config.block_sparse_gqa_num_remote_blocks self.num_prefix_blocks = self.prefix_window_size // self.block_size self.num_local_blocks = self.local_window_size // self.block_size self.query_chunk_size = config.block_sparse_gqa_query_chunk_size self.softmax_scale = self.head_dim ** -0.5 @staticmethod def _query_positions( *, batch_size, query_len, key_len, device, cache_position ): if cache_position is None: positions = torch.arange( key_len - query_len, key_len, device=device, dtype=torch.long, ) else: positions = cache_position.to( device=device, dtype=torch.long ).reshape(-1) if positions.numel() != query_len: raise ValueError( f"cache_position has {positions.numel()} entries for " f"query_len={query_len}" ) return positions.unsqueeze(0).expand(batch_size, -1) @staticmethod def _allowed_keys(*, query_positions, key_len, attention_mask): key_positions = torch.arange(key_len, device=query_positions.device) allowed = ( key_positions.view(1, 1, key_len) <= query_positions.unsqueeze(-1) ) if attention_mask is None: return allowed if attention_mask.dim() != 4 or attention_mask.dtype == torch.bool: raise ValueError( "block_sparse_gqa attention expects the model-level 4D " "additive causal mask" ) mask = attention_mask[ :, 0, -query_positions.shape[-1]:, :key_len ] return allowed & (mask >= 0) def _select_blocks(self, index_query, index_key, query_positions, allowed): """Select an independent block-max remote route for every GQA group.""" batch_size, num_groups, query_len, _ = index_query.shape key_len = index_key.shape[1] num_blocks = (key_len + self.block_size - 1) // self.block_size selected_count = min(max(self.num_remote_blocks, 1), num_blocks) block_starts = torch.arange( num_blocks, device=index_query.device, ) * self.block_size block_ends = block_starts + self.block_size selected_chunks = [] for query_start in range(0, query_len, self.query_chunk_size): query_end = min( query_start + self.query_chunk_size, query_len, ) chunk_len = query_end - query_start query_chunk = index_query[ :, :, query_start:query_end, : ] per_group_block_scores = torch.full( ( batch_size, num_groups, chunk_len, num_blocks, ), float("-inf"), dtype=torch.float32, device=index_query.device, ) # Match the Megatron max-route oracle while bounding the eager # token-score temporary for 256K contexts. key_blocks_per_chunk = max( 1, 8192 // self.block_size, ) for block_start in range( 0, num_blocks, key_blocks_per_chunk, ): block_end = min( block_start + key_blocks_per_chunk, num_blocks, ) key_start = block_start * self.block_size key_end = min( block_end * self.block_size, key_len, ) token_scores = torch.einsum( "bgqd,bkd->bgqk", query_chunk.float(), index_key[:, key_start:key_end].float(), ) token_scores.mul_(self.indexer.index_softmax_scale) token_scores.masked_fill_( ~allowed[ :, None, query_start:query_end, key_start:key_end, ], float("-inf"), ) block_count = block_end - block_start padding = ( block_count * self.block_size - (key_end - key_start) ) if padding: token_scores = F.pad( token_scores, (0, padding), value=float("-inf"), ) per_group_block_scores[ ..., block_start:block_end, ] = token_scores.view( batch_size, num_groups, chunk_len, block_count, self.block_size, ).amax(dim=-1) route_scores = per_group_block_scores.permute( 0, 2, 1, 3 ).contiguous() current_block = torch.div( query_positions[:, query_start:query_end], self.block_size, rounding_mode="floor", ) local_start_block = ( current_block - self.num_local_blocks + 1 ).clamp(min=0) local_start = local_start_block * self.block_size eligible = ( block_starts.view(1, 1, -1) >= self.prefix_window_size ) & ( block_ends.view(1, 1, -1) <= local_start.unsqueeze(-1) ) route_scores.masked_fill_( ~eligible.unsqueeze(2), float("-inf"), ) values, indices = route_scores.topk( selected_count, dim=-1, ) indices = torch.where( torch.isfinite(values), indices, -1, ) if selected_count < self.num_remote_blocks: indices = F.pad( indices, (0, self.num_remote_blocks - selected_count), value=-1, ) selected_chunks.append(indices) return torch.cat(selected_chunks, dim=1).contiguous() def _build_block_aligned_route(self, remote_blocks, query_positions): """Add the same fixed prefix/local blocks to every group route.""" if remote_blocks.ndim != 4: raise ValueError( "per-group remote blocks must be [batch, query, groups, remote]" ) batch_size, query_len, num_groups, _ = remote_blocks.shape current_block = torch.div( query_positions, self.block_size, rounding_mode="floor", ).unsqueeze(-1).unsqueeze(-1) prefix = torch.arange( self.num_prefix_blocks, device=remote_blocks.device, dtype=remote_blocks.dtype, ).view(1, 1, 1, -1) prefix = prefix.expand( batch_size, query_len, num_groups, -1 ) prefix = torch.where(prefix <= current_block, prefix, -1) local_offsets = torch.arange( 1 - self.num_local_blocks, 1, device=remote_blocks.device, dtype=remote_blocks.dtype, ).view(1, 1, 1, -1) local = current_block.to(remote_blocks.dtype) + local_offsets local = local.expand( batch_size, query_len, num_groups, -1 ) local = torch.where(local >= self.num_prefix_blocks, local, -1) return torch.cat( (prefix, remote_blocks, local), dim=-1, ).contiguous() def _sparse_attention( self, query_states, key_states, value_states, selected_blocks, allowed, ): batch_size, num_query_heads, query_len, head_dim = ( query_states.shape ) num_groups = key_states.shape[1] key_len = key_states.shape[2] if selected_blocks.ndim != 4 or selected_blocks.shape[:3] != ( batch_size, query_len, num_groups, ): raise ValueError( "selected blocks must be [batch, query, groups, blocks]" ) heads_per_group = num_query_heads // num_groups selected_tokens = selected_blocks.shape[-1] * self.block_size block_offsets = torch.arange( self.block_size, device=query_states.device, ) output = torch.empty( batch_size, num_query_heads, query_len, head_dim, dtype=value_states.dtype, device=value_states.device, ) for query_start in range(0, query_len, self.query_chunk_size): query_end = min( query_start + self.query_chunk_size, query_len, ) chunk_len = query_end - query_start selected_chunk = selected_blocks[ :, query_start:query_end ].permute(0, 2, 1, 3) token_positions = ( selected_chunk.unsqueeze(-1) * self.block_size + block_offsets ).reshape( batch_size, num_groups, chunk_len, selected_tokens, ) in_range = ( (token_positions >= 0) & (token_positions < key_len) ) gather_positions = token_positions.clamp( min=0, max=key_len - 1 ) allowed_chunk = allowed[ :, None, query_start:query_end ].expand( batch_size, num_groups, chunk_len, key_len, ) valid = in_range & torch.gather( allowed_chunk, 3, gather_positions, ) gather_index = gather_positions[..., None].expand( batch_size, num_groups, chunk_len, selected_tokens, head_dim, ) selected_key = torch.gather( key_states.unsqueeze(2).expand( batch_size, num_groups, chunk_len, key_len, head_dim, ), 3, gather_index, ) selected_value = torch.gather( value_states.unsqueeze(2).expand( batch_size, num_groups, chunk_len, key_len, head_dim, ), 3, gather_index, ) grouped_query = query_states[ :, :, query_start:query_end ].reshape( batch_size, num_groups, heads_per_group, chunk_len, head_dim, ) # Accumulate scores in fp32 like the Triton kernels; a bf16 einsum # output quantizes logits of ~1e3 to steps of several units. scores = torch.einsum( "bghqd,bgqkd->bghqk", grouped_query.float(), selected_key.float(), ) scores.mul_(self.softmax_scale) scores.masked_fill_( ~valid[:, :, None], float("-inf"), ) probabilities = torch.softmax( scores, dim=-1, dtype=torch.float32, ) probabilities = torch.nan_to_num( probabilities, nan=0.0, ).to(selected_value.dtype) context = torch.einsum( "bghqk,bgqkd->bghqd", probabilities, selected_value, ) output[ :, :, query_start:query_end ] = context.reshape( batch_size, num_query_heads, chunk_len, head_dim, ) return output def forward( self, hidden_states, attention_mask=None, position_ids=None, past_key_value=None, output_attentions=False, use_cache=False, cache_position=None, position_embeddings=None, ): if output_attentions: raise ValueError( "block_sparse_gqa does not materialize dense attention weights" ) batch_size, query_len, _ = hidden_states.shape query_states = self.q_proj(hidden_states).view( batch_size, query_len, self.num_heads, self.head_dim, ) key_states = self.k_proj(hidden_states).view( batch_size, query_len, self.num_key_value_heads, self.head_dim, ) value_states = self.v_proj(hidden_states).view( batch_size, query_len, self.num_key_value_heads, self.head_dim, ) if self.qk_layernorm: query_states = self.q_layernorm(query_states) key_states = self.k_layernorm(key_states) query_states = query_states.transpose(1, 2) key_states = key_states.transpose(1, 2) value_states = value_states.transpose(1, 2) if position_embeddings is None: cos, sin = self.rotary_emb(value_states, position_ids) else: cos, sin = position_embeddings query_states, key_states = apply_rotary_pos_emb( query_states, key_states, cos, sin, position_ids, ) index_query, index_key = self.indexer.project_qk( hidden_states, cos, sin, ) if past_key_value is not None: if not isinstance( past_key_value, MaxwellMoeBlockSparseGQACache, ): raise TypeError( "block_sparse_gqa requires " "MaxwellMoeBlockSparseGQACache" ) cache_kwargs = { "sin": sin, "cos": cos, "cache_position": cache_position, } key_states, value_states = past_key_value.update( key_states, value_states, self.layer_idx, cache_kwargs, ) index_key = past_key_value.update_indexer_key_state( index_key, self.layer_idx, ) if key_states.shape[-2] != index_key.shape[1]: raise RuntimeError( "main KV and indexer-K cache lengths diverged: " f"{key_states.shape[-2]} != {index_key.shape[1]}" ) query_positions = self._query_positions( batch_size=batch_size, query_len=query_len, key_len=key_states.shape[-2], device=query_states.device, cache_position=cache_position, ) allowed = self._allowed_keys( query_positions=query_positions, key_len=key_states.shape[-2], attention_mask=attention_mask, ) selected_blocks = self._select_blocks( index_query, index_key, query_positions, allowed, ) route_blocks = self._build_block_aligned_route( selected_blocks, query_positions, ) attention_output = self._sparse_attention( query_states, key_states, value_states, route_blocks, allowed, ) attention_output = attention_output.transpose( 1, 2 ).contiguous().reshape( batch_size, query_len, self.num_heads * self.head_dim, ) return self.o_proj(attention_output), None, past_key_value class MaxwellMoeFlashAttention2(MaxwellMoeAttention): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() def forward( self, hidden_states, attention_mask=None, position_ids=None, past_key_value=None, output_attentions=False, use_cache=False, cache_position=None, position_embeddings=None, ): bsz, q_len, _ = hidden_states.size() query_states = self.q_proj(hidden_states) key_states = self.k_proj(hidden_states) value_states = self.v_proj(hidden_states) query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim) key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim) if self.qk_layernorm: query_states = self.q_layernorm(query_states) key_states = self.k_layernorm(key_states) query_states = query_states.transpose(1, 2) key_states = key_states.transpose(1, 2) value_states = value_states.view( bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) if position_embeddings is None: cos, sin = self.rotary_emb(value_states, position_ids) else: cos, sin = position_embeddings query_states, key_states = apply_rotary_pos_emb( query_states, key_states, cos, sin) if past_key_value is not None: cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} key_states, value_states = past_key_value.update( key_states, value_states, self.layer_idx, cache_kwargs) key_states = repeat_kv(key_states, self.num_key_value_groups) value_states = repeat_kv(value_states, self.num_key_value_groups) dropout_rate = 0.0 if not self.training else self.attention_dropout input_dtype = query_states.dtype if input_dtype == torch.float32: if torch.is_autocast_enabled(): target_dtype = torch.get_autocast_gpu_dtype() elif hasattr(self.config, "_pre_quantization_dtype"): target_dtype = self.config._pre_quantization_dtype else: target_dtype = self.q_proj.weight.dtype query_states = query_states.to(target_dtype) key_states = key_states.to(target_dtype) value_states = value_states.to(target_dtype) query_states = query_states.transpose(1, 2) key_states = key_states.transpose(1, 2) value_states = value_states.transpose(1, 2) attn_output = _flash_attention_forward( query_states, key_states, value_states, attention_mask, q_len, position_ids=position_ids, dropout=dropout_rate, sliding_window=None, is_causal=self.is_causal, use_top_left_mask=self._flash_attn_uses_top_left_mask, ) attn_output = attn_output.reshape( bsz, q_len, self.num_heads * self.head_dim).contiguous() attn_output = self.o_proj(attn_output) if not output_attentions: attn_weights = None return attn_output, attn_weights, past_key_value class MaxwellMoeSdpaAttention(MaxwellMoeAttention): def forward( self, hidden_states, attention_mask=None, position_ids=None, past_key_value=None, output_attentions=False, use_cache=False, cache_position=None, position_embeddings=None, ): if output_attentions: return super().forward( hidden_states=hidden_states, attention_mask=attention_mask, position_ids=position_ids, past_key_value=past_key_value, output_attentions=output_attentions, use_cache=use_cache, ) bsz, q_len, _ = hidden_states.size() query_states = self.q_proj(hidden_states) key_states = self.k_proj(hidden_states) value_states = self.v_proj(hidden_states) query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim) key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim) if self.qk_layernorm: query_states = self.q_layernorm(query_states) key_states = self.k_layernorm(key_states) query_states = query_states.transpose(1, 2) key_states = key_states.transpose(1, 2) value_states = value_states.view( bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) if position_embeddings is None: cos, sin = self.rotary_emb(value_states, position_ids) else: cos, sin = position_embeddings query_states, key_states = apply_rotary_pos_emb( query_states, key_states, cos, sin) if past_key_value is not None: cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} key_states, value_states = past_key_value.update( key_states, value_states, self.layer_idx, cache_kwargs) key_states = repeat_kv(key_states, self.num_key_value_groups) value_states = repeat_kv(value_states, self.num_key_value_groups) causal_mask = attention_mask if attention_mask is not None: causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] if query_states.device.type == "cuda" and attention_mask is not None: query_states = query_states.contiguous() key_states = key_states.contiguous() value_states = value_states.contiguous() is_causal = True if causal_mask is None and q_len > 1 else False attn_output = torch.nn.functional.scaled_dot_product_attention( query_states, key_states, value_states, attn_mask=causal_mask, dropout_p=self.attention_dropout if self.training else 0.0, is_causal=is_causal, ) attn_output = attn_output.transpose(1, 2).contiguous() attn_output = attn_output.view(bsz, q_len, self.num_heads * self.head_dim) attn_output = self.o_proj(attn_output) return attn_output, None, past_key_value class MaxwellMoeTeAttention(MaxwellMoeAttention): """Attention using Transformer Engine's DotProductAttention kernel.""" def __init__(self, config, layer_idx=None): super().__init__(config, layer_idx) try: import transformer_engine.pytorch as te self.te_attn = te.DotProductAttention( num_attention_heads=self.num_heads, kv_channels=self.head_dim, num_gqa_groups=self.num_key_value_heads, attention_dropout=self.attention_dropout, qkv_format="bshd", attn_mask_type="causal", softmax_scale=1.0 / math.sqrt(self.head_dim), layer_number=layer_idx or 1, ) except ImportError: raise ImportError("transformer_engine is required for attn_implementation='te'") def forward( self, hidden_states, attention_mask=None, position_ids=None, past_key_value=None, output_attentions=False, use_cache=False, cache_position=None, position_embeddings=None, ): bsz, q_len, _ = hidden_states.size() query_states = self.q_proj(hidden_states) key_states = self.k_proj(hidden_states) value_states = self.v_proj(hidden_states) query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim) key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim) value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim) if self.qk_layernorm: query_states = self.q_layernorm(query_states) key_states = self.k_layernorm(key_states) if position_embeddings is None: cos, sin = self.rotary_emb(value_states, position_ids) else: cos, sin = position_embeddings # RoPE expects [bsz, heads, seq, dim] q_t = query_states.transpose(1, 2) k_t = key_states.transpose(1, 2) q_t, k_t = apply_rotary_pos_emb(q_t, k_t, cos, sin) query_states = q_t.transpose(1, 2).contiguous() key_states = k_t.transpose(1, 2).contiguous() if past_key_value is not None: k_for_cache = key_states.transpose(1, 2) v_for_cache = value_states.transpose(1, 2) cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} k_for_cache, v_for_cache = past_key_value.update( k_for_cache, v_for_cache, self.layer_idx, cache_kwargs) key_states = k_for_cache.transpose(1, 2).contiguous() value_states = v_for_cache.transpose(1, 2).contiguous() # TE DotProductAttention expects [bsz, seq, heads, dim] for bshd format attn_output = self.te_attn(query_states, key_states, value_states) attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.head_dim) attn_output = self.o_proj(attn_output) return attn_output, None, past_key_value MAXWELLMOE_ATTENTION_CLASSES = { "eager": MaxwellMoeAttention, "flash_attention_2": MaxwellMoeFlashAttention2, "sdpa": MaxwellMoeSdpaAttention, "te": MaxwellMoeTeAttention, } # --------------------------------------------------------------------------- # mHC (Manifold-Constrained Hyper-Connections) # --------------------------------------------------------------------------- def _sinkhorn_iterations(M: torch.Tensor, num_iterations: int, eps: float = 0.0): """TileKernels-aligned Sinkhorn-Knopp projection. Mirrors the TileKernels mhc_sinkhorn_kernel forward pass: 1. M = softmax(logits, dim=-1) + eps 2. M = M / (M.sum(dim=-2) + eps) # col norm 3. repeat-1 times: M = M / (M.sum(dim=-1) + eps) # row norm M = M / (M.sum(dim=-2) + eps) # col norm Input M is the softmax output from _compute_mappings. Megatron --use-tile-mhc passes eps=0.0 (tile_mhc_kernels.py:169-170). DeepSeek V4 HuggingFace uses eps=1e-6 (config.hc_eps). """ # softmax + eps (softmax already applied in _compute_mappings) M = M + eps # comb = comb / (comb.sum(-2) + eps) M = M / (M.sum(dim=-2, keepdim=True) + eps) for _ in range(num_iterations - 1): # comb = comb / (comb.sum(-1) + eps) M = M / (M.sum(dim=-1, keepdim=True) + eps) # comb = comb / (comb.sum(-2) + eps) M = M / (M.sum(dim=-2, keepdim=True) + eps) return M class HyperConnectionModule(nn.Module): def __init__(self, config: MaxwellMoeConfig): super().__init__() self.n = config.num_residual_streams self.hidden_size = config.hidden_size self.sinkhorn_iterations = config.mhc_sinkhorn_iterations self.eps = 1e-6 mix_dim = self.n * self.n + 2 * self.n self.mapping_proj = nn.Linear(self.n * self.hidden_size, mix_dim, bias=False) self.alpha_pre = nn.Parameter(torch.full((1,), config.mhc_init_gating_factor)) self.alpha_post = nn.Parameter(torch.full((1,), config.mhc_init_gating_factor)) self.alpha_res = nn.Parameter(torch.full((1,), config.mhc_init_gating_factor)) self.bias = nn.Parameter(torch.zeros(mix_dim)) def _compute_mappings(self, x): b, s, nC = x.shape n = self.n norm = x.float().norm(dim=-1, keepdim=True) r = 1.0 / (norm / math.sqrt(nC) + self.eps) proj = F.linear(x.float(), self.mapping_proj.weight.float()) alpha = torch.cat([ self.alpha_pre.expand(n), self.alpha_post.expand(n), self.alpha_res.expand(n * n), ], dim=-1).float() h = r.float() * proj * alpha + self.bias.float() h_pre = (h[..., :n].sigmoid() + self.eps).to(x.dtype) h_post = (h[..., n:2*n].sigmoid() * 2).to(x.dtype) h_res_logits = h[..., 2*n:] h_res = torch.softmax(h_res_logits.view(b, s, n, n), dim=-1) h_res = _sinkhorn_iterations(h_res, self.sinkhorn_iterations, eps=self.eps) h_res = h_res.to(x.dtype) return h_pre, h_post, h_res def aggregate(self, x, h_pre): b, s, _ = x.shape C = self.hidden_size x_streams = x.view(b, s, self.n, C).float() return (x_streams * h_pre.float().unsqueeze(-1)).sum(dim=2).to(x.dtype) def expand_and_mix(self, layer_output, residual, h_post, h_res): b, s, C = layer_output.shape n = self.n res_streams = residual.view(b, s, n, C).float() h_res_batched = h_res.view(b * s, n, n).float() res_batched = res_streams.view(b * s, n, C) mixed = torch.bmm(h_res_batched.transpose(-1, -2), res_batched).view(b, s, n, C) expanded = h_post.float().unsqueeze(-1) * layer_output.float().unsqueeze(2) return (expanded + mixed).to(residual.dtype).view(b, s, n * C) @staticmethod def input_expand(x, n): b, s, C = x.shape return x.unsqueeze(2).expand(b, s, n, C).contiguous().view(b, s, n * C) @staticmethod def output_contract(x, n): b, s, nC = x.shape C = nC // n return x.view(b, s, n, C).mean(dim=2) class HCHead(nn.Module): """Learned n-stream to 1-stream contraction replacing simple averaging.""" def __init__(self, config: MaxwellMoeConfig): super().__init__() self.n = config.num_residual_streams self.hidden_size = config.hidden_size self.eps = 1e-6 self.hc_head_fn = nn.Parameter( torch.zeros(self.n, self.n * self.hidden_size)) self.hc_head_base = nn.Parameter(torch.zeros(self.n)) self.hc_head_scale = nn.Parameter(torch.ones(1)) def forward(self, x): """ Args: x: [b, s, n*C] n-stream hidden states Returns: [b, s, C] contracted single-stream """ b, s, nC = x.shape n = self.n C = self.hidden_size dtype = x.dtype x_flat = x.float() rsqrt = torch.rsqrt(x_flat.square().mean(-1, keepdim=True) + self.eps) proj = F.linear(x_flat, self.hc_head_fn.float()) mixes = proj * rsqrt weights = torch.sigmoid(mixes * self.hc_head_scale.float() + self.hc_head_base.float()) + self.eps x_streams = x.view(b, s, n, C).float() out = (x_streams * weights.unsqueeze(-1)).sum(dim=2) return out.to(dtype) # --------------------------------------------------------------------------- # MoE Block with expert bias and sigmoid routing # --------------------------------------------------------------------------- class MaxwellMoeSparseMoeBlock(nn.Module): def __init__(self, config): super().__init__() self.num_experts = config.num_experts self.top_k = config.num_experts_per_tok self.use_shared_expert_gate = config.use_shared_expert_gate self.router_score_function = getattr(config, 'router_score_function', 'softmax') self.router_dtype = getattr(config, 'moe_router_dtype', 'fp32') self.gate = nn.Linear(config.hidden_size, config.num_experts, bias=False) if getattr(config, 'moe_router_enable_expert_bias', False): self.expert_bias = nn.Parameter(torch.zeros(config.num_experts), requires_grad=False) self.experts = nn.ModuleList( [MaxwellMoeMLP(config, intermediate_size=config.moe_intermediate_size) for _ in range(self.num_experts)]) if config.shared_expert_intermediate_size is not None: self.num_shared_experts = 1 self.shared_expert = MaxwellMoeMLP( config, intermediate_size=config.shared_expert_intermediate_size) if self.use_shared_expert_gate: self.shared_expert_gate = nn.Linear(config.hidden_size, 1, bias=False) else: self.num_shared_experts = 0 def _router_dtype(self, input_dtype): if self.router_dtype == 'fp32': return torch.float32 if self.router_dtype == 'fp64': return torch.float64 return input_dtype @staticmethod def _autocast_device_type(tensor): device_type = tensor.device.type return device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" def _compute_router_logits(self, hidden_states): router_dtype = self._router_dtype(hidden_states.dtype) device_type = self._autocast_device_type(hidden_states) with torch.autocast(device_type=device_type, enabled=False): return F.linear(hidden_states.to(router_dtype), self.gate.weight.to(router_dtype)) def _maintain_float32_expert_bias(self): if hasattr(self, 'expert_bias') and self.expert_bias.dtype != torch.float32: self.expert_bias.data = self.expert_bias.data.to(torch.float32) def forward(self, hidden_states): self._maintain_float32_expert_bias() batch_size, sequence_length, hidden_dim = hidden_states.shape hidden_states = hidden_states.view(-1, hidden_dim) router_logits = self._compute_router_logits(hidden_states) with torch.autocast(device_type=self._autocast_device_type(router_logits), enabled=False): if self.router_score_function == 'sigmoid': scores = torch.sigmoid(router_logits.float()) if hasattr(self, 'expert_bias'): scores_for_routing = scores + self.expert_bias.float() _, selected_experts = torch.topk(scores_for_routing, self.top_k, dim=-1) topk_scores = torch.gather(scores, dim=1, index=selected_experts) else: topk_scores, selected_experts = torch.topk(scores, self.top_k, dim=-1) routing_weights = ( topk_scores / (topk_scores.sum(dim=-1, keepdim=True) + 1e-20) ).type_as(router_logits) else: topk_routing_weights, selected_experts = torch.topk( router_logits, self.top_k, dim=-1) routing_weights = torch.softmax( topk_routing_weights, dim=1, dtype=torch.float).type_as(router_logits) final_hidden_states = torch.zeros( (batch_size * sequence_length, hidden_dim), dtype=hidden_states.dtype, device=hidden_states.device, ) expert_mask = torch.nn.functional.one_hot( selected_experts, num_classes=self.num_experts).permute(2, 1, 0) for expert_idx in range(self.num_experts): expert_layer = self.experts[expert_idx] idx, top_x = torch.where(expert_mask[expert_idx]) if top_x.numel() == 0: continue current_state = hidden_states[None, top_x].reshape(-1, hidden_dim) current_hidden_states = ( expert_layer(current_state) * routing_weights[top_x, idx, None]) final_hidden_states.index_add_( 0, top_x, current_hidden_states.to(hidden_states.dtype)) if self.num_shared_experts: shared_expert_output = self.shared_expert(hidden_states) if self.use_shared_expert_gate: shared_expert_output = ( F.sigmoid(self.shared_expert_gate(hidden_states)) * shared_expert_output) final_hidden_states = final_hidden_states + shared_expert_output final_hidden_states = final_hidden_states.reshape( batch_size, sequence_length, hidden_dim) return final_hidden_states, router_logits # --------------------------------------------------------------------------- # Decoder Layer # --------------------------------------------------------------------------- class MaxwellMoeDecoderLayer(nn.Module): def __init__(self, config: MaxwellMoeConfig, layer_idx: int): super().__init__() self.hidden_size = config.hidden_size self.enable_hyper_connections = config.enable_hyper_connections # Never let Transformers' backend auto-selection replace the # architecture-specific shared block route with plain dense SDPA. self.self_attn = MaxwellMoeBlockSparseGQAAttention( config, layer_idx ) is_dense = layer_idx in config.mlp_only_layers if is_dense or config.num_experts == 0: self.block_sparse_moe = MaxwellMoeMLP( config, intermediate_size=config.intermediate_size) else: self.block_sparse_moe = MaxwellMoeSparseMoeBlock(config) self.input_layernorm = MaxwellMoeRMSNorm( config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = MaxwellMoeRMSNorm( config.hidden_size, eps=config.rms_norm_eps) if self.enable_hyper_connections: self.self_attn_hyper_connection = HyperConnectionModule(config) self.mlp_hyper_connection = HyperConnectionModule(config) def forward( self, hidden_states, attention_mask=None, position_ids=None, past_key_value=None, output_attentions=False, output_router_logits=False, use_cache=False, cache_position=None, position_embeddings=None, **kwargs, ): if self.enable_hyper_connections: return self._forward_with_hc( hidden_states, attention_mask, position_ids, past_key_value, output_attentions, output_router_logits, use_cache, cache_position, position_embeddings, **kwargs) else: return self._forward_standard( hidden_states, attention_mask, position_ids, past_key_value, output_attentions, output_router_logits, use_cache, cache_position, position_embeddings, **kwargs) def _forward_standard( self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, output_router_logits, use_cache, cache_position, position_embeddings, **kwargs, ): residual = hidden_states hidden_states = self.input_layernorm(hidden_states) hidden_states, self_attn_weights, present_key_value = self.self_attn( hidden_states=hidden_states, attention_mask=attention_mask, position_ids=position_ids, past_key_value=past_key_value, output_attentions=output_attentions, use_cache=use_cache, cache_position=cache_position, position_embeddings=position_embeddings, ) hidden_states = residual + hidden_states residual = hidden_states hidden_states = self.post_attention_layernorm(hidden_states) hidden_states = self.block_sparse_moe(hidden_states) if isinstance(hidden_states, tuple): hidden_states, router_logits = hidden_states else: router_logits = None hidden_states = residual + hidden_states outputs = (hidden_states,) if output_attentions: outputs += (self_attn_weights,) if use_cache: outputs += (present_key_value,) if output_router_logits: outputs += (router_logits,) return outputs def _forward_with_hc( self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, output_router_logits, use_cache, cache_position, position_embeddings, **kwargs, ): # === Self-Attention with mHC === residual = hidden_states hc = self.self_attn_hyper_connection h_pre, h_post, h_res = hc._compute_mappings(hidden_states) aggregated = hc.aggregate(hidden_states, h_pre) aggregated = self.input_layernorm(aggregated) attn_output, self_attn_weights, present_key_value = self.self_attn( hidden_states=aggregated, attention_mask=attention_mask, position_ids=position_ids, past_key_value=past_key_value, output_attentions=output_attentions, use_cache=use_cache, cache_position=cache_position, position_embeddings=position_embeddings, ) hidden_states = hc.expand_and_mix(attn_output, residual, h_post, h_res) # === MLP with mHC === residual = hidden_states hc_mlp = self.mlp_hyper_connection h_pre, h_post, h_res = hc_mlp._compute_mappings(hidden_states) aggregated = hc_mlp.aggregate(hidden_states, h_pre) aggregated = self.post_attention_layernorm(aggregated) mlp_output = self.block_sparse_moe(aggregated) if isinstance(mlp_output, tuple): mlp_output, router_logits = mlp_output else: router_logits = None hidden_states = hc_mlp.expand_and_mix(mlp_output, residual, h_post, h_res) outputs = (hidden_states,) if output_attentions: outputs += (self_attn_weights,) if use_cache: outputs += (present_key_value,) if output_router_logits: outputs += (router_logits,) return outputs # --------------------------------------------------------------------------- # Model # --------------------------------------------------------------------------- MAXWELLMOE_START_DOCSTRING = "" MAXWELLMOE_INPUTS_DOCSTRING = "" class MaxwellMoePreTrainedModel(PreTrainedModel): config_class = MaxwellMoeConfig base_model_prefix = "model" supports_gradient_checkpointing = True _no_split_modules = ["MaxwellMoeDecoderLayer"] _skip_keys_device_placement = "past_key_values" _supports_flash_attn_2 = False _supports_sdpa = False _supports_cache_class = True def _init_weights(self, module): std = self.config.initializer_range if isinstance(module, nn.Linear): module.weight.data.normal_(mean=0.0, std=std) if module.bias is not None: module.bias.data.zero_() elif isinstance(module, nn.Embedding): module.weight.data.normal_(mean=0.0, std=std) if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() class MaxwellMoeModel(MaxwellMoePreTrainedModel): def __init__(self, config: MaxwellMoeConfig): super().__init__(config) self.padding_idx = config.pad_token_id self.vocab_size = config.vocab_size self.enable_hyper_connections = config.enable_hyper_connections self.num_residual_streams = config.num_residual_streams self.embed_tokens = nn.Embedding( config.vocab_size, config.hidden_size, self.padding_idx) self.layers = nn.ModuleList( [MaxwellMoeDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]) self._attn_implementation = config._attn_implementation self.norm = MaxwellMoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.rotary_emb = MaxwellMoeRotaryEmbedding(config=config) if self.enable_hyper_connections: self.hc_head = HCHead(config) self.gradient_checkpointing = False self.post_init() def get_input_embeddings(self): return self.embed_tokens def set_input_embeddings(self, value): self.embed_tokens = value @staticmethod def _normalize_block_sparse_gqa_attention_mask(attention_mask): """Accept only masks that do not alter physical sparse coordinates. Block Sparse GQA defines its route in physical KV-cache coordinates. Padding and packed/custom masks need a logical-to-physical token map that this bounded HF oracle does not implement, so reject them instead of silently changing the route. """ if attention_mask is None: return None if not isinstance(attention_mask, torch.Tensor): raise TypeError( "block_sparse_gqa attention_mask must be a torch.Tensor or None" ) if attention_mask.dim() == 2: if not torch.all(attention_mask == 1).item(): raise ValueError( "block_sparse_gqa HF does not support padding: a 2D " "attention_mask must contain only ones" ) # Canonicalize the no-op mask so the model constructs its causal # additive mask in one place. return None if attention_mask.dim() == 4: if attention_mask.dtype == torch.bool: raise ValueError( "block_sparse_gqa HF does not support 4D boolean masks or " "packed inputs" ) if not torch.all(attention_mask == 0).item(): raise ValueError( "block_sparse_gqa HF does not support nontrivial 4D " "additive masks, padding, or packed inputs" ) # An all-zero additive mask is a no-op. Canonicalize it to None # so it cannot accidentally replace the internal causal mask. return None raise ValueError( "block_sparse_gqa HF supports only an all-ones 2D mask, an " "all-zero 4D additive mask, or no attention mask" ) @staticmethod def _validate_block_sparse_gqa_cache(past_key_values): """Require main K/V and indexer K to remain synchronized.""" if past_key_values is None: return if not isinstance(past_key_values, MaxwellMoeBlockSparseGQACache): raise TypeError( "block_sparse_gqa requires MaxwellMoeBlockSparseGQACache; " "ordinary, legacy, static, sliding, and quantized caches omit " "historical indexer K" ) main_length = past_key_values.get_seq_length() for layer_idx, state in enumerate( past_key_values.indexer_key_states ): if state is not None and int(state.shape[1]) != main_length: raise ValueError( "block_sparse_gqa cache is inconsistent at layer " f"{layer_idx}: main length={main_length}, " f"indexer length={state.shape[1]}" ) def forward( self, input_ids=None, attention_mask=None, position_ids=None, past_key_values=None, inputs_embeds=None, use_cache=None, output_attentions=None, output_hidden_states=None, output_router_logits=None, return_dict=None, cache_position=None, output_multistream=False, ): output_attentions = (output_attentions if output_attentions is not None else self.config.output_attentions) output_router_logits = (output_router_logits if output_router_logits is not None else self.config.output_router_logits) output_hidden_states = (output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states) use_cache = use_cache if use_cache is not None else self.config.use_cache return_dict = return_dict if return_dict is not None else self.config.use_return_dict if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") if self.gradient_checkpointing and self.training: raise ValueError( "block_sparse_gqa HF is an inference reference and does not " "support gradient checkpointing" ) if use_cache: if past_key_values is None: past_key_values = MaxwellMoeBlockSparseGQACache() elif not isinstance( past_key_values, MaxwellMoeBlockSparseGQACache ): existing_length = ( past_key_values.get_seq_length() if isinstance(past_key_values, Cache) else len(past_key_values) if isinstance(past_key_values, (tuple, list)) else None ) if existing_length not in (0, None): raise ValueError( "block_sparse_gqa cannot restore a non-empty cache " "without historical indexer K" ) if existing_length is None: raise TypeError( "block_sparse_gqa received an unsupported cache type" ) past_key_values = MaxwellMoeBlockSparseGQACache() self._validate_block_sparse_gqa_cache(past_key_values) attention_mask = self._normalize_block_sparse_gqa_attention_mask( attention_mask ) if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) if cache_position is None: past_seen_tokens = (past_key_values.get_seq_length() if past_key_values is not None else 0) cache_position = torch.arange( past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device) if position_ids is None: position_ids = cache_position.unsqueeze(0) causal_mask = self._update_causal_mask( attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions) hidden_states = inputs_embeds if self.enable_hyper_connections: hidden_states = HyperConnectionModule.input_expand( hidden_states, self.num_residual_streams) position_embeddings = self.rotary_emb(hidden_states, position_ids) all_hidden_states = () if output_hidden_states else None all_self_attns = () if output_attentions else None all_router_logits = () if output_router_logits else None next_decoder_cache = None for decoder_layer in self.layers: if output_hidden_states: all_hidden_states += (hidden_states,) if self.gradient_checkpointing and self.training: layer_outputs = self._gradient_checkpointing_func( decoder_layer.__call__, hidden_states, causal_mask, position_ids, past_key_values, output_attentions, output_router_logits, use_cache, cache_position, position_embeddings, ) else: layer_outputs = decoder_layer( hidden_states, attention_mask=causal_mask, position_ids=position_ids, past_key_value=past_key_values, output_attentions=output_attentions, output_router_logits=output_router_logits, use_cache=use_cache, cache_position=cache_position, position_embeddings=position_embeddings, ) hidden_states = layer_outputs[0] if use_cache: next_decoder_cache = layer_outputs[2 if output_attentions else 1] if output_attentions: all_self_attns += (layer_outputs[1],) if output_router_logits and layer_outputs[-1] is not None: all_router_logits += (layer_outputs[-1],) multistream_hidden_states = None if output_multistream and self.enable_hyper_connections: multistream_hidden_states = hidden_states if self.enable_hyper_connections: hidden_states = self.hc_head(hidden_states) hidden_states = self.norm(hidden_states) if output_hidden_states: all_hidden_states += (hidden_states,) next_cache = next_decoder_cache if use_cache else None if not return_dict: result = tuple(v for v in [ hidden_states, next_cache, all_hidden_states, all_self_attns, all_router_logits ] if v is not None) if output_multistream: return result, multistream_hidden_states, position_embeddings, causal_mask, position_ids, cache_position return result model_output = MoeModelOutputWithPast( last_hidden_state=hidden_states, past_key_values=next_cache, hidden_states=all_hidden_states, attentions=all_self_attns, router_logits=all_router_logits, ) if output_multistream: return model_output, multistream_hidden_states, position_embeddings, causal_mask, position_ids, cache_position return model_output def _update_causal_mask(self, attention_mask, input_tensor, cache_position, past_key_values, output_attentions): if self.config._attn_implementation == "flash_attention_2": if attention_mask is not None and 0.0 in attention_mask: return attention_mask return None past_seen_tokens = (past_key_values.get_seq_length() if past_key_values is not None else 0) using_static_cache = isinstance(past_key_values, StaticCache) if (self.config._attn_implementation == "sdpa" and not using_static_cache and not output_attentions): if AttentionMaskConverter._ignore_causal_mask_sdpa( attention_mask, inputs_embeds=input_tensor, past_key_values_length=past_seen_tokens, sliding_window=None, is_training=self.training, ): return None dtype, device = input_tensor.dtype, input_tensor.device min_dtype = torch.finfo(dtype).min sequence_length = input_tensor.shape[1] if using_static_cache: target_length = past_key_values.get_max_cache_shape() else: target_length = ( attention_mask.shape[-1] if isinstance(attention_mask, torch.Tensor) else past_seen_tokens + sequence_length + 1) causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position( attention_mask, sequence_length=sequence_length, target_length=target_length, dtype=dtype, device=device, cache_position=cache_position, batch_size=input_tensor.shape[0], config=self.config, past_key_values=past_key_values, ) if (self.config._attn_implementation == "sdpa" and attention_mask is not None and attention_mask.device.type == "cuda" and not output_attentions): causal_mask = AttentionMaskConverter._unmask_unattended( causal_mask, min_dtype) return causal_mask @staticmethod def _prepare_4d_causal_attention_mask_with_cache_position( attention_mask, sequence_length, target_length, dtype, device, cache_position, batch_size, config, past_key_values, ): if attention_mask is not None and attention_mask.dim() == 4: causal_mask = attention_mask else: min_dtype = torch.finfo(dtype).min causal_mask = torch.full( (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device) diagonal_attend_mask = ( torch.arange(target_length, device=device) > cache_position.reshape(-1, 1)) causal_mask *= diagonal_attend_mask causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1) if attention_mask is not None: causal_mask = causal_mask.clone() if attention_mask.shape[-1] > target_length: attention_mask = attention_mask[:, :target_length] mask_length = attention_mask.shape[-1] padding_mask = ( causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :]) == 0 causal_mask[:, :, :, :mask_length] = causal_mask[ :, :, :, :mask_length].masked_fill(padding_mask, min_dtype) return causal_mask # --------------------------------------------------------------------------- # MTP (Multi-Token Prediction) # --------------------------------------------------------------------------- # # The MTP layer's inner transformer layer uses plain (non block-sparse) GQA # attention -- confirmed against the actual mcore checkpoint tensors # (mtp.layers.0.transformer_layer.self_attention.* has no indexer.* # counterpart). So MaxwellMoeMTPDecoderLayer reuses this file's existing # plain attention classes (MAXWELLMOE_ATTENTION_CLASSES) rather than the # block-sparse MaxwellMoeDecoderLayer used by the main 45-layer decoder. class MaxwellMoeMTPDecoderLayer(nn.Module): def __init__(self, config: MaxwellMoeConfig, layer_idx: int): super().__init__() self.hidden_size = config.hidden_size self.enable_hyper_connections = config.enable_hyper_connections self.self_attn = MAXWELLMOE_ATTENTION_CLASSES[ config._attn_implementation](config, layer_idx) is_dense = layer_idx in config.mlp_only_layers if is_dense or config.num_experts == 0: self.block_sparse_moe = MaxwellMoeMLP( config, intermediate_size=config.intermediate_size) else: self.block_sparse_moe = MaxwellMoeSparseMoeBlock(config) self.input_layernorm = MaxwellMoeRMSNorm( config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = MaxwellMoeRMSNorm( config.hidden_size, eps=config.rms_norm_eps) if self.enable_hyper_connections: self.self_attn_hyper_connection = HyperConnectionModule(config) self.mlp_hyper_connection = HyperConnectionModule(config) def forward( self, hidden_states, attention_mask=None, position_ids=None, past_key_value=None, output_attentions=False, output_router_logits=False, use_cache=False, cache_position=None, position_embeddings=None, **kwargs, ): if self.enable_hyper_connections: return self._forward_with_hc( hidden_states, attention_mask, position_ids, past_key_value, output_attentions, output_router_logits, use_cache, cache_position, position_embeddings, **kwargs) else: return self._forward_standard( hidden_states, attention_mask, position_ids, past_key_value, output_attentions, output_router_logits, use_cache, cache_position, position_embeddings, **kwargs) def _forward_standard( self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, output_router_logits, use_cache, cache_position, position_embeddings, **kwargs, ): residual = hidden_states hidden_states = self.input_layernorm(hidden_states) hidden_states, self_attn_weights, present_key_value = self.self_attn( hidden_states=hidden_states, attention_mask=attention_mask, position_ids=position_ids, past_key_value=past_key_value, output_attentions=output_attentions, use_cache=use_cache, cache_position=cache_position, position_embeddings=position_embeddings, ) hidden_states = residual + hidden_states residual = hidden_states hidden_states = self.post_attention_layernorm(hidden_states) hidden_states = self.block_sparse_moe(hidden_states) if isinstance(hidden_states, tuple): hidden_states, router_logits = hidden_states else: router_logits = None hidden_states = residual + hidden_states outputs = (hidden_states,) if output_attentions: outputs += (self_attn_weights,) if use_cache: outputs += (present_key_value,) if output_router_logits: outputs += (router_logits,) return outputs def _forward_with_hc( self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, output_router_logits, use_cache, cache_position, position_embeddings, **kwargs, ): residual = hidden_states hc = self.self_attn_hyper_connection h_pre, h_post, h_res = hc._compute_mappings(hidden_states) aggregated = hc.aggregate(hidden_states, h_pre) aggregated = self.input_layernorm(aggregated) attn_output, self_attn_weights, present_key_value = self.self_attn( hidden_states=aggregated, attention_mask=attention_mask, position_ids=position_ids, past_key_value=past_key_value, output_attentions=output_attentions, use_cache=use_cache, cache_position=cache_position, position_embeddings=position_embeddings, ) hidden_states = hc.expand_and_mix(attn_output, residual, h_post, h_res) residual = hidden_states hc_mlp = self.mlp_hyper_connection h_pre, h_post, h_res = hc_mlp._compute_mappings(hidden_states) aggregated = hc_mlp.aggregate(hidden_states, h_pre) aggregated = self.post_attention_layernorm(aggregated) mlp_output = self.block_sparse_moe(aggregated) if isinstance(mlp_output, tuple): mlp_output, router_logits = mlp_output else: router_logits = None hidden_states = hc_mlp.expand_and_mix(mlp_output, residual, h_post, h_res) outputs = (hidden_states,) if output_attentions: outputs += (self_attn_weights,) if use_cache: outputs += (present_key_value,) if output_router_logits: outputs += (router_logits,) return outputs class MaxwellMoeMTPLayer(nn.Module): """Multi-Token Prediction layer. Takes the multi-stream hidden states from the decoder, combines with rolled (next-position) token embeddings via separate projections, passes through a full MoE transformer layer, and contracts back to a single-stream representation for the shared lm_head. """ def __init__(self, config: MaxwellMoeConfig, layer_idx: int): super().__init__() self.config = config n = config.num_residual_streams h = config.hidden_size self.enorm = MaxwellMoeRMSNorm(h, eps=config.rms_norm_eps) self.hnorm = MaxwellMoeRMSNorm(h, eps=config.rms_norm_eps) self.e_proj = nn.Linear(h, h, bias=False) self.h_proj = nn.Linear(h, h, bias=False) mtp_layer_idx = config.num_hidden_layers + layer_idx self.decoder_layer = MaxwellMoeMTPDecoderLayer(config, layer_idx=mtp_layer_idx) self.hc_head = HCHead(config) self.final_layernorm = MaxwellMoeRMSNorm(h, eps=config.rms_norm_eps) def forward( self, multistream_hidden_states, input_ids, position_ids, embed_tokens, attention_mask=None, position_embeddings=None, cache_position=None, ): """ Args: multistream_hidden_states: [b, s, n*h] multi-stream from decoder input_ids: [b, s] current token IDs position_ids: [b, s] or [1, s] embed_tokens: embedding layer (shared with main model) attention_mask: causal mask position_embeddings: (cos, sin) from rotary embedding Returns: hidden_states: [b, s, h] contracted single-stream output mtp_multistream: [b, s, n*h] multi-stream output (for chaining) rolled_input_ids: [b, s] rolled input IDs rolled_position_ids: [b, s] rolled position IDs """ b, s, nC = multistream_hidden_states.shape n = self.config.num_residual_streams h = self.config.hidden_size rolled_input_ids = torch.roll(input_ids, shifts=-1, dims=-1) rolled_input_ids[:, -1] = self.config.pad_token_id rolled_position_ids = torch.roll(position_ids, shifts=-1, dims=-1) rolled_position_ids[:, -1] = 0 decoder_input = embed_tokens(rolled_input_ids) decoder_input = self.enorm(decoder_input) hs_streams = multistream_hidden_states.view(b, s, n, h) hs_streams = self.hnorm(hs_streams) e_out = self.e_proj(decoder_input) h_out = self.h_proj(hs_streams) e_out = e_out.unsqueeze(2).expand(b, s, n, h) hidden_states = (e_out + h_out).reshape(b, s, n * h) layer_outputs = self.decoder_layer( hidden_states, attention_mask=attention_mask, position_ids=position_ids, position_embeddings=position_embeddings, cache_position=cache_position, ) mtp_multistream = layer_outputs[0] hidden_states = self.hc_head(mtp_multistream) hidden_states = self.final_layernorm(hidden_states) return hidden_states, mtp_multistream, rolled_input_ids, rolled_position_ids @dataclass class MTPCausalLMOutput(ModelOutput): loss: Optional[torch.FloatTensor] = None logits: torch.FloatTensor = None mtp_logits: Optional[torch.FloatTensor] = None past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None attentions: Optional[Tuple[torch.FloatTensor, ...]] = None router_logits: Optional[Tuple[torch.FloatTensor]] = None class MaxwellMoeForCausalLM(MaxwellMoePreTrainedModel, GenerationMixin): """MaxwellMoE with Multi-Token Prediction head for speculative decoding.""" _tied_weights_keys = ["lm_head.weight"] def __init__(self, config): super().__init__(config) self.model = MaxwellMoeModel(config) self.vocab_size = config.vocab_size self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) self.router_aux_loss_coef = config.router_aux_loss_coef self.num_experts = config.num_experts self.num_experts_per_tok = config.num_experts_per_tok self.mtp_num_layers = getattr(config, 'mtp_num_layers', 0) if self.mtp_num_layers > 0: self.mtp_layers = nn.ModuleList([ MaxwellMoeMTPLayer(config, layer_idx=i) for i in range(self.mtp_num_layers) ]) else: self.mtp_layers = nn.ModuleList() self.post_init() def get_input_embeddings(self): return self.model.embed_tokens def set_input_embeddings(self, value): self.model.embed_tokens = value def get_output_embeddings(self): return self.lm_head def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings def set_decoder(self, decoder): self.model = decoder def get_decoder(self): return self.model def _fix_expert_bias_precision(self): """Restore decoder and MTP expert_bias to fp32 from safetensors (first forward). from_pretrained(dtype=bf16) truncates expert_bias, and the per-expert spread (range ~0.005) is below bf16's quantization step (~0.06), so routing collapses without it. """ self._expert_bias_fp32_done = True import json, os from safetensors import safe_open model_dir = self.config._name_or_path idx_path = os.path.join(model_dir, "model.safetensors.index.json") if not os.path.exists(idx_path): return with open(idx_path) as f: weight_map = json.load(f)["weight_map"] for layer_idx, layer in enumerate(self.model.layers): moe = getattr(layer, 'block_sparse_moe', None) if moe is None or not hasattr(moe, 'expert_bias'): continue key = f"model.layers.{layer_idx}.block_sparse_moe.expert_bias" shard = weight_map.get(key) if shard is None: continue with safe_open(os.path.join(model_dir, shard), framework="pt") as f: moe.expert_bias.data = f.get_tensor(key).float().to(moe.expert_bias.device) for mtp_idx, mtp_layer in enumerate(self.mtp_layers): moe = getattr(mtp_layer.decoder_layer, 'block_sparse_moe', None) if moe is None or not hasattr(moe, 'expert_bias'): continue key = f"mtp_layers.{mtp_idx}.decoder_layer.block_sparse_moe.expert_bias" shard = weight_map.get(key) if shard is None: continue with safe_open(os.path.join(model_dir, shard), framework="pt") as f: moe.expert_bias.data = f.get_tensor(key).float().to(moe.expert_bias.device) def forward( self, input_ids=None, attention_mask=None, position_ids=None, past_key_values=None, inputs_embeds=None, labels=None, use_cache=None, output_attentions=None, output_hidden_states=None, output_router_logits=None, return_dict=None, cache_position=None, num_logits_to_keep=0, output_mtp_logits=False, ): if not getattr(self, '_expert_bias_fp32_done', False): self._fix_expert_bias_precision() output_attentions = (output_attentions if output_attentions is not None else self.config.output_attentions) output_router_logits = (output_router_logits if output_router_logits is not None else self.config.output_router_logits) output_hidden_states = (output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states) return_dict = return_dict if return_dict is not None else self.config.use_return_dict need_mtp = output_mtp_logits and len(self.mtp_layers) > 0 if need_mtp: result = self.model( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, output_router_logits=output_router_logits, return_dict=return_dict, cache_position=cache_position, output_multistream=True, ) if return_dict: outputs, multistream_hidden_states, position_embeddings, causal_mask, resolved_position_ids, resolved_cache_position = result else: base_result, multistream_hidden_states, position_embeddings, causal_mask, resolved_position_ids, resolved_cache_position = result outputs = base_result else: outputs = self.model( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, output_router_logits=output_router_logits, return_dict=return_dict, cache_position=cache_position, ) hidden_states = outputs[0] if not return_dict else outputs.last_hidden_state lm_input = hidden_states[:, -num_logits_to_keep:, :] logits = torch.nn.functional.linear(lm_input.float(), self.lm_head.weight.float(), self.lm_head.bias if self.lm_head.bias is not None else None) mtp_logits = None if need_mtp and multistream_hidden_states is not None: cur_input_ids = input_ids cur_position_ids = resolved_position_ids cur_multistream = multistream_hidden_states for mtp_layer in self.mtp_layers: mtp_hidden, cur_multistream, cur_input_ids, cur_position_ids = mtp_layer( multistream_hidden_states=cur_multistream, input_ids=cur_input_ids, position_ids=cur_position_ids, embed_tokens=self.model.embed_tokens, attention_mask=causal_mask, position_embeddings=position_embeddings, cache_position=resolved_cache_position, ) mtp_logits = torch.nn.functional.linear(mtp_hidden.float(), self.lm_head.weight.float(), self.lm_head.bias if self.lm_head.bias is not None else None) loss = None if labels is not None: loss = self.loss_function(logits, labels, self.vocab_size) if not return_dict: output = (logits,) + outputs[1:] if mtp_logits is not None: output = output + (mtp_logits,) return (loss,) + output if loss is not None else output return MTPCausalLMOutput( loss=loss, logits=logits, mtp_logits=mtp_logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions, router_logits=outputs.router_logits, )
# coding=utf-8
# Copyright 2026 the Maxwell team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""PyTorch MaxwellMoE model — w35a3 variant.
Supports: mHC, QK LayerNorm, expert bias, sigmoid routing, learned HC head,
mixed dense/MoE layers, shared experts.
"""
import math
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union
import torch
import torch.nn.functional as F
import torch.utils.checkpoint
from torch import nn
from transformers.activations import ACT2FN
from transformers.cache_utils import Cache, DynamicCache, StaticCache
from transformers.generation import GenerationMixin
from transformers.modeling_attn_mask_utils import AttentionMaskConverter
from transformers.modeling_outputs import (
MoeModelOutputWithPast,
ModelOutput,
)
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
from transformers.modeling_utils import PreTrainedModel
from transformers.utils import (
add_start_docstrings,
add_start_docstrings_to_model_forward,
is_flash_attn_2_available,
logging,
replace_return_docstrings,
)
try:
from transformers.utils import is_flash_attn_greater_or_equal_2_10
except ImportError:
try:
from transformers.utils import is_flash_attn_greater_or_equal
def is_flash_attn_greater_or_equal_2_10():
return is_flash_attn_greater_or_equal("2.1.0")
except ImportError:
def is_flash_attn_greater_or_equal_2_10():
return True
from .configuration_maxwell_moe import MaxwellMoeConfig
if is_flash_attn_2_available():
from transformers.modeling_flash_attention_utils import _flash_attention_forward
logger = logging.get_logger(__name__)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
class MaxwellMoeRMSNorm(nn.Module):
def __init__(self, hidden_size, eps=1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, hidden_states):
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
# Apply the weight in fp32 and round once, as TE RMSNorm does. Rounding
# before the multiply is visible at large norm weights (q/k norms of the
# last layer reach ~14) where attention logits run into the thousands.
return (self.weight.float() * hidden_states).to(input_dtype)
class MaxwellMoeRotaryEmbedding(nn.Module):
def __init__(self, dim=None, max_position_embeddings=2048, base=10000,
device=None, scaling_factor=1.0, rope_type="default",
config: Optional[MaxwellMoeConfig] = None):
super().__init__()
self.rope_kwargs = {}
if config is None:
self.rope_kwargs = {
"rope_type": rope_type, "factor": scaling_factor,
"dim": dim, "base": base,
"max_position_embeddings": max_position_embeddings,
}