# Copyright 2026 FrontiersMind. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Nandi encoder: the Nandi decoder architecture with bidirectional attention, trained with masked language modeling. Forward pass: embed_tokens -> embedding_proj (factorized) -> each of the N layers applied `layer_sharing_repeats` times back to back (0, 0, 1, 1, ...) (pre-norm: x + attn(rmsnorm(x)), x + swiglu(rmsnorm(x)); GQA attention with RoPE, NOT causal) -> final RMSNorm -> MLM head: either lm_head_proj (factorized) -> lm_head (tied to embed_tokens), or (mlm_head='dense') dense -> act -> RMSNorm -> untied lm_head. Optional QK-norm (per head, before RoPE). Sequence / token classifiers: with classifier_head='mlm' the pretrained dense MLM head (dense -> act -> RMSNorm) sits between the encoder and the new linear layer (sequence: pool -> head -> linear; token: head per token -> linear). Inputs start with `<|endoftext|>` (the tokenizer adds it), as every training document did. """ from collections.abc import Callable import torch import torch.nn as nn from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from transformers.activations import ACT2FN from transformers.masking_utils import create_bidirectional_mask from transformers.modeling_layers import GradientCheckpointingLayer from transformers.modeling_outputs import ( BaseModelOutput, MaskedLMOutput, SequenceClassifierOutput, TokenClassifierOutput, ) from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from transformers.processing_utils import Unpack from transformers.utils import TransformersKwargs from .configuration_nandi_encoder import NandiEncoderConfig class NandiRMSNorm(nn.Module): def __init__(self, hidden_size, eps=1e-5): super().__init__() self.weight = nn.Parameter(torch.ones(hidden_size)) self.variance_epsilon = eps def forward(self, hidden_states): # fp32 normalize AND weight multiply, then cast: matches flash-attn's Triton RMSNorm used in training input_dtype = hidden_states.dtype hidden_states = hidden_states.to(torch.float32) variance = hidden_states.pow(2).mean(-1, keepdim=True) hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) return (self.weight.float() * hidden_states).to(input_dtype) def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" class NandiRotaryEmbedding(nn.Module): inv_freq: torch.Tensor def __init__(self, config: NandiEncoderConfig, device=None): super().__init__() self.max_seq_len_cached = config.max_position_embeddings self.original_max_seq_len = config.max_position_embeddings self.config = config self.rope_type = self.config.rope_parameters.get("rope_type", "default") rope_init_fn: Callable = self.compute_default_rope_parameters if self.rope_type != "default": rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] inv_freq, self.attention_scaling = rope_init_fn(self.config, device) self.register_buffer("inv_freq", inv_freq, persistent=False) self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False) @staticmethod def compute_default_rope_parameters(config=None, device=None, seq_len=None): del seq_len base = config.rope_parameters["rope_theta"] dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim)) return inv_freq, 1.0 @torch.no_grad() @dynamic_rope_update def forward(self, x, position_ids): inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device) position_ids_expanded = position_ids[:, None, :].float() device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" with torch.autocast(device_type=device_type, enabled=False): freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) emb = torch.cat((freqs, freqs), dim=-1) cos = emb.cos() * self.attention_scaling sin = emb.sin() * self.attention_scaling return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) def rotate_half(x): x1 = x[..., : x.shape[-1] // 2] x2 = x[..., x.shape[-1] // 2 :] return torch.cat((-x2, x1), dim=-1) def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): cos = cos.unsqueeze(unsqueeze_dim) sin = sin.unsqueeze(unsqueeze_dim) return (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin) def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: batch, num_key_value_heads, slen, head_dim = hidden_states.shape if n_rep == 1: return hidden_states hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) def eager_attention_forward( module: nn.Module, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, attention_mask: torch.Tensor | None, scaling: float, dropout: float = 0.0, **kwargs, ): del kwargs key_states = repeat_kv(key, module.num_key_value_groups) value_states = repeat_kv(value, module.num_key_value_groups) attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling if attention_mask is not None: attn_weights = attn_weights + attention_mask[:, :, :, : key_states.shape[-2]] attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) attn_output = torch.matmul(attn_weights, value_states) return attn_output.transpose(1, 2).contiguous(), attn_weights class NandiEncoderAttention(nn.Module): """GQA self-attention with RoPE. Bidirectional: every token attends to every non-padding token.""" def __init__(self, config: NandiEncoderConfig, layer_idx: int): super().__init__() self.config = config self.layer_idx = layer_idx self.head_dim = config.head_dim self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads self.scaling = self.head_dim**-0.5 self.attention_dropout = config.attention_dropout self.is_causal = False self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias) self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias) self.v_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias) self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias) self.qk_norm = getattr(config, "qk_norm", False) if self.qk_norm: self.q_norm = NandiRMSNorm(self.head_dim, eps=config.rms_norm_eps) self.k_norm = NandiRMSNorm(self.head_dim, eps=config.rms_norm_eps) def forward( self, hidden_states: torch.Tensor, position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] hidden_shape = (*input_shape, -1, self.head_dim) query_states = self.q_proj(hidden_states).view(hidden_shape) key_states = self.k_proj(hidden_states).view(hidden_shape) if self.qk_norm: # per head over head_dim, before RoPE (as in training) query_states = self.q_norm(query_states) key_states = self.k_norm(key_states) query_states = query_states.transpose(1, 2) key_states = key_states.transpose(1, 2) value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) cos, sin = position_embeddings query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) attention_interface: Callable = eager_attention_forward if self.config._attn_implementation != "eager": attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] attn_output, attn_weights = attention_interface( self, query_states, key_states, value_states, attention_mask, dropout=0.0 if not self.training else self.attention_dropout, scaling=self.scaling, is_causal=False, **kwargs, ) attn_output = attn_output.reshape(*input_shape, -1).contiguous() return self.o_proj(attn_output), attn_weights class NandiMLP(nn.Module): def __init__(self, config): super().__init__() self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=config.mlp_bias) self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=config.mlp_bias) self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=config.mlp_bias) self.act_fn = ACT2FN[config.hidden_act] def forward(self, x): return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) class NandiEncoderLayer(GradientCheckpointingLayer): def __init__(self, config: NandiEncoderConfig, layer_idx: int): super().__init__() self.hidden_size = config.hidden_size self.self_attn = NandiEncoderAttention(config=config, layer_idx=layer_idx) self.mlp = NandiMLP(config) self.input_layernorm = NandiRMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = NandiRMSNorm(config.hidden_size, eps=config.rms_norm_eps) def forward( self, hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: residual = hidden_states hidden_states, _ = self.self_attn( hidden_states=self.input_layernorm(hidden_states), attention_mask=attention_mask, position_embeddings=position_embeddings, **kwargs, ) hidden_states = residual + hidden_states residual = hidden_states hidden_states = residual + self.mlp(self.post_attention_layernorm(hidden_states)) return hidden_states class NandiEncoderPreTrainedModel(PreTrainedModel): config_class = NandiEncoderConfig config: NandiEncoderConfig base_model_prefix = "model" supports_gradient_checkpointing = True _no_split_modules = ["NandiEncoderLayer"] _supports_flash_attn = False # untested with bidirectional masks; use "sdpa" (default) or "eager" _supports_sdpa = True _supports_flex_attn = False _supports_attention_backend = False class NandiEncoderModel(NandiEncoderPreTrainedModel): """Bare encoder. `last_hidden_state` is after the final RMSNorm (what the MLM head and classifiers see).""" def __init__(self, config: NandiEncoderConfig): super().__init__(config) self.vocab_size = config.vocab_size embedding_dim = config.embedding_rank if config.factorized_embedding else config.hidden_size # No padding_idx, as in training (the pad row gets gradients like any other token) self.embed_tokens = nn.Embedding(config.vocab_size, embedding_dim) self.embedding_proj = ( nn.Linear(config.embedding_rank, config.hidden_size, bias=False) if config.factorized_embedding else None ) self.layers = nn.ModuleList([NandiEncoderLayer(config, i) for i in range(config.num_hidden_layers)]) self.norm = NandiRMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.rotary_emb = NandiRotaryEmbedding(config=config) self.gradient_checkpointing = False self.post_init() def get_input_embeddings(self): return self.embed_tokens def set_input_embeddings(self, value): self.embed_tokens = value def forward( self, input_ids: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, inputs_embeds: torch.FloatTensor | None = None, output_hidden_states: bool | None = None, return_dict: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutput: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) if self.embedding_proj is not None: inputs_embeds = self.embedding_proj(inputs_embeds) if position_ids is None: # RoPE is relative, so any per-row offset (e.g. left padding) gives the same attention scores position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) bidirectional_mask = create_bidirectional_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask ) hidden_states = inputs_embeds position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) all_hidden_states = (hidden_states,) if output_hidden_states else None # Shared layers run back to back (layer 0, layer 0, layer 1, layer 1, ...), exactly as in training repeats = self.config.layer_sharing_repeats if self.config.layer_sharing else 1 for layer in self.layers[: self.config.num_hidden_layers]: for _ in range(repeats): hidden_states = layer( hidden_states, attention_mask=bidirectional_mask, position_embeddings=position_embeddings, **kwargs, ) if output_hidden_states: all_hidden_states += (hidden_states,) hidden_states = self.norm(hidden_states) return BaseModelOutput(last_hidden_state=hidden_states, hidden_states=all_hidden_states) class NandiPredictionHead(nn.Module): """`mlm_head: dense`: dense -> activation -> RMSNorm between the encoder's final norm and the decoder.""" def __init__(self, config): super().__init__() self.dense = nn.Linear(config.hidden_size, config.hidden_size, bias=False) self.act = ACT2FN[config.mlm_head_act] self.norm = NandiRMSNorm(config.hidden_size, eps=config.rms_norm_eps) def forward(self, hidden_states): return self.norm(self.act(self.dense(hidden_states))) class NandiForMaskedLM(NandiEncoderPreTrainedModel): """Masked LM head = the pretrained LM head, after the encoder's final norm. `mlm_head="dense"` (NandiBERT-150M): dense -> act -> RMSNorm -> untied full-width lm_head. `mlm_head="tied_factorized"`: lm_head_proj (hidden -> embedding_rank) -> lm_head tied to embed_tokens. """ _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} def __init__(self, config: NandiEncoderConfig): super().__init__(config) self.model = NandiEncoderModel(config) self.vocab_size = config.vocab_size if getattr(config, "mlm_head", "tied_factorized") == "dense": # dense -> act -> RMSNorm, then an untied full-width decoder self.head = NandiPredictionHead(config) self.lm_head_proj = None self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=config.mlm_head_decoder_bias) else: self.head = None lm_head_in = config.embedding_rank if config.factorized_embedding else config.hidden_size self.lm_head_proj = ( nn.Linear(config.hidden_size, config.embedding_rank, bias=False) if config.factorized_embedding else None ) self.lm_head = nn.Linear(lm_head_in, config.vocab_size, bias=False) self.post_init() def get_output_embeddings(self): return self.lm_head def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings def forward( self, input_ids: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, output_hidden_states: bool | None = None, return_dict: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MaskedLMOutput: outputs = self.model( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, inputs_embeds=inputs_embeds, output_hidden_states=output_hidden_states, **kwargs, ) hidden_states = outputs.last_hidden_state if self.head is not None: hidden_states = self.head(hidden_states) if self.lm_head_proj is not None: hidden_states = self.lm_head_proj(hidden_states) logits = self.lm_head(hidden_states) loss = None if labels is not None: # -100 = not predicted loss = nn.functional.cross_entropy( logits.float().view(-1, self.vocab_size), labels.to(logits.device).view(-1), ignore_index=-100 ) return MaskedLMOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states) def _pool(hidden_states: torch.Tensor, attention_mask: torch.Tensor | None, pooling: str) -> torch.Tensor: if pooling == "first": # the <|endoftext|> every input starts with return hidden_states[:, 0] if attention_mask is None: return hidden_states.mean(dim=1) mask = attention_mask.to(hidden_states.dtype).unsqueeze(-1) return (hidden_states * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1.0) class NandiForSequenceClassification(NandiEncoderPreTrainedModel): """Pooled encoder output (`config.classifier_pooling`) -> [pretrained MLM head] -> dropout -> linear. With `config.classifier_head = "mlm"` the pooled vector goes through the pretrained MLM head (dense -> act -> RMSNorm, loaded from `head.*`) before the linear layer. Only the linear layer is newly initialized. """ def __init__(self, config: NandiEncoderConfig): super().__init__(config) self.num_labels = config.num_labels self.model = NandiEncoderModel(config) self.dropout = nn.Dropout(config.classifier_dropout) self.classifier = nn.Linear(config.hidden_size, config.num_labels) # classifier_head "mlm": the pretrained MLM head (dense, act, RMSNorm; loads from head.*) sits before the linear self.head = NandiPredictionHead(config) if getattr(config, "classifier_head", None) == "mlm" else None self.post_init() def forward( self, input_ids: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, output_hidden_states: bool | None = None, return_dict: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> SequenceClassifierOutput: outputs = self.model( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, inputs_embeds=inputs_embeds, output_hidden_states=output_hidden_states, **kwargs, ) pooled = _pool(outputs.last_hidden_state, attention_mask, self.config.classifier_pooling) if self.head is not None: pooled = self.head(pooled) logits = self.classifier(self.dropout(pooled)) loss = None if labels is not None: labels = labels.to(logits.device) if self.config.problem_type is None: if self.num_labels == 1: self.config.problem_type = "regression" elif labels.dtype in (torch.long, torch.int): self.config.problem_type = "single_label_classification" else: self.config.problem_type = "multi_label_classification" if self.config.problem_type == "regression": loss = MSELoss()(logits.squeeze(), labels.squeeze().to(logits.dtype)) if self.num_labels == 1 else MSELoss()(logits, labels) elif self.config.problem_type == "single_label_classification": loss = CrossEntropyLoss()(logits.view(-1, self.num_labels), labels.view(-1)) else: loss = BCEWithLogitsLoss()(logits, labels.to(logits.dtype)) return SequenceClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states) class NandiForTokenClassification(NandiEncoderPreTrainedModel): """Per-token encoder output -> [pretrained MLM head] -> dropout -> linear. With `config.classifier_head = "mlm"` every token goes through the pretrained MLM head (dense -> act -> RMSNorm, loaded from `head.*`) before the linear layer. Only the linear layer is newly initialized. """ def __init__(self, config: NandiEncoderConfig): super().__init__(config) self.num_labels = config.num_labels self.model = NandiEncoderModel(config) self.dropout = nn.Dropout(config.classifier_dropout) self.classifier = nn.Linear(config.hidden_size, config.num_labels) # classifier_head "mlm": the pretrained MLM head (dense, act, RMSNorm; loads from head.*) sits before the linear self.head = NandiPredictionHead(config) if getattr(config, "classifier_head", None) == "mlm" else None self.post_init() def forward( self, input_ids: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, output_hidden_states: bool | None = None, return_dict: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> TokenClassifierOutput: outputs = self.model( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, inputs_embeds=inputs_embeds, output_hidden_states=output_hidden_states, **kwargs, ) h = outputs.last_hidden_state if self.head is None else self.head(outputs.last_hidden_state) logits = self.classifier(self.dropout(h)) loss = None if labels is not None: loss = CrossEntropyLoss()(logits.view(-1, self.num_labels), labels.to(logits.device).view(-1)) return TokenClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states) __all__ = [ "NandiEncoderPreTrainedModel", "NandiEncoderModel", "NandiForMaskedLM", "NandiForSequenceClassification", "NandiForTokenClassification", ]