NandiBERT-150M / modeling_nandi_encoder.py
akanyaani's picture
Remove internal code reference from the modeling docstring; 450M parameter count 445M
3d28371 verified
Raw History Blame Contribute Delete
24.5 kB
# Copyright 2026 FrontiersMind. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Nandi encoder: the Nandi decoder architecture with bidirectional attention, trained with masked language modeling.
Forward pass:
embed_tokens -> embedding_proj (factorized)
-> each of the N layers applied `layer_sharing_repeats` times back to back (0, 0, 1, 1, ...)
(pre-norm: x + attn(rmsnorm(x)), x + swiglu(rmsnorm(x)); GQA attention with RoPE, NOT causal)
-> final RMSNorm -> MLM head: either lm_head_proj (factorized) -> lm_head (tied to embed_tokens),
or (mlm_head='dense') dense -> act -> RMSNorm -> untied lm_head. Optional QK-norm (per head, before RoPE).
Sequence / token classifiers: with classifier_head='mlm' the pretrained dense MLM head (dense -> act -> RMSNorm) sits
between the encoder and the new linear layer (sequence: pool -> head -> linear; token: head per token -> linear).
Inputs start with `<|endoftext|>` (the tokenizer adds it), as every training document did.
"""
from collections.abc import Callable
import torch
import torch.nn as nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
from transformers.activations import ACT2FN
from transformers.masking_utils import create_bidirectional_mask
from transformers.modeling_layers import GradientCheckpointingLayer
from transformers.modeling_outputs import (
BaseModelOutput,
MaskedLMOutput,
SequenceClassifierOutput,
TokenClassifierOutput,
)
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
from transformers.processing_utils import Unpack
from transformers.utils import TransformersKwargs
from .configuration_nandi_encoder import NandiEncoderConfig
class NandiRMSNorm(nn.Module):
def __init__(self, hidden_size, eps=1e-5):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, hidden_states):
# fp32 normalize AND weight multiply, then cast: matches flash-attn's Triton RMSNorm used in training
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
return (self.weight.float() * hidden_states).to(input_dtype)
def extra_repr(self):
return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
class NandiRotaryEmbedding(nn.Module):
inv_freq: torch.Tensor
def __init__(self, config: NandiEncoderConfig, device=None):
super().__init__()
self.max_seq_len_cached = config.max_position_embeddings
self.original_max_seq_len = config.max_position_embeddings
self.config = config
self.rope_type = self.config.rope_parameters.get("rope_type", "default")
rope_init_fn: Callable = self.compute_default_rope_parameters
if self.rope_type != "default":
rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
inv_freq, self.attention_scaling = rope_init_fn(self.config, device)
self.register_buffer("inv_freq", inv_freq, persistent=False)
self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False)
@staticmethod
def compute_default_rope_parameters(config=None, device=None, seq_len=None):
del seq_len
base = config.rope_parameters["rope_theta"]
dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim))
return inv_freq, 1.0
@torch.no_grad()
@dynamic_rope_update
def forward(self, x, position_ids):
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
position_ids_expanded = position_ids[:, None, :].float()
device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
with torch.autocast(device_type=device_type, enabled=False):
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
emb = torch.cat((freqs, freqs), dim=-1)
cos = emb.cos() * self.attention_scaling
sin = emb.sin() * self.attention_scaling
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
def rotate_half(x):
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2 :]
return torch.cat((-x2, x1), dim=-1)
def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
cos = cos.unsqueeze(unsqueeze_dim)
sin = sin.unsqueeze(unsqueeze_dim)
return (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin)
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
batch, num_key_value_heads, slen, head_dim = hidden_states.shape
if n_rep == 1:
return hidden_states
hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
def eager_attention_forward(
module: nn.Module,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attention_mask: torch.Tensor | None,
scaling: float,
dropout: float = 0.0,
**kwargs,
):
del kwargs
key_states = repeat_kv(key, module.num_key_value_groups)
value_states = repeat_kv(value, module.num_key_value_groups)
attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
if attention_mask is not None:
attn_weights = attn_weights + attention_mask[:, :, :, : key_states.shape[-2]]
attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
attn_output = torch.matmul(attn_weights, value_states)
return attn_output.transpose(1, 2).contiguous(), attn_weights
class NandiEncoderAttention(nn.Module):
"""GQA self-attention with RoPE. Bidirectional: every token attends to every non-padding token."""
def __init__(self, config: NandiEncoderConfig, layer_idx: int):
super().__init__()
self.config = config
self.layer_idx = layer_idx
self.head_dim = config.head_dim
self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
self.scaling = self.head_dim**-0.5
self.attention_dropout = config.attention_dropout
self.is_causal = False
self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias)
self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias)
self.v_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias)
self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias)
self.qk_norm = getattr(config, "qk_norm", False)
if self.qk_norm:
self.q_norm = NandiRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = NandiRMSNorm(self.head_dim, eps=config.rms_norm_eps)
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
attention_mask: torch.Tensor | None,
**kwargs: Unpack[TransformersKwargs],
) -> tuple[torch.Tensor, torch.Tensor]:
input_shape = hidden_states.shape[:-1]
hidden_shape = (*input_shape, -1, self.head_dim)
query_states = self.q_proj(hidden_states).view(hidden_shape)
key_states = self.k_proj(hidden_states).view(hidden_shape)
if self.qk_norm: # per head over head_dim, before RoPE (as in training)
query_states = self.q_norm(query_states)
key_states = self.k_norm(key_states)
query_states = query_states.transpose(1, 2)
key_states = key_states.transpose(1, 2)
value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
cos, sin = position_embeddings
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
attention_interface: Callable = eager_attention_forward
if self.config._attn_implementation != "eager":
attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
attn_output, attn_weights = attention_interface(
self,
query_states,
key_states,
value_states,
attention_mask,
dropout=0.0 if not self.training else self.attention_dropout,
scaling=self.scaling,
is_causal=False,
**kwargs,
)
attn_output = attn_output.reshape(*input_shape, -1).contiguous()
return self.o_proj(attn_output), attn_weights
class NandiMLP(nn.Module):
def __init__(self, config):
super().__init__()
self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=config.mlp_bias)
self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=config.mlp_bias)
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=config.mlp_bias)
self.act_fn = ACT2FN[config.hidden_act]
def forward(self, x):
return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
class NandiEncoderLayer(GradientCheckpointingLayer):
def __init__(self, config: NandiEncoderConfig, layer_idx: int):
super().__init__()
self.hidden_size = config.hidden_size
self.self_attn = NandiEncoderAttention(config=config, layer_idx=layer_idx)
self.mlp = NandiMLP(config)
self.input_layernorm = NandiRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = NandiRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor | None = None,
position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
**kwargs: Unpack[TransformersKwargs],
) -> torch.Tensor:
residual = hidden_states
hidden_states, _ = self.self_attn(
hidden_states=self.input_layernorm(hidden_states),
attention_mask=attention_mask,
position_embeddings=position_embeddings,
**kwargs,
)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = residual + self.mlp(self.post_attention_layernorm(hidden_states))
return hidden_states
class NandiEncoderPreTrainedModel(PreTrainedModel):
config_class = NandiEncoderConfig
config: NandiEncoderConfig
base_model_prefix = "model"
supports_gradient_checkpointing = True
_no_split_modules = ["NandiEncoderLayer"]
_supports_flash_attn = False # untested with bidirectional masks; use "sdpa" (default) or "eager"
_supports_sdpa = True
_supports_flex_attn = False
_supports_attention_backend = False
class NandiEncoderModel(NandiEncoderPreTrainedModel):
"""Bare encoder. `last_hidden_state` is after the final RMSNorm (what the MLM head and classifiers see)."""
def __init__(self, config: NandiEncoderConfig):
super().__init__(config)
self.vocab_size = config.vocab_size
embedding_dim = config.embedding_rank if config.factorized_embedding else config.hidden_size
# No padding_idx, as in training (the pad row gets gradients like any other token)
self.embed_tokens = nn.Embedding(config.vocab_size, embedding_dim)
self.embedding_proj = (
nn.Linear(config.embedding_rank, config.hidden_size, bias=False) if config.factorized_embedding else None
)
self.layers = nn.ModuleList([NandiEncoderLayer(config, i) for i in range(config.num_hidden_layers)])
self.norm = NandiRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.rotary_emb = NandiRotaryEmbedding(config=config)
self.gradient_checkpointing = False
self.post_init()
def get_input_embeddings(self):
return self.embed_tokens
def set_input_embeddings(self, value):
self.embed_tokens = value
def forward(
self,
input_ids: torch.LongTensor | None = None,
attention_mask: torch.Tensor | None = None,
position_ids: torch.LongTensor | None = None,
inputs_embeds: torch.FloatTensor | None = None,
output_hidden_states: bool | None = None,
return_dict: bool | None = None,
**kwargs: Unpack[TransformersKwargs],
) -> BaseModelOutput:
if (input_ids is None) ^ (inputs_embeds is not None):
raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
output_hidden_states = (
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
)
if inputs_embeds is None:
inputs_embeds = self.embed_tokens(input_ids)
if self.embedding_proj is not None:
inputs_embeds = self.embedding_proj(inputs_embeds)
if position_ids is None:
# RoPE is relative, so any per-row offset (e.g. left padding) gives the same attention scores
position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0)
bidirectional_mask = create_bidirectional_mask(
config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask
)
hidden_states = inputs_embeds
position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids)
all_hidden_states = (hidden_states,) if output_hidden_states else None
# Shared layers run back to back (layer 0, layer 0, layer 1, layer 1, ...), exactly as in training
repeats = self.config.layer_sharing_repeats if self.config.layer_sharing else 1
for layer in self.layers[: self.config.num_hidden_layers]:
for _ in range(repeats):
hidden_states = layer(
hidden_states,
attention_mask=bidirectional_mask,
position_embeddings=position_embeddings,
**kwargs,
)
if output_hidden_states:
all_hidden_states += (hidden_states,)
hidden_states = self.norm(hidden_states)
return BaseModelOutput(last_hidden_state=hidden_states, hidden_states=all_hidden_states)
class NandiPredictionHead(nn.Module):
"""`mlm_head: dense`: dense -> activation -> RMSNorm between the encoder's final norm and the decoder."""
def __init__(self, config):
super().__init__()
self.dense = nn.Linear(config.hidden_size, config.hidden_size, bias=False)
self.act = ACT2FN[config.mlm_head_act]
self.norm = NandiRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(self, hidden_states):
return self.norm(self.act(self.dense(hidden_states)))
class NandiForMaskedLM(NandiEncoderPreTrainedModel):
"""Masked LM head = the pretrained LM head, after the encoder's final norm.
`mlm_head="dense"` (NandiBERT-150M): dense -> act -> RMSNorm -> untied full-width lm_head.
`mlm_head="tied_factorized"`: lm_head_proj (hidden -> embedding_rank) -> lm_head tied to embed_tokens.
"""
_tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
def __init__(self, config: NandiEncoderConfig):
super().__init__(config)
self.model = NandiEncoderModel(config)
self.vocab_size = config.vocab_size
if getattr(config, "mlm_head", "tied_factorized") == "dense":
# dense -> act -> RMSNorm, then an untied full-width decoder
self.head = NandiPredictionHead(config)
self.lm_head_proj = None
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=config.mlm_head_decoder_bias)
else:
self.head = None
lm_head_in = config.embedding_rank if config.factorized_embedding else config.hidden_size
self.lm_head_proj = (
nn.Linear(config.hidden_size, config.embedding_rank, bias=False) if config.factorized_embedding else None
)
self.lm_head = nn.Linear(lm_head_in, config.vocab_size, bias=False)
self.post_init()
def get_output_embeddings(self):
return self.lm_head
def set_output_embeddings(self, new_embeddings):
self.lm_head = new_embeddings
def forward(
self,
input_ids: torch.LongTensor | None = None,
attention_mask: torch.Tensor | None = None,
position_ids: torch.LongTensor | None = None,
inputs_embeds: torch.FloatTensor | None = None,
labels: torch.LongTensor | None = None,
output_hidden_states: bool | None = None,
return_dict: bool | None = None,
**kwargs: Unpack[TransformersKwargs],
) -> MaskedLMOutput:
outputs = self.model(
input_ids=input_ids,
attention_mask=attention_mask,
position_ids=position_ids,
inputs_embeds=inputs_embeds,
output_hidden_states=output_hidden_states,
**kwargs,
)
hidden_states = outputs.last_hidden_state
if self.head is not None:
hidden_states = self.head(hidden_states)
if self.lm_head_proj is not None:
hidden_states = self.lm_head_proj(hidden_states)
logits = self.lm_head(hidden_states)
loss = None
if labels is not None: # -100 = not predicted
loss = nn.functional.cross_entropy(
logits.float().view(-1, self.vocab_size), labels.to(logits.device).view(-1), ignore_index=-100
)
return MaskedLMOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states)
def _pool(hidden_states: torch.Tensor, attention_mask: torch.Tensor | None, pooling: str) -> torch.Tensor:
if pooling == "first": # the <|endoftext|> every input starts with
return hidden_states[:, 0]
if attention_mask is None:
return hidden_states.mean(dim=1)
mask = attention_mask.to(hidden_states.dtype).unsqueeze(-1)
return (hidden_states * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1.0)
class NandiForSequenceClassification(NandiEncoderPreTrainedModel):
"""Pooled encoder output (`config.classifier_pooling`) -> [pretrained MLM head] -> dropout -> linear.
With `config.classifier_head = "mlm"` the pooled vector goes through the pretrained MLM head (dense -> act -> RMSNorm,
loaded from `head.*`) before the linear layer. Only the linear layer is newly initialized.
"""
def __init__(self, config: NandiEncoderConfig):
super().__init__(config)
self.num_labels = config.num_labels
self.model = NandiEncoderModel(config)
self.dropout = nn.Dropout(config.classifier_dropout)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
# classifier_head "mlm": the pretrained MLM head (dense, act, RMSNorm; loads from head.*) sits before the linear
self.head = NandiPredictionHead(config) if getattr(config, "classifier_head", None) == "mlm" else None
self.post_init()
def forward(
self,
input_ids: torch.LongTensor | None = None,
attention_mask: torch.Tensor | None = None,
position_ids: torch.LongTensor | None = None,
inputs_embeds: torch.FloatTensor | None = None,
labels: torch.LongTensor | None = None,
output_hidden_states: bool | None = None,
return_dict: bool | None = None,
**kwargs: Unpack[TransformersKwargs],
) -> SequenceClassifierOutput:
outputs = self.model(
input_ids=input_ids,
attention_mask=attention_mask,
position_ids=position_ids,
inputs_embeds=inputs_embeds,
output_hidden_states=output_hidden_states,
**kwargs,
)
pooled = _pool(outputs.last_hidden_state, attention_mask, self.config.classifier_pooling)
if self.head is not None:
pooled = self.head(pooled)
logits = self.classifier(self.dropout(pooled))
loss = None
if labels is not None:
labels = labels.to(logits.device)
if self.config.problem_type is None:
if self.num_labels == 1:
self.config.problem_type = "regression"
elif labels.dtype in (torch.long, torch.int):
self.config.problem_type = "single_label_classification"
else:
self.config.problem_type = "multi_label_classification"
if self.config.problem_type == "regression":
loss = MSELoss()(logits.squeeze(), labels.squeeze().to(logits.dtype)) if self.num_labels == 1 else MSELoss()(logits, labels)
elif self.config.problem_type == "single_label_classification":
loss = CrossEntropyLoss()(logits.view(-1, self.num_labels), labels.view(-1))
else:
loss = BCEWithLogitsLoss()(logits, labels.to(logits.dtype))
return SequenceClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states)
class NandiForTokenClassification(NandiEncoderPreTrainedModel):
"""Per-token encoder output -> [pretrained MLM head] -> dropout -> linear.
With `config.classifier_head = "mlm"` every token goes through the pretrained MLM head (dense -> act -> RMSNorm,
loaded from `head.*`) before the linear layer. Only the linear layer is newly initialized.
"""
def __init__(self, config: NandiEncoderConfig):
super().__init__(config)
self.num_labels = config.num_labels
self.model = NandiEncoderModel(config)
self.dropout = nn.Dropout(config.classifier_dropout)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
# classifier_head "mlm": the pretrained MLM head (dense, act, RMSNorm; loads from head.*) sits before the linear
self.head = NandiPredictionHead(config) if getattr(config, "classifier_head", None) == "mlm" else None
self.post_init()
def forward(
self,
input_ids: torch.LongTensor | None = None,
attention_mask: torch.Tensor | None = None,
position_ids: torch.LongTensor | None = None,
inputs_embeds: torch.FloatTensor | None = None,
labels: torch.LongTensor | None = None,
output_hidden_states: bool | None = None,
return_dict: bool | None = None,
**kwargs: Unpack[TransformersKwargs],
) -> TokenClassifierOutput:
outputs = self.model(
input_ids=input_ids,
attention_mask=attention_mask,
position_ids=position_ids,
inputs_embeds=inputs_embeds,
output_hidden_states=output_hidden_states,
**kwargs,
)
h = outputs.last_hidden_state if self.head is None else self.head(outputs.last_hidden_state)
logits = self.classifier(self.dropout(h))
loss = None
if labels is not None:
loss = CrossEntropyLoss()(logits.view(-1, self.num_labels), labels.to(logits.device).view(-1))
return TokenClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states)
__all__ = [
"NandiEncoderPreTrainedModel",
"NandiEncoderModel",
"NandiForMaskedLM",
"NandiForSequenceClassification",
"NandiForTokenClassification",
]