Fill-Mask
Transformers
Safetensors
nandi_encoder
encoder
masked-language-modeling
indic
multilingual
code
long-context
frontiersmind
nandi
custom_code
Instructions to use FrontiersMind/NandiBERT-450M with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use FrontiersMind/NandiBERT-450M with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("fill-mask", model="FrontiersMind/NandiBERT-450M", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForMaskedLM model = AutoModelForMaskedLM.from_pretrained("FrontiersMind/NandiBERT-450M", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download modeling_nandi_encoder.py from FrontiersMind/NandiBERT-450M: direct link, hf CLI and curl.
- Browser
- Download file 24.3 kB
-
https://huggingface.co/FrontiersMind/NandiBERT-450M/resolve/main/modeling_nandi_encoder.py
- Command line
-
hf download hf://FrontiersMind/NandiBERT-450M/modeling_nandi_encoder.py
-
curl -L -o modeling_nandi_encoder.py https://huggingface.co/FrontiersMind/NandiBERT-450M/resolve/main/modeling_nandi_encoder.py
24.3 kB
| # Copyright 2026 FrontiersMind. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| """Nandi encoder: the Nandi decoder architecture with bidirectional attention, trained with masked language modeling. | |
| Forward pass: | |
| embed_tokens -> embedding_proj (factorized) | |
| -> each of the N layers applied `layer_sharing_repeats` times back to back (0, 0, 1, 1, ...) | |
| (pre-norm: x + attn(rmsnorm(x)), x + swiglu(rmsnorm(x)); GQA attention with RoPE, NOT causal) | |
| -> final RMSNorm -> MLM head: either lm_head_proj (factorized) -> lm_head (tied to embed_tokens), | |
| or (mlm_head='dense') dense -> act -> RMSNorm -> untied lm_head. Optional QK-norm (per head, before RoPE). | |
| Sequence / token classifiers: with classifier_head='mlm' the pretrained dense MLM head (dense -> act -> RMSNorm) sits | |
| between the encoder and the new linear layer (sequence: pool -> head -> linear; token: head per token -> linear). | |
| Inputs start with `<|endoftext|>` (the tokenizer adds it), as every training document did. | |
| """ | |
| from collections.abc import Callable | |
| import torch | |
| import torch.nn as nn | |
| from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss | |
| from transformers.activations import ACT2FN | |
| from transformers.masking_utils import create_bidirectional_mask | |
| from transformers.modeling_layers import GradientCheckpointingLayer | |
| from transformers.modeling_outputs import ( | |
| BaseModelOutput, | |
| MaskedLMOutput, | |
| SequenceClassifierOutput, | |
| TokenClassifierOutput, | |
| ) | |
| from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update | |
| from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel | |
| from transformers.processing_utils import Unpack | |
| from transformers.utils import TransformersKwargs | |
| from .configuration_nandi_encoder import NandiEncoderConfig | |
| class NandiRMSNorm(nn.Module): | |
| def __init__(self, hidden_size, eps=1e-5): | |
| super().__init__() | |
| self.weight = nn.Parameter(torch.ones(hidden_size)) | |
| self.variance_epsilon = eps | |
| def forward(self, hidden_states): | |
| # fp32 normalize AND weight multiply, then cast: matches flash-attn's Triton RMSNorm used in training | |
| input_dtype = hidden_states.dtype | |
| hidden_states = hidden_states.to(torch.float32) | |
| variance = hidden_states.pow(2).mean(-1, keepdim=True) | |
| hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) | |
| return (self.weight.float() * hidden_states).to(input_dtype) | |
| def extra_repr(self): | |
| return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" | |
| class NandiRotaryEmbedding(nn.Module): | |
| inv_freq: torch.Tensor | |
| def __init__(self, config: NandiEncoderConfig, device=None): | |
| super().__init__() | |
| self.max_seq_len_cached = config.max_position_embeddings | |
| self.original_max_seq_len = config.max_position_embeddings | |
| self.config = config | |
| self.rope_type = self.config.rope_parameters.get("rope_type", "default") | |
| rope_init_fn: Callable = self.compute_default_rope_parameters | |
| if self.rope_type != "default": | |
| rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] | |
| inv_freq, self.attention_scaling = rope_init_fn(self.config, device) | |
| self.register_buffer("inv_freq", inv_freq, persistent=False) | |
| self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False) | |
| def compute_default_rope_parameters(config=None, device=None, seq_len=None): | |
| del seq_len | |
| base = config.rope_parameters["rope_theta"] | |
| dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads | |
| inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim)) | |
| return inv_freq, 1.0 | |
| def forward(self, x, position_ids): | |
| inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device) | |
| position_ids_expanded = position_ids[:, None, :].float() | |
| device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" | |
| with torch.autocast(device_type=device_type, enabled=False): | |
| freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) | |
| emb = torch.cat((freqs, freqs), dim=-1) | |
| cos = emb.cos() * self.attention_scaling | |
| sin = emb.sin() * self.attention_scaling | |
| return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) | |
| def rotate_half(x): | |
| x1 = x[..., : x.shape[-1] // 2] | |
| x2 = x[..., x.shape[-1] // 2 :] | |
| return torch.cat((-x2, x1), dim=-1) | |
| def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): | |
| cos = cos.unsqueeze(unsqueeze_dim) | |
| sin = sin.unsqueeze(unsqueeze_dim) | |
| return (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin) | |
| def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: | |
| batch, num_key_value_heads, slen, head_dim = hidden_states.shape | |
| if n_rep == 1: | |
| return hidden_states | |
| hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) | |
| return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) | |
| def eager_attention_forward( | |
| module: nn.Module, | |
| query: torch.Tensor, | |
| key: torch.Tensor, | |
| value: torch.Tensor, | |
| attention_mask: torch.Tensor | None, | |
| scaling: float, | |
| dropout: float = 0.0, | |
| **kwargs, | |
| ): | |
| del kwargs | |
| key_states = repeat_kv(key, module.num_key_value_groups) | |
| value_states = repeat_kv(value, module.num_key_value_groups) | |
| attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling | |
| if attention_mask is not None: | |
| attn_weights = attn_weights + attention_mask[:, :, :, : key_states.shape[-2]] | |
| attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) | |
| attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) | |
| attn_output = torch.matmul(attn_weights, value_states) | |
| return attn_output.transpose(1, 2).contiguous(), attn_weights | |
| class NandiEncoderAttention(nn.Module): | |
| """GQA self-attention with RoPE. Bidirectional: every token attends to every non-padding token.""" | |
| def __init__(self, config: NandiEncoderConfig, layer_idx: int): | |
| super().__init__() | |
| self.config = config | |
| self.layer_idx = layer_idx | |
| self.head_dim = config.head_dim | |
| self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads | |
| self.scaling = self.head_dim**-0.5 | |
| self.attention_dropout = config.attention_dropout | |
| self.is_causal = False | |
| self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias) | |
| self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias) | |
| self.v_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias) | |
| self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias) | |
| self.qk_norm = getattr(config, "qk_norm", False) | |
| if self.qk_norm: | |
| self.q_norm = NandiRMSNorm(self.head_dim, eps=config.rms_norm_eps) | |
| self.k_norm = NandiRMSNorm(self.head_dim, eps=config.rms_norm_eps) | |
| def forward( | |
| self, | |
| hidden_states: torch.Tensor, | |
| position_embeddings: tuple[torch.Tensor, torch.Tensor], | |
| attention_mask: torch.Tensor | None, | |
| **kwargs: Unpack[TransformersKwargs], | |
| ) -> tuple[torch.Tensor, torch.Tensor]: | |
| input_shape = hidden_states.shape[:-1] | |
| hidden_shape = (*input_shape, -1, self.head_dim) | |
| query_states = self.q_proj(hidden_states).view(hidden_shape) | |
| key_states = self.k_proj(hidden_states).view(hidden_shape) | |
| if self.qk_norm: # per head over head_dim, before RoPE (as in training) | |
| query_states = self.q_norm(query_states) | |
| key_states = self.k_norm(key_states) | |
| query_states = query_states.transpose(1, 2) | |
| key_states = key_states.transpose(1, 2) | |
| value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) | |
| cos, sin = position_embeddings | |
| query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) | |
| attention_interface: Callable = eager_attention_forward | |
| if self.config._attn_implementation != "eager": | |
| attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] | |
| attn_output, attn_weights = attention_interface( | |
| self, | |
| query_states, | |
| key_states, | |
| value_states, | |
| attention_mask, | |
| dropout=0.0 if not self.training else self.attention_dropout, | |
| scaling=self.scaling, | |
| is_causal=False, | |
| **kwargs, | |
| ) | |
| attn_output = attn_output.reshape(*input_shape, -1).contiguous() | |
| return self.o_proj(attn_output), attn_weights | |
| class NandiMLP(nn.Module): | |
| def __init__(self, config): | |
| super().__init__() | |
| self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=config.mlp_bias) | |
| self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=config.mlp_bias) | |
| self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=config.mlp_bias) | |
| self.act_fn = ACT2FN[config.hidden_act] | |
| def forward(self, x): | |
| return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) | |
| class NandiEncoderLayer(GradientCheckpointingLayer): | |
| def __init__(self, config: NandiEncoderConfig, layer_idx: int): | |
| super().__init__() | |
| self.hidden_size = config.hidden_size | |
| self.self_attn = NandiEncoderAttention(config=config, layer_idx=layer_idx) | |
| self.mlp = NandiMLP(config) | |
| self.input_layernorm = NandiRMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| self.post_attention_layernorm = NandiRMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| def forward( | |
| self, | |
| hidden_states: torch.Tensor, | |
| attention_mask: torch.Tensor | None = None, | |
| position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, | |
| **kwargs: Unpack[TransformersKwargs], | |
| ) -> torch.Tensor: | |
| residual = hidden_states | |
| hidden_states, _ = self.self_attn( | |
| hidden_states=self.input_layernorm(hidden_states), | |
| attention_mask=attention_mask, | |
| position_embeddings=position_embeddings, | |
| **kwargs, | |
| ) | |
| hidden_states = residual + hidden_states | |
| residual = hidden_states | |
| hidden_states = residual + self.mlp(self.post_attention_layernorm(hidden_states)) | |
| return hidden_states | |
| class NandiEncoderPreTrainedModel(PreTrainedModel): | |
| config_class = NandiEncoderConfig | |
| config: NandiEncoderConfig | |
| base_model_prefix = "model" | |
| supports_gradient_checkpointing = True | |
| _no_split_modules = ["NandiEncoderLayer"] | |
| _supports_flash_attn = False # untested with bidirectional masks; use "sdpa" (default) or "eager" | |
| _supports_sdpa = True | |
| _supports_flex_attn = False | |
| _supports_attention_backend = False | |
| class NandiEncoderModel(NandiEncoderPreTrainedModel): | |
| """Bare encoder. `last_hidden_state` is after the final RMSNorm (what the MLM head and classifiers see).""" | |
| def __init__(self, config: NandiEncoderConfig): | |
| super().__init__(config) | |
| self.vocab_size = config.vocab_size | |
| embedding_dim = config.embedding_rank if config.factorized_embedding else config.hidden_size | |
| # No padding_idx, as in training (the pad row gets gradients like any other token) | |
| self.embed_tokens = nn.Embedding(config.vocab_size, embedding_dim) | |
| self.embedding_proj = ( | |
| nn.Linear(config.embedding_rank, config.hidden_size, bias=False) if config.factorized_embedding else None | |
| ) | |
| self.layers = nn.ModuleList([NandiEncoderLayer(config, i) for i in range(config.num_hidden_layers)]) | |
| self.norm = NandiRMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| self.rotary_emb = NandiRotaryEmbedding(config=config) | |
| self.gradient_checkpointing = False | |
| self.post_init() | |
| def get_input_embeddings(self): | |
| return self.embed_tokens | |
| def set_input_embeddings(self, value): | |
| self.embed_tokens = value | |
| def forward( | |
| self, | |
| input_ids: torch.LongTensor | None = None, | |
| attention_mask: torch.Tensor | None = None, | |
| position_ids: torch.LongTensor | None = None, | |
| inputs_embeds: torch.FloatTensor | None = None, | |
| output_hidden_states: bool | None = None, | |
| return_dict: bool | None = None, | |
| **kwargs: Unpack[TransformersKwargs], | |
| ) -> BaseModelOutput: | |
| if (input_ids is None) ^ (inputs_embeds is not None): | |
| raise ValueError("You must specify exactly one of input_ids or inputs_embeds") | |
| output_hidden_states = ( | |
| output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states | |
| ) | |
| if inputs_embeds is None: | |
| inputs_embeds = self.embed_tokens(input_ids) | |
| if self.embedding_proj is not None: | |
| inputs_embeds = self.embedding_proj(inputs_embeds) | |
| if position_ids is None: | |
| # RoPE is relative, so any per-row offset (e.g. left padding) gives the same attention scores | |
| position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0) | |
| bidirectional_mask = create_bidirectional_mask( | |
| config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask | |
| ) | |
| hidden_states = inputs_embeds | |
| position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) | |
| all_hidden_states = (hidden_states,) if output_hidden_states else None | |
| # Shared layers run back to back (layer 0, layer 0, layer 1, layer 1, ...), exactly as in training | |
| repeats = self.config.layer_sharing_repeats if self.config.layer_sharing else 1 | |
| for layer in self.layers[: self.config.num_hidden_layers]: | |
| for _ in range(repeats): | |
| hidden_states = layer( | |
| hidden_states, | |
| attention_mask=bidirectional_mask, | |
| position_embeddings=position_embeddings, | |
| **kwargs, | |
| ) | |
| if output_hidden_states: | |
| all_hidden_states += (hidden_states,) | |
| hidden_states = self.norm(hidden_states) | |
| return BaseModelOutput(last_hidden_state=hidden_states, hidden_states=all_hidden_states) | |
| class NandiPredictionHead(nn.Module): | |
| """`mlm_head: dense`: dense -> activation -> RMSNorm between the encoder's final norm and the decoder.""" | |
| def __init__(self, config): | |
| super().__init__() | |
| self.dense = nn.Linear(config.hidden_size, config.hidden_size, bias=False) | |
| self.act = ACT2FN[config.mlm_head_act] | |
| self.norm = NandiRMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| def forward(self, hidden_states): | |
| return self.norm(self.act(self.dense(hidden_states))) | |
| class NandiForMaskedLM(NandiEncoderPreTrainedModel): | |
| """Masked LM head = the pretrained LM head: final norm (in the encoder) -> lm_head_proj -> tied lm_head.""" | |
| _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} | |
| def __init__(self, config: NandiEncoderConfig): | |
| super().__init__(config) | |
| self.model = NandiEncoderModel(config) | |
| self.vocab_size = config.vocab_size | |
| if getattr(config, "mlm_head", "tied_factorized") == "dense": | |
| # dense -> act -> RMSNorm, then an untied full-width decoder | |
| self.head = NandiPredictionHead(config) | |
| self.lm_head_proj = None | |
| self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=config.mlm_head_decoder_bias) | |
| else: | |
| self.head = None | |
| lm_head_in = config.embedding_rank if config.factorized_embedding else config.hidden_size | |
| self.lm_head_proj = ( | |
| nn.Linear(config.hidden_size, config.embedding_rank, bias=False) if config.factorized_embedding else None | |
| ) | |
| self.lm_head = nn.Linear(lm_head_in, config.vocab_size, bias=False) | |
| self.post_init() | |
| def get_output_embeddings(self): | |
| return self.lm_head | |
| def set_output_embeddings(self, new_embeddings): | |
| self.lm_head = new_embeddings | |
| def forward( | |
| self, | |
| input_ids: torch.LongTensor | None = None, | |
| attention_mask: torch.Tensor | None = None, | |
| position_ids: torch.LongTensor | None = None, | |
| inputs_embeds: torch.FloatTensor | None = None, | |
| labels: torch.LongTensor | None = None, | |
| output_hidden_states: bool | None = None, | |
| return_dict: bool | None = None, | |
| **kwargs: Unpack[TransformersKwargs], | |
| ) -> MaskedLMOutput: | |
| outputs = self.model( | |
| input_ids=input_ids, | |
| attention_mask=attention_mask, | |
| position_ids=position_ids, | |
| inputs_embeds=inputs_embeds, | |
| output_hidden_states=output_hidden_states, | |
| **kwargs, | |
| ) | |
| hidden_states = outputs.last_hidden_state | |
| if self.head is not None: | |
| hidden_states = self.head(hidden_states) | |
| if self.lm_head_proj is not None: | |
| hidden_states = self.lm_head_proj(hidden_states) | |
| logits = self.lm_head(hidden_states) | |
| loss = None | |
| if labels is not None: # -100 = not predicted | |
| loss = nn.functional.cross_entropy( | |
| logits.float().view(-1, self.vocab_size), labels.to(logits.device).view(-1), ignore_index=-100 | |
| ) | |
| return MaskedLMOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states) | |
| def _pool(hidden_states: torch.Tensor, attention_mask: torch.Tensor | None, pooling: str) -> torch.Tensor: | |
| if pooling == "first": # the <|endoftext|> every input starts with | |
| return hidden_states[:, 0] | |
| if attention_mask is None: | |
| return hidden_states.mean(dim=1) | |
| mask = attention_mask.to(hidden_states.dtype).unsqueeze(-1) | |
| return (hidden_states * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1.0) | |
| class NandiForSequenceClassification(NandiEncoderPreTrainedModel): | |
| """Pooled encoder output (`config.classifier_pooling`) -> [pretrained MLM head] -> dropout -> linear. | |
| With `config.classifier_head = "mlm"` the pooled vector goes through the pretrained MLM head (dense -> act -> RMSNorm, | |
| loaded from `head.*`) before the linear layer. Only the linear layer is newly initialized. | |
| """ | |
| def __init__(self, config: NandiEncoderConfig): | |
| super().__init__(config) | |
| self.num_labels = config.num_labels | |
| self.model = NandiEncoderModel(config) | |
| self.dropout = nn.Dropout(config.classifier_dropout) | |
| self.classifier = nn.Linear(config.hidden_size, config.num_labels) | |
| # classifier_head "mlm": the pretrained MLM head (dense, act, RMSNorm; loads from head.*) sits before the linear | |
| self.head = NandiPredictionHead(config) if getattr(config, "classifier_head", None) == "mlm" else None | |
| self.post_init() | |
| def forward( | |
| self, | |
| input_ids: torch.LongTensor | None = None, | |
| attention_mask: torch.Tensor | None = None, | |
| position_ids: torch.LongTensor | None = None, | |
| inputs_embeds: torch.FloatTensor | None = None, | |
| labels: torch.LongTensor | None = None, | |
| output_hidden_states: bool | None = None, | |
| return_dict: bool | None = None, | |
| **kwargs: Unpack[TransformersKwargs], | |
| ) -> SequenceClassifierOutput: | |
| outputs = self.model( | |
| input_ids=input_ids, | |
| attention_mask=attention_mask, | |
| position_ids=position_ids, | |
| inputs_embeds=inputs_embeds, | |
| output_hidden_states=output_hidden_states, | |
| **kwargs, | |
| ) | |
| pooled = _pool(outputs.last_hidden_state, attention_mask, self.config.classifier_pooling) | |
| if self.head is not None: | |
| pooled = self.head(pooled) | |
| logits = self.classifier(self.dropout(pooled)) | |
| loss = None | |
| if labels is not None: | |
| labels = labels.to(logits.device) | |
| if self.config.problem_type is None: | |
| if self.num_labels == 1: | |
| self.config.problem_type = "regression" | |
| elif labels.dtype in (torch.long, torch.int): | |
| self.config.problem_type = "single_label_classification" | |
| else: | |
| self.config.problem_type = "multi_label_classification" | |
| if self.config.problem_type == "regression": | |
| loss = MSELoss()(logits.squeeze(), labels.squeeze().to(logits.dtype)) if self.num_labels == 1 else MSELoss()(logits, labels) | |
| elif self.config.problem_type == "single_label_classification": | |
| loss = CrossEntropyLoss()(logits.view(-1, self.num_labels), labels.view(-1)) | |
| else: | |
| loss = BCEWithLogitsLoss()(logits, labels.to(logits.dtype)) | |
| return SequenceClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states) | |
| class NandiForTokenClassification(NandiEncoderPreTrainedModel): | |
| """Per-token encoder output -> [pretrained MLM head] -> dropout -> linear. | |
| With `config.classifier_head = "mlm"` every token goes through the pretrained MLM head (dense -> act -> RMSNorm, | |
| loaded from `head.*`) before the linear layer. Only the linear layer is newly initialized. | |
| """ | |
| def __init__(self, config: NandiEncoderConfig): | |
| super().__init__(config) | |
| self.num_labels = config.num_labels | |
| self.model = NandiEncoderModel(config) | |
| self.dropout = nn.Dropout(config.classifier_dropout) | |
| self.classifier = nn.Linear(config.hidden_size, config.num_labels) | |
| # classifier_head "mlm": the pretrained MLM head (dense, act, RMSNorm; loads from head.*) sits before the linear | |
| self.head = NandiPredictionHead(config) if getattr(config, "classifier_head", None) == "mlm" else None | |
| self.post_init() | |
| def forward( | |
| self, | |
| input_ids: torch.LongTensor | None = None, | |
| attention_mask: torch.Tensor | None = None, | |
| position_ids: torch.LongTensor | None = None, | |
| inputs_embeds: torch.FloatTensor | None = None, | |
| labels: torch.LongTensor | None = None, | |
| output_hidden_states: bool | None = None, | |
| return_dict: bool | None = None, | |
| **kwargs: Unpack[TransformersKwargs], | |
| ) -> TokenClassifierOutput: | |
| outputs = self.model( | |
| input_ids=input_ids, | |
| attention_mask=attention_mask, | |
| position_ids=position_ids, | |
| inputs_embeds=inputs_embeds, | |
| output_hidden_states=output_hidden_states, | |
| **kwargs, | |
| ) | |
| h = outputs.last_hidden_state if self.head is None else self.head(outputs.last_hidden_state) | |
| logits = self.classifier(self.dropout(h)) | |
| loss = None | |
| if labels is not None: | |
| loss = CrossEntropyLoss()(logits.view(-1, self.num_labels), labels.to(logits.device).view(-1)) | |
| return TokenClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states) | |
| __all__ = [ | |
| "NandiEncoderPreTrainedModel", | |
| "NandiEncoderModel", | |
| "NandiForMaskedLM", | |
| "NandiForSequenceClassification", | |
| "NandiForTokenClassification", | |
| ] | |