Fill-Mask
Transformers
Safetensors
nandi_encoder
encoder
masked-language-modeling
indic
multilingual
code
efficient
frontiersmind
nandi
custom_code
Instructions to use FrontiersMind/NandiBERT-150M with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use FrontiersMind/NandiBERT-150M with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("fill-mask", model="FrontiersMind/NandiBERT-150M", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForMaskedLM model = AutoModelForMaskedLM.from_pretrained("FrontiersMind/NandiBERT-150M", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download tokenization_nandi_encoder.py from FrontiersMind/NandiBERT-150M: direct link, hf CLI and curl.
- Browser
- Download file 5.76 kB
-
https://huggingface.co/FrontiersMind/NandiBERT-150M/resolve/main/tokenization_nandi_encoder.py
- Command line
-
hf download hf://FrontiersMind/NandiBERT-150M/tokenization_nandi_encoder.py
-
curl -L -o tokenization_nandi_encoder.py https://huggingface.co/FrontiersMind/NandiBERT-150M/resolve/main/tokenization_nandi_encoder.py
5.76 kB
| # Copyright 2026 FrontiersMind. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| """Tokenizer of the Nandi encoder: the Nandi BPE tokenizer, with `<|endoftext|>` prepended to every input | |
| (`add_special_tokens=True`, the default) because every training document started with it. Unlike the decoder | |
| tokenizer, no `<|im_start|>` and no extra space are added. With `is_split_into_words=True` (NER) every word gets its | |
| leading space, as in running text.""" | |
| import re | |
| from tokenizers import Regex, Tokenizer, decoders, normalizers, pre_tokenizers | |
| from tokenizers.models import BPE | |
| from transformers.tokenization_utils_tokenizers import TokenizersBackend | |
| from transformers.utils import logging | |
| logger = logging.get_logger(__name__) | |
| PRETOKENIZE_REGEX = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?(?:\p{L}\p{M}*)+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+""" | |
| # Training replaced a masked word *token* (which carries its own leading space, e.g. " Paris") by <|mask|>, so a mask never | |
| # has a separate space token in front of it, while newlines / indentation before a word stay as their own tokens. | |
| # To get the same ids from natural text ("is <|mask|>."), exactly ONE space directly before <|mask|> is absorbed. | |
| # (The mask is NOT lstrip: that would also swallow newlines and indentation, which training keeps.) | |
| _SPACE_BEFORE_MASK = re.compile(r" (?=<\|mask\|>)") | |
| def _absorb_space_before_mask(x): | |
| if isinstance(x, str): | |
| return _SPACE_BEFORE_MASK.sub("", x) | |
| if isinstance(x, list) and all(isinstance(i, str) for i in x): | |
| return [_SPACE_BEFORE_MASK.sub("", i) for i in x] | |
| return x | |
| def _space_split_words(x): | |
| """`is_split_into_words=True`: give every word the leading space it has in running text ("Delhi" -> " Delhi"), as | |
| in our NER fine-tuning. Words that already start with whitespace are kept as they are. A `<|mask|>` word gets no | |
| space (same rule as above). Works for one example (list of words) and for a batch (list of lists of words).""" | |
| if isinstance(x, (list, tuple)): | |
| if all(isinstance(w, str) for w in x): | |
| return [_SPACE_BEFORE_MASK.sub("", w if (not w or w[0].isspace()) else " " + w) for w in x] | |
| return type(x)(_space_split_words(i) for i in x) | |
| return x | |
| class NandiEncoderTokenizer(TokenizersBackend): | |
| model_input_names = ["input_ids", "attention_mask"] | |
| model = BPE | |
| def _encode_plus(self, text, text_pair=None, *args, is_split_into_words=False, **kwargs): | |
| if not is_split_into_words: | |
| text, text_pair = _absorb_space_before_mask(text), _absorb_space_before_mask(text_pair) | |
| elif getattr(self, "add_prefix_space", None) is not False: # add_prefix_space=False turns the word spaces off | |
| text, text_pair = _space_split_words(text), _space_split_words(text_pair) | |
| return super()._encode_plus(text, text_pair, *args, is_split_into_words=is_split_into_words, **kwargs) | |
| def __init__( | |
| self, | |
| vocab: str | dict[str, int] | None = None, | |
| merges: str | list[str] | None = None, | |
| vocab_file=None, | |
| merges_file=None, | |
| unk_token: str = "<|endoftext|>", | |
| bos_token: str = "<|endoftext|>", | |
| eos_token: str = "<|endoftext|>", | |
| pad_token: str = "<|pad|>", | |
| mask_token: str = "<|mask|>", | |
| add_prefix_space: bool | None = None, | |
| **kwargs, | |
| ): | |
| self._vocab = ( | |
| vocab | |
| if vocab is not None | |
| else { | |
| "<|endoftext|>": 0, | |
| } | |
| ) | |
| self._merges = merges or [] | |
| self._tokenizer = Tokenizer( | |
| BPE( | |
| vocab=self._vocab, | |
| merges=self._merges, | |
| dropout=None, | |
| unk_token=None, | |
| continuing_subword_prefix="", | |
| end_of_word_suffix="", | |
| fuse_unk=False, | |
| byte_fallback=False, | |
| ) | |
| ) | |
| self._tokenizer.decoder = decoders.ByteLevel() | |
| self._tokenizer.normalizer = normalizers.NFC() | |
| self._tokenizer.pre_tokenizer = pre_tokenizers.Sequence( | |
| [ | |
| pre_tokenizers.Split( | |
| Regex(PRETOKENIZE_REGEX), | |
| behavior="isolated", | |
| invert=False, | |
| ), | |
| pre_tokenizers.ByteLevel( | |
| add_prefix_space=False, | |
| trim_offsets=True, | |
| use_regex=False | |
| ), | |
| ] | |
| ) | |
| # Every training document started with <|endoftext|> (the bos_token here): prepend it by default. | |
| # Passed explicitly so TokenizersBackend builds the "<|endoftext|> $A" post-processor. | |
| kwargs["add_bos_token"] = kwargs.get("add_bos_token", True) | |
| kwargs["add_eos_token"] = kwargs.get("add_eos_token", False) | |
| super().__init__( | |
| vocab_file=vocab_file, | |
| merges_file=merges_file, | |
| unk_token=unk_token, | |
| bos_token=bos_token, | |
| eos_token=eos_token, | |
| pad_token=pad_token, | |
| mask_token=mask_token, | |
| add_prefix_space=add_prefix_space, | |
| **kwargs, | |
| ) | |
| __all__ = ["NandiEncoderTokenizer"] | |