NandiBERT-450M / tokenization_nandi_encoder.py
akanyaani's picture
Classifiers use the pretrained MLM head; tokenizer word spaces; README tokenizer needs trust_remote_code
4acb258 verified
Raw History Blame Contribute Delete
5.76 kB
# Copyright 2026 FrontiersMind. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tokenizer of the Nandi encoder: the Nandi BPE tokenizer, with `<|endoftext|>` prepended to every input
(`add_special_tokens=True`, the default) because every training document started with it. Unlike the decoder
tokenizer, no `<|im_start|>` and no extra space are added. With `is_split_into_words=True` (NER) every word gets its
leading space, as in running text."""
import re
from tokenizers import Regex, Tokenizer, decoders, normalizers, pre_tokenizers
from tokenizers.models import BPE
from transformers.tokenization_utils_tokenizers import TokenizersBackend
from transformers.utils import logging
logger = logging.get_logger(__name__)
PRETOKENIZE_REGEX = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?(?:\p{L}\p{M}*)+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
# Training replaced a masked word *token* (which carries its own leading space, e.g. " Paris") by <|mask|>, so a mask never
# has a separate space token in front of it, while newlines / indentation before a word stay as their own tokens.
# To get the same ids from natural text ("is <|mask|>."), exactly ONE space directly before <|mask|> is absorbed.
# (The mask is NOT lstrip: that would also swallow newlines and indentation, which training keeps.)
_SPACE_BEFORE_MASK = re.compile(r" (?=<\|mask\|>)")
def _absorb_space_before_mask(x):
if isinstance(x, str):
return _SPACE_BEFORE_MASK.sub("", x)
if isinstance(x, list) and all(isinstance(i, str) for i in x):
return [_SPACE_BEFORE_MASK.sub("", i) for i in x]
return x
def _space_split_words(x):
"""`is_split_into_words=True`: give every word the leading space it has in running text ("Delhi" -> " Delhi"), as
in our NER fine-tuning. Words that already start with whitespace are kept as they are. A `<|mask|>` word gets no
space (same rule as above). Works for one example (list of words) and for a batch (list of lists of words)."""
if isinstance(x, (list, tuple)):
if all(isinstance(w, str) for w in x):
return [_SPACE_BEFORE_MASK.sub("", w if (not w or w[0].isspace()) else " " + w) for w in x]
return type(x)(_space_split_words(i) for i in x)
return x
class NandiEncoderTokenizer(TokenizersBackend):
model_input_names = ["input_ids", "attention_mask"]
model = BPE
def _encode_plus(self, text, text_pair=None, *args, is_split_into_words=False, **kwargs):
if not is_split_into_words:
text, text_pair = _absorb_space_before_mask(text), _absorb_space_before_mask(text_pair)
elif getattr(self, "add_prefix_space", None) is not False: # add_prefix_space=False turns the word spaces off
text, text_pair = _space_split_words(text), _space_split_words(text_pair)
return super()._encode_plus(text, text_pair, *args, is_split_into_words=is_split_into_words, **kwargs)
def __init__(
self,
vocab: str | dict[str, int] | None = None,
merges: str | list[str] | None = None,
vocab_file=None,
merges_file=None,
unk_token: str = "<|endoftext|>",
bos_token: str = "<|endoftext|>",
eos_token: str = "<|endoftext|>",
pad_token: str = "<|pad|>",
mask_token: str = "<|mask|>",
add_prefix_space: bool | None = None,
**kwargs,
):
self._vocab = (
vocab
if vocab is not None
else {
"<|endoftext|>": 0,
}
)
self._merges = merges or []
self._tokenizer = Tokenizer(
BPE(
vocab=self._vocab,
merges=self._merges,
dropout=None,
unk_token=None,
continuing_subword_prefix="",
end_of_word_suffix="",
fuse_unk=False,
byte_fallback=False,
)
)
self._tokenizer.decoder = decoders.ByteLevel()
self._tokenizer.normalizer = normalizers.NFC()
self._tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
[
pre_tokenizers.Split(
Regex(PRETOKENIZE_REGEX),
behavior="isolated",
invert=False,
),
pre_tokenizers.ByteLevel(
add_prefix_space=False,
trim_offsets=True,
use_regex=False
),
]
)
# Every training document started with <|endoftext|> (the bos_token here): prepend it by default.
# Passed explicitly so TokenizersBackend builds the "<|endoftext|> $A" post-processor.
kwargs["add_bos_token"] = kwargs.get("add_bos_token", True)
kwargs["add_eos_token"] = kwargs.get("add_eos_token", False)
super().__init__(
vocab_file=vocab_file,
merges_file=merges_file,
unk_token=unk_token,
bos_token=bos_token,
eos_token=eos_token,
pad_token=pad_token,
mask_token=mask_token,
add_prefix_space=add_prefix_space,
**kwargs,
)
__all__ = ["NandiEncoderTokenizer"]