# Copyright 2026 FrontiersMind. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tokenizer of the Nandi encoder: the Nandi BPE tokenizer, with `<|endoftext|>` prepended to every input (`add_special_tokens=True`, the default) because every training document started with it. Unlike the decoder tokenizer, no `<|im_start|>` and no extra space are added. With `is_split_into_words=True` (NER) every word gets its leading space, as in running text.""" import re from tokenizers import Regex, Tokenizer, decoders, normalizers, pre_tokenizers from tokenizers.models import BPE from transformers.tokenization_utils_tokenizers import TokenizersBackend from transformers.utils import logging logger = logging.get_logger(__name__) PRETOKENIZE_REGEX = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?(?:\p{L}\p{M}*)+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+""" # Training replaced a masked word *token* (which carries its own leading space, e.g. " Paris") by <|mask|>, so a mask never # has a separate space token in front of it, while newlines / indentation before a word stay as their own tokens. # To get the same ids from natural text ("is <|mask|>."), exactly ONE space directly before <|mask|> is absorbed. # (The mask is NOT lstrip: that would also swallow newlines and indentation, which training keeps.) _SPACE_BEFORE_MASK = re.compile(r" (?=<\|mask\|>)") def _absorb_space_before_mask(x): if isinstance(x, str): return _SPACE_BEFORE_MASK.sub("", x) if isinstance(x, list) and all(isinstance(i, str) for i in x): return [_SPACE_BEFORE_MASK.sub("", i) for i in x] return x def _space_split_words(x): """`is_split_into_words=True`: give every word the leading space it has in running text ("Delhi" -> " Delhi"), as in our NER fine-tuning. Words that already start with whitespace are kept as they are. A `<|mask|>` word gets no space (same rule as above). Works for one example (list of words) and for a batch (list of lists of words).""" if isinstance(x, (list, tuple)): if all(isinstance(w, str) for w in x): return [_SPACE_BEFORE_MASK.sub("", w if (not w or w[0].isspace()) else " " + w) for w in x] return type(x)(_space_split_words(i) for i in x) return x class NandiEncoderTokenizer(TokenizersBackend): model_input_names = ["input_ids", "attention_mask"] model = BPE def _encode_plus(self, text, text_pair=None, *args, is_split_into_words=False, **kwargs): if not is_split_into_words: text, text_pair = _absorb_space_before_mask(text), _absorb_space_before_mask(text_pair) elif getattr(self, "add_prefix_space", None) is not False: # add_prefix_space=False turns the word spaces off text, text_pair = _space_split_words(text), _space_split_words(text_pair) return super()._encode_plus(text, text_pair, *args, is_split_into_words=is_split_into_words, **kwargs) def __init__( self, vocab: str | dict[str, int] | None = None, merges: str | list[str] | None = None, vocab_file=None, merges_file=None, unk_token: str = "<|endoftext|>", bos_token: str = "<|endoftext|>", eos_token: str = "<|endoftext|>", pad_token: str = "<|pad|>", mask_token: str = "<|mask|>", add_prefix_space: bool | None = None, **kwargs, ): self._vocab = ( vocab if vocab is not None else { "<|endoftext|>": 0, } ) self._merges = merges or [] self._tokenizer = Tokenizer( BPE( vocab=self._vocab, merges=self._merges, dropout=None, unk_token=None, continuing_subword_prefix="", end_of_word_suffix="", fuse_unk=False, byte_fallback=False, ) ) self._tokenizer.decoder = decoders.ByteLevel() self._tokenizer.normalizer = normalizers.NFC() self._tokenizer.pre_tokenizer = pre_tokenizers.Sequence( [ pre_tokenizers.Split( Regex(PRETOKENIZE_REGEX), behavior="isolated", invert=False, ), pre_tokenizers.ByteLevel( add_prefix_space=False, trim_offsets=True, use_regex=False ), ] ) # Every training document started with <|endoftext|> (the bos_token here): prepend it by default. # Passed explicitly so TokenizersBackend builds the "<|endoftext|> $A" post-processor. kwargs["add_bos_token"] = kwargs.get("add_bos_token", True) kwargs["add_eos_token"] = kwargs.get("add_eos_token", False) super().__init__( vocab_file=vocab_file, merges_file=merges_file, unk_token=unk_token, bos_token=bos_token, eos_token=eos_token, pad_token=pad_token, mask_token=mask_token, add_prefix_space=add_prefix_space, **kwargs, ) __all__ = ["NandiEncoderTokenizer"]