Blame for ml/tokenizer/regex_split.py

"""GPT-2 style pre-tokenization. Splits text into chunks (words with a

leading space, number runs, punctuation runs, whitespace) so BPE only

ever merges *within* a chunk, never across word boundaries."""

import regex as re

# The GPT-2 pattern. Order matters: contractions, spaced letters,
# spaced numbers, spaced punctuation, the whitespace.
GPT2_PATTERN = (
    r"""'s|'t|'re|'ve|'m|'ll|'d"""
    r"""| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
)


_PAT = re.compile(GPT2_PATTERN)

def split(text):
    """Return list of chunk strings."""
    return _PAT.findall(text)