Spaces:
Runtime error
Runtime error
| """GPT-2 style pre-tokenization. Splits text into chunks (words with a | |
| leading space, number runs, punctuation runs, whitespace) so BPE only | |
| ever merges *within* a chunk, never across word boundaries.""" | |
| import regex as re | |
| # The GPT-2 pattern. Order matters: contractions, spaced letters, | |
| # spaced numbers, spaced punctuation, the whitespace. | |
| GPT2_PATTERN = ( | |
| r"""'s|'t|'re|'ve|'m|'ll|'d""" | |
| r"""| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" | |
| ) | |
| _PAT = re.compile(GPT2_PATTERN) | |
| def split(text): | |
| """Return list of chunk strings.""" | |
| return _PAT.findall(text) |