File size: 725 Bytes
c03e652 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | import re
import unicodedata
_ZERO_WIDTH = ('\u200b','\u200c','\u200d','\ufeff')
def strip_special_token_strings(text, special_tokens):
out = str(text or '')
for tok in sorted([t for t in (special_tokens or []) if isinstance(t, str) and t], key=len, reverse=True):
out = out.replace(tok, '')
return out
def normalize_for_metric(text, newline_token='<NL>'):
out = str(text or '')
out = out.replace('\r\n', '\n').replace('\r', '\n')
out = out.replace('<NL>', '\n')
for ch in _ZERO_WIDTH:
out = out.replace(ch, '')
out = unicodedata.normalize('NFC', out)
out = re.sub(r'[ \t]+', ' ', out)
out = re.sub(r' *\n *', '\n', out)
_ = newline_token
return out.strip()
|