repro/text_normalization.py

import re
import unicodedata

_ZERO_WIDTH = ('\u200b','\u200c','\u200d','\ufeff')

def strip_special_token_strings(text, special_tokens):
    out = str(text or '')
    for tok in sorted([t for t in (special_tokens or []) if isinstance(t, str) and t], key=len, reverse=True):
        out = out.replace(tok, '')
    return out

def normalize_for_metric(text, newline_token='<NL>'):
    out = str(text or '')
    out = out.replace('\r\n', '\n').replace('\r', '\n')
    out = out.replace('<NL>', '\n')
    for ch in _ZERO_WIDTH:
        out = out.replace(ch, '')
    out = unicodedata.normalize('NFC', out)
    out = re.sub(r'[ \t]+', ' ', out)
    out = re.sub(r' *\n *', '\n', out)
    _ = newline_token
    return out.strip()