repro/text_normalization.json

{
  "unicode_normalization": "NFC",
  "newline_normalization": "\\r\\n and \\r -> \\n",
  "special_token_cleanup": "tokenizer.batch_decode(skip_special_tokens=True) plus explicit string removal of tokenizer.all_special_tokens",
  "zero_width_removed": [
    "\\u200b",
    "\\u200c",
    "\\u200d",
    "\\ufeff"
  ],
  "whitespace_normalization": "collapse horizontal spaces/tabs, trim around newlines, strip",
  "metric_newline_token": "<NL>"
}