Spaces:
Sleeping
Sleeping
| import os | |
| _QUOTE_NORMALIZER = [ | |
| ("\u201c", '"'), ("\u201d", '"'), ("\u201e", '"'), ("\u201f", '"'), | |
| ("\u2033", '"'), ("\u2036", '"'), | |
| ("\u2018", "'"), ("\u2019", "'"), ("\u201a", "'"), ("\u201b", "'"), | |
| ("\u2013", "-"), ("\u2014", "-"), | |
| ("\u2026", "..."), | |
| ] | |
| DEBUG = os.getenv("DEBUG_NORMALIZER", "false").lower() == "true" | |
| def normalize_text(text: str) -> str: | |
| if DEBUG: | |
| print("[DEBUG] normalize_text called") | |
| # Replace curly quotes, dashes, ellipses | |
| for bad, good in _QUOTE_NORMALIZER: | |
| text = text.replace(bad, good) | |
| # Final safeguard: strip unsupported characters | |
| if DEBUG: | |
| print("[DEBUG] Final Normalized Text: ", | |
| text.encode("utf-8", "ignore").decode("utf-8", "ignore")) | |
| return text.encode("utf-8", "ignore").decode("utf-8", "ignore") | |