Spaces:
Running
Running
File size: 4,597 Bytes
5c76335 5ed7f16 5c76335 5ed7f16 5c76335 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 | """
app/classifiers/text_preprocessor.py
======================================
Unicode-aware text preprocessor that runs before any classifier.
Responsibilities:
1. Strip invisible / control Unicode characters and flag their presence
(zero-width spaces, RTL overrides, soft hyphens, etc.)
2. Apply NFKC Unicode normalization to collapse compatibility variants and
homoglyph lookalikes (e.g. Cyrillic 'а' → Latin 'a')
3. Return cleaned text + metadata dict for downstream bias adjustments
The preprocessor is a pure-Python, ML-free module with no heavy dependencies.
It is called before every classifier (not just hf2) so that even the fast
sklearn path benefits from Unicode normalization.
Metadata flags drive the invisible-char probability bias in input_guard.py:
- had_invisible_chars: True if zero-width or other invisible chars were found
- had_rtl_override: True if a bidirectional control override was found
- had_homoglyphs: True if characters that NFKC normalized significantly differ
- unicode_normalized: True if NFKC changed the text at all
"""
from __future__ import annotations
import unicodedata
class TextPreprocessor:
"""
Stateless text preprocessor for adversarial Unicode normalization.
Usage:
preprocessor = TextPreprocessor()
cleaned, meta = preprocessor.preprocess(raw_text)
# Use cleaned for classifier, meta for bias decisions
"""
# Characters that are invisible / zero-width in most rendering contexts
# and have no semantic value in legitimate user input.
INVISIBLE_CHARS: frozenset[str] = frozenset(
[
"\u200b", # zero-width space
"\u200c", # zero-width non-joiner
"\u200d", # zero-width joiner
"\u2060", # word joiner
"\u00ad", # soft hyphen
"\ufeff", # BOM / zero-width no-break space
"\u200e", # left-to-right mark
"\u200f", # right-to-left mark
"\u202a", # LTR embedding
"\u202b", # RTL embedding
"\u202c", # pop directional formatting
"\u202d", # LTR override
"\u202e", # RTL override (most dangerous — reverses displayed text)
"\u2066", # LTR isolate
"\u2067", # RTL isolate
"\u2068", # first strong isolate
"\u2069", # pop directional isolate
]
)
# RTL override chars are a strong signal on their own
RTL_OVERRIDE_CHARS: frozenset[str] = frozenset(
[
"\u202e", # RTL override
"\u202b", # RTL embedding
"\u2067", # RTL isolate
]
)
def preprocess(self, text: str) -> tuple[str, dict[str, bool]]:
"""
Normalize *text* for safe classifier input.
Returns:
(cleaned_text, metadata)
metadata keys:
had_invisible_chars: bool — invisible/zero-width chars were found
had_rtl_override: bool — RTL control chars were found (strong signal)
unicode_normalized: bool — NFKC changed the text (covers homoglyphs)
"""
if not text:
return text, {
"had_invisible_chars": False,
"had_rtl_override": False,
"unicode_normalized": False,
}
had_invisible = False
had_rtl = False
# ── Step 1: detect and strip invisible/control chars ──────────────────
stripped_chars: list[str] = []
for ch in text:
if ch in self.INVISIBLE_CHARS:
had_invisible = True
if ch in self.RTL_OVERRIDE_CHARS:
had_rtl = True
# Strip — do not add to output
continue
stripped_chars.append(ch)
stripped = "".join(stripped_chars)
# ── Step 2: NFKC normalization ─────────────────────────────────────────
# NFKC collapses compatibility variants and maps many Unicode lookalikes
# (Cyrillic, Greek, fullwidth, superscript digits) to their ASCII
# equivalents. This handles the 'homoglyph' attack class automatically.
normalized = unicodedata.normalize("NFKC", stripped)
unicode_normalized = normalized != text
return normalized, {
"had_invisible_chars": had_invisible,
"had_rtl_override": had_rtl,
"unicode_normalized": unicode_normalized,
}
|