Commit ·
904945c
1
Parent(s): 5f5d641
FIX-50: Whitelist bypass for FIX-38 pronoun guard
Browse filesROOT CAUSE FOUND: The spelling guard (FIX-38) blocked الحكومه→الحكومة
because stem الحكوم is IV in BERT vocab. But الحكوم is a tokenizer
fragment, not a real Arabic word. This caused a false positive in the
pronoun suffix detection.
Fix: Known feminine nouns (الحكومه, المدرسه, الشركه, etc.) bypass
the stem-IV check entirely. The whitelist contains ~50 common words.
This ALSO means the spelling stage will create a proper patch for
these words, which the final output builder uses.
Expected impact: PC034, PC035, and other three_stage_collision cases.
Tests: 39 passing.
- src/app.py +18 -0
src/app.py
CHANGED
|
@@ -987,6 +987,24 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
|
|
| 987 |
# Strategy (from legacy AraSpell WordAligner): if the STEM (word without ه)
|
| 988 |
# is itself IV, then ه is likely a pronoun suffix → block the change.
|
| 989 |
# If the stem is NOT IV, ه is likely a misspelled ة → allow.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 990 |
stem = orig_word[:-1]
|
| 991 |
if len(stem) >= 2 and vocab_manager.is_iv(stem):
|
| 992 |
logger.info(
|
|
|
|
| 987 |
# Strategy (from legacy AraSpell WordAligner): if the STEM (word without ه)
|
| 988 |
# is itself IV, then ه is likely a pronoun suffix → block the change.
|
| 989 |
# If the stem is NOT IV, ه is likely a misspelled ة → allow.
|
| 990 |
+
#
|
| 991 |
+
# FIX-50: Whitelist bypass — known feminine nouns always allowed.
|
| 992 |
+
# BERT vocab includes subword fragments (الحكوم, المدرس) as IV,
|
| 993 |
+
# causing false pronoun detection. These known words bypass the guard.
|
| 994 |
+
_KNOWN_FEMININE = {
|
| 995 |
+
'الحكومه', 'المدرسه', 'الشركه', 'الجامعه', 'المدينه',
|
| 996 |
+
'القصه', 'المكتبه', 'الطائره', 'الوزاره', 'المديره',
|
| 997 |
+
'المعلمه', 'الطالبه', 'القريه', 'الحديقه', 'المحكمه',
|
| 998 |
+
'المنطقه', 'الدوله', 'السياره', 'الغرفه', 'المحطه',
|
| 999 |
+
'الوظيفه', 'العائله', 'الحياه', 'الصلاه',
|
| 1000 |
+
'حكومه', 'مدرسه', 'شركه', 'جامعه', 'مدينه',
|
| 1001 |
+
'قصه', 'مكتبه', 'طائره', 'وزاره', 'مديره',
|
| 1002 |
+
'معلمه', 'طالبه', 'قريه', 'حديقه', 'محكمه',
|
| 1003 |
+
'منطقه', 'دوله', 'سياره', 'غرفه', 'محطه',
|
| 1004 |
+
'وظيفه', 'عائله', 'حياه', 'صلاه',
|
| 1005 |
+
}
|
| 1006 |
+
if orig_word in _KNOWN_FEMININE:
|
| 1007 |
+
return 0.9
|
| 1008 |
stem = orig_word[:-1]
|
| 1009 |
if len(stem) >= 2 and vocab_manager.is_iv(stem):
|
| 1010 |
logger.info(
|