Commit ·
417db6e
1
Parent(s): 1502667
Audit fixes: narrow FIX-35 suffixes, raise FIX-37 threshold, remove duplicate sort
Browse files- FIX-35: Narrowed suffix set from {ن,ت,ا,ي,ة,و,ه} to {ن,ت} only.
Only verb conjugation suffixes are blocked. ة/ا/ي/و/ه are legitimate
typo corrections that should not be blocked.
- FIX-37: Raised word threshold from 2 to 4. Prevents noisy period
suggestions while user is still typing short phrases.
- FIX-32b: Removed duplicate .sort() in applyAllSuggestions — already
sorted at line 679, second sort at 695 was redundant.
- Tests updated to match narrowed suffix expectations.
- src/app.py +10 -6
- src/js/editor.js +0 -4
- tests/test_recent_fixes.py +8 -7
src/app.py
CHANGED
|
@@ -1062,11 +1062,13 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
|
|
| 1062 |
_orig_oov = not vocab_manager.is_iv(orig_word)
|
| 1063 |
_corr_iv = vocab_manager.is_iv(corr_word)
|
| 1064 |
if _orig_oov and _corr_iv:
|
| 1065 |
-
# FIX-35: Don't strip
|
| 1066 |
-
#
|
| 1067 |
-
#
|
| 1068 |
-
#
|
| 1069 |
-
|
|
|
|
|
|
|
| 1070 |
_removed_char = None
|
| 1071 |
for _di2 in range(len(orig_word)):
|
| 1072 |
if orig_word[:_di2] + orig_word[_di2 + 1:] == corr_word:
|
|
@@ -2314,12 +2316,14 @@ def analyze_text():
|
|
| 2314 |
# The punctuation model often fails to add a period at the end
|
| 2315 |
# of longer sentences. If no terminal punctuation exists after
|
| 2316 |
# model processing, inject a period suggestion for the last word.
|
|
|
|
|
|
|
| 2317 |
import re as _re_punc
|
| 2318 |
_TERMINAL_PUNCT = set('.،؛؟!?!')
|
| 2319 |
_current_stripped = ctx.current_text.rstrip()
|
| 2320 |
_has_terminal = _current_stripped and _current_stripped[-1] in _TERMINAL_PUNCT
|
| 2321 |
_word_count_fb = len(_re_punc.findall(r'[\u0600-\u06FFa-zA-Z]+', ctx.current_text))
|
| 2322 |
-
if not _has_terminal and _word_count_fb >=
|
| 2323 |
# Find the last word's position in current_text
|
| 2324 |
_last_word_match = _re_punc.search(r'([\u0600-\u06FF]+)\s*$', _current_stripped)
|
| 2325 |
if _last_word_match:
|
|
|
|
| 1062 |
_orig_oov = not vocab_manager.is_iv(orig_word)
|
| 1063 |
_corr_iv = vocab_manager.is_iv(corr_word)
|
| 1064 |
if _orig_oov and _corr_iv:
|
| 1065 |
+
# FIX-35: Don't strip verb conjugation suffixes.
|
| 1066 |
+
# Only block ن (feminine plural: ذهبن→ذهب) and
|
| 1067 |
+
# ت (feminine past: كتبت→كتب) — these are the
|
| 1068 |
+
# suffixes grammar commonly adds that spelling
|
| 1069 |
+
# would try to strip. Other endings (ة,ا,ي,و,ه)
|
| 1070 |
+
# are more likely genuine typos than grammar fixes.
|
| 1071 |
+
_CONJUGATION_SUFFIXES = {'ن', 'ت'}
|
| 1072 |
_removed_char = None
|
| 1073 |
for _di2 in range(len(orig_word)):
|
| 1074 |
if orig_word[:_di2] + orig_word[_di2 + 1:] == corr_word:
|
|
|
|
| 2316 |
# The punctuation model often fails to add a period at the end
|
| 2317 |
# of longer sentences. If no terminal punctuation exists after
|
| 2318 |
# model processing, inject a period suggestion for the last word.
|
| 2319 |
+
# Threshold=4 words to avoid noisy suggestions while user is
|
| 2320 |
+
# still typing short phrases.
|
| 2321 |
import re as _re_punc
|
| 2322 |
_TERMINAL_PUNCT = set('.،؛؟!?!')
|
| 2323 |
_current_stripped = ctx.current_text.rstrip()
|
| 2324 |
_has_terminal = _current_stripped and _current_stripped[-1] in _TERMINAL_PUNCT
|
| 2325 |
_word_count_fb = len(_re_punc.findall(r'[\u0600-\u06FFa-zA-Z]+', ctx.current_text))
|
| 2326 |
+
if not _has_terminal and _word_count_fb >= 4:
|
| 2327 |
# Find the last word's position in current_text
|
| 2328 |
_last_word_match = _re_punc.search(r'([\u0600-\u06FF]+)\s*$', _current_stripped)
|
| 2329 |
if _last_word_match:
|
src/js/editor.js
CHANGED
|
@@ -689,10 +689,6 @@ function applyAllSuggestions() {
|
|
| 689 |
// Reverse-order text replacement is safe — each patch's offsets are
|
| 690 |
// valid because we haven't modified anything before them yet.
|
| 691 |
let text = getEditorText();
|
| 692 |
-
// FIX-32b: MUST sort by descending offset — replacing from end-to-start
|
| 693 |
-
// ensures earlier offsets remain valid. Without this, applying a fix at
|
| 694 |
-
// offset 5 shifts everything after it, corrupting offset 20's fix.
|
| 695 |
-
suggestions.sort((a, b) => b.start - a.start);
|
| 696 |
suggestions.forEach((s) => {
|
| 697 |
if (s.start >= 0 && s.end <= text.length && s.start <= s.end) {
|
| 698 |
text = text.substring(0, s.start) + s.correction + text.substring(s.end);
|
|
|
|
| 689 |
// Reverse-order text replacement is safe — each patch's offsets are
|
| 690 |
// valid because we haven't modified anything before them yet.
|
| 691 |
let text = getEditorText();
|
|
|
|
|
|
|
|
|
|
|
|
|
| 692 |
suggestions.forEach((s) => {
|
| 693 |
if (s.start >= 0 && s.end <= text.length && s.start <= s.end) {
|
| 694 |
text = text.substring(0, s.start) + s.correction + text.substring(s.end);
|
tests/test_recent_fixes.py
CHANGED
|
@@ -86,7 +86,7 @@ for input_text, expected in plurals:
|
|
| 86 |
# ══════════════════════════════════════════════════════════════
|
| 87 |
print("\n═══ FIX-35: Spelling — conjugation suffix protection ═══")
|
| 88 |
|
| 89 |
-
_CONJUGATION_SUFFIXES = {'ن', 'ت'
|
| 90 |
|
| 91 |
def simulate_insertion_fix_check(orig_word, corr_word):
|
| 92 |
"""Simulate the FIX-35 suffix strip check logic from app.py."""
|
|
@@ -105,7 +105,7 @@ def simulate_insertion_fix_check(orig_word, corr_word):
|
|
| 105 |
return "allowed"
|
| 106 |
return "not_applicable"
|
| 107 |
|
| 108 |
-
# BLOCKED cases (suffix stripping)
|
| 109 |
test("'ذهبن'→'ذهب' blocked (ن suffix)",
|
| 110 |
simulate_insertion_fix_check("ذهبن", "ذهب") == "blocked",
|
| 111 |
f"got {simulate_insertion_fix_check('ذهبن', 'ذهب')}")
|
|
@@ -114,15 +114,16 @@ test("'كتبت'→'كتب' blocked (ت suffix)",
|
|
| 114 |
simulate_insertion_fix_check("كتبت", "كتب") == "blocked",
|
| 115 |
f"got {simulate_insertion_fix_check('كتبت', 'كتب')}")
|
| 116 |
|
| 117 |
-
|
| 118 |
-
|
|
|
|
| 119 |
f"got {simulate_insertion_fix_check('درسة', 'درس')}")
|
| 120 |
|
| 121 |
-
test("'جلسوا'→'جلسو'
|
| 122 |
-
simulate_insertion_fix_check("جلسوا", "جلسو") == "
|
| 123 |
f"got {simulate_insertion_fix_check('جلسوا', 'جلسو')}")
|
| 124 |
|
| 125 |
-
# ALLOWED cases (mid-word insertion fix)
|
| 126 |
test("'الكتتاب'→'الكتاب' allowed (mid-word extra ت)",
|
| 127 |
simulate_insertion_fix_check("الكتتاب", "الكتاب") == "allowed",
|
| 128 |
f"got {simulate_insertion_fix_check('الكتتاب', 'الكتاب')}")
|
|
|
|
| 86 |
# ══════════════════════════════════════════════════════════════
|
| 87 |
print("\n═══ FIX-35: Spelling — conjugation suffix protection ═══")
|
| 88 |
|
| 89 |
+
_CONJUGATION_SUFFIXES = {'ن', 'ت'}
|
| 90 |
|
| 91 |
def simulate_insertion_fix_check(orig_word, corr_word):
|
| 92 |
"""Simulate the FIX-35 suffix strip check logic from app.py."""
|
|
|
|
| 105 |
return "allowed"
|
| 106 |
return "not_applicable"
|
| 107 |
|
| 108 |
+
# BLOCKED cases (verb conjugation suffix stripping)
|
| 109 |
test("'ذهبن'→'ذهب' blocked (ن suffix)",
|
| 110 |
simulate_insertion_fix_check("ذهبن", "ذهب") == "blocked",
|
| 111 |
f"got {simulate_insertion_fix_check('ذهبن', 'ذهب')}")
|
|
|
|
| 114 |
simulate_insertion_fix_check("كتبت", "كتب") == "blocked",
|
| 115 |
f"got {simulate_insertion_fix_check('كتبت', 'كتب')}")
|
| 116 |
|
| 117 |
+
# NOW ALLOWED cases (narrowed set — ة, و no longer blocked)
|
| 118 |
+
test("'درسة'→'درس' allowed (ة not in narrowed set)",
|
| 119 |
+
simulate_insertion_fix_check("درسة", "درس") == "allowed",
|
| 120 |
f"got {simulate_insertion_fix_check('درسة', 'درس')}")
|
| 121 |
|
| 122 |
+
test("'جلسوا'→'جلسو' allowed (ا not in narrowed set)",
|
| 123 |
+
simulate_insertion_fix_check("جلسوا", "جلسو") == "allowed",
|
| 124 |
f"got {simulate_insertion_fix_check('جلسوا', 'جلسو')}")
|
| 125 |
|
| 126 |
+
# ALLOWED cases (mid-word insertion fix — always allowed)
|
| 127 |
test("'الكتتاب'→'الكتاب' allowed (mid-word extra ت)",
|
| 128 |
simulate_insertion_fix_check("الكتتاب", "الكتاب") == "allowed",
|
| 129 |
f"got {simulate_insertion_fix_check('الكتتاب', 'الكتاب')}")
|