youssefreda9 commited on
Commit
417db6e
·
1 Parent(s): 1502667

Audit fixes: narrow FIX-35 suffixes, raise FIX-37 threshold, remove duplicate sort

Browse files

- FIX-35: Narrowed suffix set from {ن,ت,ا,ي,ة,و,ه} to {ن,ت} only.
Only verb conjugation suffixes are blocked. ة/ا/ي/و/ه are legitimate
typo corrections that should not be blocked.
- FIX-37: Raised word threshold from 2 to 4. Prevents noisy period
suggestions while user is still typing short phrases.
- FIX-32b: Removed duplicate .sort() in applyAllSuggestions — already
sorted at line 679, second sort at 695 was redundant.
- Tests updated to match narrowed suffix expectations.

Files changed (3) hide show
  1. src/app.py +10 -6
  2. src/js/editor.js +0 -4
  3. tests/test_recent_fixes.py +8 -7
src/app.py CHANGED
@@ -1062,11 +1062,13 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
1062
  _orig_oov = not vocab_manager.is_iv(orig_word)
1063
  _corr_iv = vocab_manager.is_iv(corr_word)
1064
  if _orig_oov and _corr_iv:
1065
- # FIX-35: Don't strip valid conjugation suffixes.
1066
- # If the "extra char" is at the END and is a common
1067
- # Arabic verb/noun suffix, this is likely a grammar
1068
- # conjugation, not a typo. Skip it.
1069
- _CONJUGATION_SUFFIXES = {'ن', 'ت', 'ا', 'ي', 'ة', 'و', 'ه'}
 
 
1070
  _removed_char = None
1071
  for _di2 in range(len(orig_word)):
1072
  if orig_word[:_di2] + orig_word[_di2 + 1:] == corr_word:
@@ -2314,12 +2316,14 @@ def analyze_text():
2314
  # The punctuation model often fails to add a period at the end
2315
  # of longer sentences. If no terminal punctuation exists after
2316
  # model processing, inject a period suggestion for the last word.
 
 
2317
  import re as _re_punc
2318
  _TERMINAL_PUNCT = set('.،؛؟!?!')
2319
  _current_stripped = ctx.current_text.rstrip()
2320
  _has_terminal = _current_stripped and _current_stripped[-1] in _TERMINAL_PUNCT
2321
  _word_count_fb = len(_re_punc.findall(r'[\u0600-\u06FFa-zA-Z]+', ctx.current_text))
2322
- if not _has_terminal and _word_count_fb >= 2:
2323
  # Find the last word's position in current_text
2324
  _last_word_match = _re_punc.search(r'([\u0600-\u06FF]+)\s*$', _current_stripped)
2325
  if _last_word_match:
 
1062
  _orig_oov = not vocab_manager.is_iv(orig_word)
1063
  _corr_iv = vocab_manager.is_iv(corr_word)
1064
  if _orig_oov and _corr_iv:
1065
+ # FIX-35: Don't strip verb conjugation suffixes.
1066
+ # Only block ن (feminine plural: ذهبن→ذهب) and
1067
+ # ت (feminine past: كتبت→كتب) these are the
1068
+ # suffixes grammar commonly adds that spelling
1069
+ # would try to strip. Other endings (ة,ا,ي,و,ه)
1070
+ # are more likely genuine typos than grammar fixes.
1071
+ _CONJUGATION_SUFFIXES = {'ن', 'ت'}
1072
  _removed_char = None
1073
  for _di2 in range(len(orig_word)):
1074
  if orig_word[:_di2] + orig_word[_di2 + 1:] == corr_word:
 
2316
  # The punctuation model often fails to add a period at the end
2317
  # of longer sentences. If no terminal punctuation exists after
2318
  # model processing, inject a period suggestion for the last word.
2319
+ # Threshold=4 words to avoid noisy suggestions while user is
2320
+ # still typing short phrases.
2321
  import re as _re_punc
2322
  _TERMINAL_PUNCT = set('.،؛؟!?!')
2323
  _current_stripped = ctx.current_text.rstrip()
2324
  _has_terminal = _current_stripped and _current_stripped[-1] in _TERMINAL_PUNCT
2325
  _word_count_fb = len(_re_punc.findall(r'[\u0600-\u06FFa-zA-Z]+', ctx.current_text))
2326
+ if not _has_terminal and _word_count_fb >= 4:
2327
  # Find the last word's position in current_text
2328
  _last_word_match = _re_punc.search(r'([\u0600-\u06FF]+)\s*$', _current_stripped)
2329
  if _last_word_match:
src/js/editor.js CHANGED
@@ -689,10 +689,6 @@ function applyAllSuggestions() {
689
  // Reverse-order text replacement is safe — each patch's offsets are
690
  // valid because we haven't modified anything before them yet.
691
  let text = getEditorText();
692
- // FIX-32b: MUST sort by descending offset — replacing from end-to-start
693
- // ensures earlier offsets remain valid. Without this, applying a fix at
694
- // offset 5 shifts everything after it, corrupting offset 20's fix.
695
- suggestions.sort((a, b) => b.start - a.start);
696
  suggestions.forEach((s) => {
697
  if (s.start >= 0 && s.end <= text.length && s.start <= s.end) {
698
  text = text.substring(0, s.start) + s.correction + text.substring(s.end);
 
689
  // Reverse-order text replacement is safe — each patch's offsets are
690
  // valid because we haven't modified anything before them yet.
691
  let text = getEditorText();
 
 
 
 
692
  suggestions.forEach((s) => {
693
  if (s.start >= 0 && s.end <= text.length && s.start <= s.end) {
694
  text = text.substring(0, s.start) + s.correction + text.substring(s.end);
tests/test_recent_fixes.py CHANGED
@@ -86,7 +86,7 @@ for input_text, expected in plurals:
86
  # ══════════════════════════════════════════════════════════════
87
  print("\n═══ FIX-35: Spelling — conjugation suffix protection ═══")
88
 
89
- _CONJUGATION_SUFFIXES = {'ن', 'ت', 'ا', 'ي', 'ة', 'و', 'ه'}
90
 
91
  def simulate_insertion_fix_check(orig_word, corr_word):
92
  """Simulate the FIX-35 suffix strip check logic from app.py."""
@@ -105,7 +105,7 @@ def simulate_insertion_fix_check(orig_word, corr_word):
105
  return "allowed"
106
  return "not_applicable"
107
 
108
- # BLOCKED cases (suffix stripping)
109
  test("'ذهبن'→'ذهب' blocked (ن suffix)",
110
  simulate_insertion_fix_check("ذهبن", "ذهب") == "blocked",
111
  f"got {simulate_insertion_fix_check('ذهبن', 'ذهب')}")
@@ -114,15 +114,16 @@ test("'كتبت'→'كتب' blocked (ت suffix)",
114
  simulate_insertion_fix_check("كتبت", "كتب") == "blocked",
115
  f"got {simulate_insertion_fix_check('كتبت', 'كتب')}")
116
 
117
- test("'درسة'→'درس' blockedsuffix)",
118
- simulate_insertion_fix_check("درسة", "درس") == "blocked",
 
119
  f"got {simulate_insertion_fix_check('درسة', 'درس')}")
120
 
121
- test("'جلسوا'→'جلسو' not applicable (len diff ok but وا→و)",
122
- simulate_insertion_fix_check("جلسوا", "جلسو") == "blocked",
123
  f"got {simulate_insertion_fix_check('جلسوا', 'جلسو')}")
124
 
125
- # ALLOWED cases (mid-word insertion fix)
126
  test("'الكتتاب'→'الكتاب' allowed (mid-word extra ت)",
127
  simulate_insertion_fix_check("الكتتاب", "الكتاب") == "allowed",
128
  f"got {simulate_insertion_fix_check('الكتتاب', 'الكتاب')}")
 
86
  # ══════════════════════════════════════════════════════════════
87
  print("\n═══ FIX-35: Spelling — conjugation suffix protection ═══")
88
 
89
+ _CONJUGATION_SUFFIXES = {'ن', 'ت'}
90
 
91
  def simulate_insertion_fix_check(orig_word, corr_word):
92
  """Simulate the FIX-35 suffix strip check logic from app.py."""
 
105
  return "allowed"
106
  return "not_applicable"
107
 
108
+ # BLOCKED cases (verb conjugation suffix stripping)
109
  test("'ذهبن'→'ذهب' blocked (ن suffix)",
110
  simulate_insertion_fix_check("ذهبن", "ذهب") == "blocked",
111
  f"got {simulate_insertion_fix_check('ذهبن', 'ذهب')}")
 
114
  simulate_insertion_fix_check("كتبت", "كتب") == "blocked",
115
  f"got {simulate_insertion_fix_check('كتبت', 'كتب')}")
116
 
117
+ # NOW ALLOWED cases (narrowed set — ة, و no longer blocked)
118
+ test("'درسة'→'درس' allowed (ة not in narrowed set)",
119
+ simulate_insertion_fix_check("درسة", "درس") == "allowed",
120
  f"got {simulate_insertion_fix_check('درسة', 'درس')}")
121
 
122
+ test("'جلسوا'→'جلسو' allowed (ا not in narrowed set)",
123
+ simulate_insertion_fix_check("جلسوا", "جلسو") == "allowed",
124
  f"got {simulate_insertion_fix_check('جلسوا', 'جلسو')}")
125
 
126
+ # ALLOWED cases (mid-word insertion fix — always allowed)
127
  test("'الكتتاب'→'الكتاب' allowed (mid-word extra ت)",
128
  simulate_insertion_fix_check("الكتتاب", "الكتاب") == "allowed",
129
  f"got {simulate_insertion_fix_check('الكتتاب', 'الكتاب')}")