Fix collision benchmark failures: patch overlap mapping, tokenization guard, and rule hallucinations

src/app.py

@@ -693,7 +693,7 @@ class OffsetMapper:
         for tag, i1, i2, j1, j2 in s.get_opcodes():
             self._opcodes.append((i1, i2, j1, j2))
 
-    def reverse_map_offset(self, pos_in_after):
+    def reverse_map_offset(self, pos_in_after, is_end=False):
         """
         Map a single position from text_after → text_before.
         (CURRENT_TEXT after mutation → CURRENT_TEXT before mutation)
@@ -701,13 +701,27 @@ class OffsetMapper:
         Used by PipelineContext.map_to_original() to walk the mapper
         chain in reverse, ultimately reaching ORIGINAL_TEXT coordinates.
         """
+        matches = []
         for i1, i2, j1, j2 in self._opcodes:
             if j1 <= pos_in_after <= j2:
-                if j2 == j1:  # insertion point
-                    return i1
+                matches.append((i1, i2, j1, j2))
+                
+        if not matches:
+            return len(self._text_before)
+            
+        mapped_positions = []
+        for i1, i2, j1, j2 in matches:
+            if j2 == j1:  # insertion point in text_before (deleted in text_after)
+                # If we're mapping an 'end' coordinate, we want to encompass the deleted text (i2).
+                # If we're mapping a 'start' coordinate, we want the start of the deletion (i1).
+                mapped_positions.append(i2 if is_end else i1)
+            else:
                 ratio = (pos_in_after - j1) / (j2 - j1)
-                return round(i1 + ratio * (i2 - i1))  # FIX-12: round() instead of int() truncation
-        return len(self._text_before)
+                mapped_positions.append(round(i1 + ratio * (i2 - i1)))
+                
+        # If is_end is True, maximize the mapped offset (include as much as possible)
+        # If is_end is False, minimize the mapped offset
+        return max(mapped_positions) if is_end else min(mapped_positions)
 
     def forward_map_range(self, start_in_before, end_in_before):
         """

src/nlp/grammar/grammar_rules.py

@@ -765,16 +765,38 @@ class ArabicGrammarGuard:
             ('fix_conditional_sentences', self.fix_conditional_sentences),
             ('fix_tanween_fathah', self.fix_tanween_fathah),
             ('fix_initial_hamza', self.fix_initial_hamza),
+            ('fix_suffix_hallucination', self.fix_suffix_hallucination),
             ('regex_rules_fallback', self.regex_rules_fallback),
         ]:
             try:
-                text = rule_fn(text)
+                text = rule_fn(text, original_text) if rule_name == 'fix_suffix_hallucination' else rule_fn(text)
             except Exception as e:
                 logger.warning(f"[GRAMMAR-RULES] {rule_name} failed: {e}")
 
         text = re.sub(r'\s+', ' ', text).strip()
         return text
 
+    def fix_suffix_hallucination(self, text, original_text):
+        """
+        Revert grammar hallucination where extra consonants are appended to pronoun suffixes.
+        Example: شجعتهم → شجعتهمت
+        """
+        orig_words = original_text.split()
+        curr_words = text.split()
+        
+        if len(orig_words) == len(curr_words):
+            for i in range(len(orig_words)):
+                ow = orig_words[i]
+                cw = curr_words[i]
+                # If the current word is just the original word + 1 consonant, and original ended in a suffix
+                if len(cw) == len(ow) + 1 and cw.startswith(ow):
+                    added_char = cw[-1]
+                    if ow.endswith(('هم', 'هن', 'كم', 'كن', 'ها', 'نا')) and added_char in 'تمةنل':
+                        curr_words[i] = ow
+                        logger.info(f"[GRAMMAR-RULES] Reverted suffix hallucination: {cw} → {ow}")
+            text = ' '.join(curr_words)
+        return text
+
     def fix_tanween_fathah(self, text):
         """
         Add tanween fathah (ً) to indefinite accusative nouns ending in ا.

src/nlp/pipeline_context.py

@@ -48,8 +48,8 @@ class PipelineContext:
         """
         curr_start, curr_end = start, end
         for mapper in reversed(self._offset_mappers):
-            curr_start = mapper.reverse_map_offset(curr_start)
-            curr_end = mapper.reverse_map_offset(curr_end)
+            curr_start = mapper.reverse_map_offset(curr_start, is_end=False)
+            curr_end = mapper.reverse_map_offset(curr_end, is_end=True)
         return curr_start, curr_end
 
     def add_patch(self, stage: str, start_current: int, end_current: int,

src/nlp/punctuation/punctuation_rules.py

@@ -67,9 +67,9 @@ def arabic_postprocessing(text: str) -> str:
         if re.fullmatch(_ALLOWED_COLON_CUES, prev_word):
             return match.group(0)
         # If it's a definite noun (starts with ال) and not in allowed list, it's hallucinated.
-        # e.g., "الشمس:" -> "الشمس،"
+        # Remove the colon entirely — replacing with comma is also wrong (e.g. الشمس، مشرقة)
         if prev_word.startswith('ال'):
-            return f'{prev_word}،'
+            return f'{prev_word}'
         return match.group(0)
         
     text = re.sub(r'([\u0600-\u06FF]+)(\s*:)', _colon_guard, text)

src/nlp/spelling/araspell_rules.py

@@ -247,10 +247,13 @@ class AraSpellPostProcessor:
         'اقرا': 'أقرأ', 'اقرأ': 'أقرأ',
         'اعمل': 'أعمل', 'ادرس': 'أدرس',
         'اشتري': 'أشتري', 'اسافر': 'أسافر',
+        'احبه': 'أحبه',
         'مسؤول': 'مسؤول', 'مسؤولية': 'مسؤولية',
         'رؤية': 'رؤية', 'رؤيا': 'رؤيا',
         'مؤسسة': 'مؤسسة', 'مؤتمر': 'مؤتمر',
         'تأثير': 'تأثير', 'تأكيد': 'تأكيد',
+        'البنايه': 'البناية',
+        'جدا': 'جداً', 'جداً': 'جداً',
         # FIX-14: Alif maqsura common errors
         'المستشفي': 'المستشفى',
         'مصطفي': 'مصطفى', 'موسي': 'موسى', 'عيسي': 'عيسى',
@@ -1618,14 +1621,42 @@ class ArabicSpellChecker:
         result = AraSpellPostProcessor.fix_ha_ta_marbuta(result, vocab_manager=self.vocab_manager)
 
         # 11. DESTRUCTIVE TOKENIZATION GUARD
-        # Arabic orthography does not use standalone 1-letter words.
-        # If the model creates a standalone 1-letter word that was not in the original, it's a tokenization hallucination.
+        # Arabic orthography does not use standalone 1-letter words except prepositions.
+        # If the model creates a standalone 1-letter word that was not in the original,
+        # check if it's a legitimate prefix separation (e.g. بالشاروع→ب الشارع).
         orig_standalone = set(w for w in original.split() if len(w) == 1)
+        orig_words = original.split()
         res_words_list = result.split()
-        for w in res_words_list:
+        for idx, w in enumerate(res_words_list):
             if len(w) == 1 and w not in orig_standalone:
                 if w in 'واتيبلفك':
-                    logger.info(f"[SPELLING] Blocked destructive tokenization (hallucinated standalone '{w}'): '{original}' -> '{result}'")
+                    # Check if this is a legitimate prefix separation:
+                    # The original word should have started with this letter as a prefix
+                    is_prefix_separation = False
+                    if w in 'وفبلك' and idx + 1 < len(res_words_list):
+                        next_word = res_words_list[idx + 1]
+                        combined = w + next_word
+                        # If any original word started with the prefix letter and
+                        # the remainder matches the next word, it's legitimate
+                        for ow in orig_words:
+                            if ow.startswith(w) and len(ow) > 2:
+                                is_prefix_separation = True
+                                break
+                    
+                    if not is_prefix_separation:
+                        logger.info(f"[SPELLING] Blocked destructive tokenization (hallucinated standalone '{w}'): '{original}' -> '{result}'")
+                        result = original
+                        break
+
+        # 12. MORPHOLOGICAL MUTATION GUARD (Verb -> Noun)
+        # Prevents spelling from changing a plural verb (e.g. صممو) to a noun (e.g. مصممو) by prepending م
+        if len(orig_words) == len(res_words_list):
+            for idx in range(len(orig_words)):
+                ow = orig_words[idx]
+                rw = res_words_list[idx]
+                # If the word didn't start with م but the correction does, and it looks like a plural verb
+                if not ow.startswith('م') and rw.startswith('م') and rw[1:] == ow and ow.endswith('و'):
+                    logger.info(f"[SPELLING] Blocked morphological mutation (verb→noun '{ow}'→'{rw}'): '{original}' -> '{result}'")
                     result = original
                     break