youssefreda9 commited on
Commit
1e3f46f
·
1 Parent(s): 8fe6a99

Phase 14b: Fix spelling punct hallucination + cross-stage overlap

Browse files

FIX-31: Strip trailing punctuation (... . etc) hallucinated by the
AraSpell model that wasn't in the input. This prevents dot
accumulation across pipeline runs.

PatchSet: Spelling + Punctuation patches from different stages now
always coexist (they're compatible corrections). Previously,
punctuation's higher priority caused spelling patches on
adjacent/overlapping words to be dropped ('فيالمشروع' phantom).

Files changed (2) hide show
  1. src/app.py +21 -0
  2. src/nlp/correction_patch.py +18 -7
src/app.py CHANGED
@@ -1528,6 +1528,27 @@ def analyze_text():
1528
  timing_ms['spelling_ms'] = int((time.time() - t0) * 1000)
1529
  logger.info(f"[ANALYZE] Step 1: Spelling done in {timing_ms['spelling_ms']}ms")
1530
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1531
  # ── Phase 12 (A4): Output Stability Test ──
1532
  # If re-preprocessing the correction changes it significantly,
1533
  # the correction is unstable → fall back to re-preprocessed version.
 
1528
  timing_ms['spelling_ms'] = int((time.time() - t0) * 1000)
1529
  logger.info(f"[ANALYZE] Step 1: Spelling done in {timing_ms['spelling_ms']}ms")
1530
 
1531
+ # ── Phase 14 (FIX-31): Strip hallucinated trailing punctuation ──
1532
+ # The AraSpell model sometimes hallucinates trailing '...' or '.'
1533
+ # that weren't in the input. Strip them to prevent dot accumulation.
1534
+ import re as _re_strip
1535
+ _input_trailing = _re_strip.search(r'[\.،؛؟!]+$', current_text)
1536
+ _output_trailing = _re_strip.search(r'[\.،؛؟!]+$', raw_corrected)
1537
+ if _output_trailing and not _input_trailing:
1538
+ raw_corrected = raw_corrected[:_output_trailing.start()]
1539
+ logger.info(
1540
+ f"[SPELLING] Stripped hallucinated trailing punct: "
1541
+ f"'{_output_trailing.group()}'"
1542
+ )
1543
+ elif _output_trailing and _input_trailing:
1544
+ # If input had some trailing punct, preserve only what was there
1545
+ if len(_output_trailing.group()) > len(_input_trailing.group()):
1546
+ raw_corrected = raw_corrected[:_output_trailing.start()] + _input_trailing.group()
1547
+ logger.info(
1548
+ f"[SPELLING] Trimmed extra trailing punct: "
1549
+ f"'{_output_trailing.group()}' → '{_input_trailing.group()}'"
1550
+ )
1551
+
1552
  # ── Phase 12 (A4): Output Stability Test ──
1553
  # If re-preprocessing the correction changes it significantly,
1554
  # the correction is unstable → fall back to re-preprocessed version.
src/nlp/correction_patch.py CHANGED
@@ -97,24 +97,35 @@ class PatchSet:
97
  Single owner per range. Deterministic resolution.
98
  Uses ORIGINAL coordinates for overlap detection.
99
 
100
- Phase 14: Relaxed overlap — patches that touch at a boundary or
101
- have minimal overlap (<= 50% of the smaller patch) are allowed
102
- to coexist. This prevents punctuation patches from killing spelling
103
- patches on adjacent words (e.g., 'المشروع.' punct vs 'في' spelling).
 
104
  """
105
  sorted_patches = sorted(
106
  self.patches,
107
  key=lambda p: (-p.priority, -p.confidence, p.start_original, p.id)
108
  )
109
 
110
- claimed_ranges = []
111
  resolved = []
112
 
113
  for patch in sorted_patches:
114
  has_substantial_overlap = False
115
- for cs, ce in claimed_ranges:
116
  # Check if there's any overlap at all
117
  if patch.start_original < ce and patch.end_original > cs:
 
 
 
 
 
 
 
 
 
 
118
  # Calculate overlap amount
119
  overlap_start = max(patch.start_original, cs)
120
  overlap_end = min(patch.end_original, ce)
@@ -130,7 +141,7 @@ class PatchSet:
130
 
131
  if not has_substantial_overlap:
132
  resolved.append(patch)
133
- claimed_ranges.append((patch.start_original, patch.end_original))
134
  else:
135
  logger.info(
136
  f"[OVERLAP] Dropped {patch.stage} [{patch.start_original}:{patch.end_original}] "
 
97
  Single owner per range. Deterministic resolution.
98
  Uses ORIGINAL coordinates for overlap detection.
99
 
100
+ Phase 14: Relaxed overlap rules:
101
+ 1. Patches with < 50% overlap of the smaller patch coexist freely
102
+ 2. Spelling + Punctuation patches from different stages always coexist
103
+ (they're compatible: one fixes the word, the other adds punct)
104
+ 3. Same-stage overlaps are always resolved (higher confidence wins)
105
  """
106
  sorted_patches = sorted(
107
  self.patches,
108
  key=lambda p: (-p.priority, -p.confidence, p.start_original, p.id)
109
  )
110
 
111
+ claimed_ranges = [] # list of (start, end, stage)
112
  resolved = []
113
 
114
  for patch in sorted_patches:
115
  has_substantial_overlap = False
116
+ for cs, ce, claimed_stage in claimed_ranges:
117
  # Check if there's any overlap at all
118
  if patch.start_original < ce and patch.end_original > cs:
119
+ # ── Phase 14: Cross-stage compatibility ──
120
+ # Spelling + Punctuation are COMPATIBLE stages:
121
+ # spelling fixes the word, punctuation adds marks.
122
+ # They should never conflict.
123
+ _compatible_pair = {
124
+ frozenset({'spelling', 'punctuation'}),
125
+ }
126
+ if frozenset({patch.stage, claimed_stage}) in _compatible_pair:
127
+ continue # Compatible stages — allow coexistence
128
+
129
  # Calculate overlap amount
130
  overlap_start = max(patch.start_original, cs)
131
  overlap_end = min(patch.end_original, ce)
 
141
 
142
  if not has_substantial_overlap:
143
  resolved.append(patch)
144
+ claimed_ranges.append((patch.start_original, patch.end_original, patch.stage))
145
  else:
146
  logger.info(
147
  f"[OVERLAP] Dropped {patch.stage} [{patch.start_original}:{patch.end_original}] "