Commit ·
1e3f46f
1
Parent(s): 8fe6a99
Phase 14b: Fix spelling punct hallucination + cross-stage overlap
Browse filesFIX-31: Strip trailing punctuation (... . etc) hallucinated by the
AraSpell model that wasn't in the input. This prevents dot
accumulation across pipeline runs.
PatchSet: Spelling + Punctuation patches from different stages now
always coexist (they're compatible corrections). Previously,
punctuation's higher priority caused spelling patches on
adjacent/overlapping words to be dropped ('فيالمشروع' phantom).
- src/app.py +21 -0
- src/nlp/correction_patch.py +18 -7
src/app.py
CHANGED
|
@@ -1528,6 +1528,27 @@ def analyze_text():
|
|
| 1528 |
timing_ms['spelling_ms'] = int((time.time() - t0) * 1000)
|
| 1529 |
logger.info(f"[ANALYZE] Step 1: Spelling done in {timing_ms['spelling_ms']}ms")
|
| 1530 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1531 |
# ── Phase 12 (A4): Output Stability Test ──
|
| 1532 |
# If re-preprocessing the correction changes it significantly,
|
| 1533 |
# the correction is unstable → fall back to re-preprocessed version.
|
|
|
|
| 1528 |
timing_ms['spelling_ms'] = int((time.time() - t0) * 1000)
|
| 1529 |
logger.info(f"[ANALYZE] Step 1: Spelling done in {timing_ms['spelling_ms']}ms")
|
| 1530 |
|
| 1531 |
+
# ── Phase 14 (FIX-31): Strip hallucinated trailing punctuation ──
|
| 1532 |
+
# The AraSpell model sometimes hallucinates trailing '...' or '.'
|
| 1533 |
+
# that weren't in the input. Strip them to prevent dot accumulation.
|
| 1534 |
+
import re as _re_strip
|
| 1535 |
+
_input_trailing = _re_strip.search(r'[\.،؛؟!]+$', current_text)
|
| 1536 |
+
_output_trailing = _re_strip.search(r'[\.،؛؟!]+$', raw_corrected)
|
| 1537 |
+
if _output_trailing and not _input_trailing:
|
| 1538 |
+
raw_corrected = raw_corrected[:_output_trailing.start()]
|
| 1539 |
+
logger.info(
|
| 1540 |
+
f"[SPELLING] Stripped hallucinated trailing punct: "
|
| 1541 |
+
f"'{_output_trailing.group()}'"
|
| 1542 |
+
)
|
| 1543 |
+
elif _output_trailing and _input_trailing:
|
| 1544 |
+
# If input had some trailing punct, preserve only what was there
|
| 1545 |
+
if len(_output_trailing.group()) > len(_input_trailing.group()):
|
| 1546 |
+
raw_corrected = raw_corrected[:_output_trailing.start()] + _input_trailing.group()
|
| 1547 |
+
logger.info(
|
| 1548 |
+
f"[SPELLING] Trimmed extra trailing punct: "
|
| 1549 |
+
f"'{_output_trailing.group()}' → '{_input_trailing.group()}'"
|
| 1550 |
+
)
|
| 1551 |
+
|
| 1552 |
# ── Phase 12 (A4): Output Stability Test ──
|
| 1553 |
# If re-preprocessing the correction changes it significantly,
|
| 1554 |
# the correction is unstable → fall back to re-preprocessed version.
|
src/nlp/correction_patch.py
CHANGED
|
@@ -97,24 +97,35 @@ class PatchSet:
|
|
| 97 |
Single owner per range. Deterministic resolution.
|
| 98 |
Uses ORIGINAL coordinates for overlap detection.
|
| 99 |
|
| 100 |
-
Phase 14: Relaxed overlap
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
|
|
|
| 104 |
"""
|
| 105 |
sorted_patches = sorted(
|
| 106 |
self.patches,
|
| 107 |
key=lambda p: (-p.priority, -p.confidence, p.start_original, p.id)
|
| 108 |
)
|
| 109 |
|
| 110 |
-
claimed_ranges = []
|
| 111 |
resolved = []
|
| 112 |
|
| 113 |
for patch in sorted_patches:
|
| 114 |
has_substantial_overlap = False
|
| 115 |
-
for cs, ce in claimed_ranges:
|
| 116 |
# Check if there's any overlap at all
|
| 117 |
if patch.start_original < ce and patch.end_original > cs:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
# Calculate overlap amount
|
| 119 |
overlap_start = max(patch.start_original, cs)
|
| 120 |
overlap_end = min(patch.end_original, ce)
|
|
@@ -130,7 +141,7 @@ class PatchSet:
|
|
| 130 |
|
| 131 |
if not has_substantial_overlap:
|
| 132 |
resolved.append(patch)
|
| 133 |
-
claimed_ranges.append((patch.start_original, patch.end_original))
|
| 134 |
else:
|
| 135 |
logger.info(
|
| 136 |
f"[OVERLAP] Dropped {patch.stage} [{patch.start_original}:{patch.end_original}] "
|
|
|
|
| 97 |
Single owner per range. Deterministic resolution.
|
| 98 |
Uses ORIGINAL coordinates for overlap detection.
|
| 99 |
|
| 100 |
+
Phase 14: Relaxed overlap rules:
|
| 101 |
+
1. Patches with < 50% overlap of the smaller patch coexist freely
|
| 102 |
+
2. Spelling + Punctuation patches from different stages always coexist
|
| 103 |
+
(they're compatible: one fixes the word, the other adds punct)
|
| 104 |
+
3. Same-stage overlaps are always resolved (higher confidence wins)
|
| 105 |
"""
|
| 106 |
sorted_patches = sorted(
|
| 107 |
self.patches,
|
| 108 |
key=lambda p: (-p.priority, -p.confidence, p.start_original, p.id)
|
| 109 |
)
|
| 110 |
|
| 111 |
+
claimed_ranges = [] # list of (start, end, stage)
|
| 112 |
resolved = []
|
| 113 |
|
| 114 |
for patch in sorted_patches:
|
| 115 |
has_substantial_overlap = False
|
| 116 |
+
for cs, ce, claimed_stage in claimed_ranges:
|
| 117 |
# Check if there's any overlap at all
|
| 118 |
if patch.start_original < ce and patch.end_original > cs:
|
| 119 |
+
# ── Phase 14: Cross-stage compatibility ──
|
| 120 |
+
# Spelling + Punctuation are COMPATIBLE stages:
|
| 121 |
+
# spelling fixes the word, punctuation adds marks.
|
| 122 |
+
# They should never conflict.
|
| 123 |
+
_compatible_pair = {
|
| 124 |
+
frozenset({'spelling', 'punctuation'}),
|
| 125 |
+
}
|
| 126 |
+
if frozenset({patch.stage, claimed_stage}) in _compatible_pair:
|
| 127 |
+
continue # Compatible stages — allow coexistence
|
| 128 |
+
|
| 129 |
# Calculate overlap amount
|
| 130 |
overlap_start = max(patch.start_original, cs)
|
| 131 |
overlap_end = min(patch.end_original, ce)
|
|
|
|
| 141 |
|
| 142 |
if not has_substantial_overlap:
|
| 143 |
resolved.append(patch)
|
| 144 |
+
claimed_ranges.append((patch.start_original, patch.end_original, patch.stage))
|
| 145 |
else:
|
| 146 |
logger.info(
|
| 147 |
f"[OVERLAP] Dropped {patch.stage} [{patch.start_original}:{patch.end_original}] "
|