Commit ·
920c24b
1
Parent(s): 00d416d
feat: NLP-3.5 Hardening Sprint — Performance + Overlap Resolver TASK 2: Per-stage timing instrumentation (spelling_ms, grammar_ms, punctuation_ms, total_ms) TASK 3: Smart text processing — skip AraSpell for texts > 300 chars TASK 4+5: Global overlap resolver with priority system grammar(3) > punctuation(2) > spelling(1) > autocomplete(0) Full span collision detection (exact + partial overlaps) TASK 6: AutoComplete priority hook prepared (priority=0)
Browse files- src/app.py +63 -23
src/app.py
CHANGED
|
@@ -718,6 +718,7 @@ def analyze_text():
|
|
| 718 |
suggestions = []
|
| 719 |
mappers = []
|
| 720 |
total_start = time.time()
|
|
|
|
| 721 |
|
| 722 |
def map_range_to_original(start, end):
|
| 723 |
curr_start, curr_end = start, end
|
|
@@ -749,16 +750,25 @@ def analyze_text():
|
|
| 749 |
result = [best_correction] + alts + [original_word]
|
| 750 |
return result[:max_alts + 1] # cap at max_alts + keep-as-is
|
| 751 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 752 |
# 1. Spelling (with conservative post-filtering to avoid over-editing)
|
| 753 |
-
|
| 754 |
-
if has_spelling:
|
| 755 |
try:
|
| 756 |
t0 = time.time()
|
| 757 |
logger.info(f"[ANALYZE] Step 1: Spelling correction starting...")
|
| 758 |
from nlp.spelling.araspell_service import get_spelling_model
|
| 759 |
spell_checker = get_spelling_model()
|
| 760 |
raw_corrected = spell_checker.correct(current_text)
|
| 761 |
-
|
|
|
|
| 762 |
|
| 763 |
if raw_corrected != current_text:
|
| 764 |
orig_word_positions = get_word_positions(current_text)
|
|
@@ -890,7 +900,8 @@ def analyze_text():
|
|
| 890 |
from nlp.grammar.grammar_service import get_grammar_model
|
| 891 |
grammar_checker = get_grammar_model()
|
| 892 |
corrected_grammar = grammar_checker.correct(current_text)
|
| 893 |
-
|
|
|
|
| 894 |
if corrected_grammar != current_text:
|
| 895 |
diffs = get_word_diffs(current_text, corrected_grammar)
|
| 896 |
for d in diffs:
|
|
@@ -914,7 +925,8 @@ def analyze_text():
|
|
| 914 |
from nlp.punctuation.punctuation_service import get_punctuation_model
|
| 915 |
punc_checker = get_punctuation_model()
|
| 916 |
corrected_punc = punc_checker.correct(current_text)
|
| 917 |
-
|
|
|
|
| 918 |
if corrected_punc != current_text:
|
| 919 |
diffs = get_word_diffs(current_text, corrected_punc)
|
| 920 |
for d in diffs:
|
|
@@ -932,32 +944,60 @@ def analyze_text():
|
|
| 932 |
logger.error(f"[ANALYZE] Punctuation failed: {e}")
|
| 933 |
|
| 934 |
total_time = time.time() - total_start
|
|
|
|
| 935 |
|
| 936 |
-
#
|
| 937 |
-
#
|
| 938 |
-
#
|
| 939 |
-
# grammar
|
| 940 |
-
|
| 941 |
-
|
| 942 |
-
|
| 943 |
-
|
|
|
|
| 944 |
|
| 945 |
-
|
| 946 |
-
|
| 947 |
-
suggestions = [
|
| 948 |
-
s for s in suggestions
|
| 949 |
-
if s['type'] != 'spelling' or (s['start'], s['end']) not in grammar_positions
|
| 950 |
-
]
|
| 951 |
-
removed = before_count - len(suggestions)
|
| 952 |
-
if removed > 0:
|
| 953 |
-
logger.info(f"[ANALYZE] Removed {removed} spelling suggestions that overlap with grammar")
|
| 954 |
|
| 955 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 956 |
|
| 957 |
return jsonify({
|
| 958 |
'original': text,
|
| 959 |
'corrected': current_text,
|
| 960 |
'suggestions': suggestions,
|
|
|
|
| 961 |
'status': 'success'
|
| 962 |
})
|
| 963 |
|
|
|
|
| 718 |
suggestions = []
|
| 719 |
mappers = []
|
| 720 |
total_start = time.time()
|
| 721 |
+
timing_ms = {'spelling_ms': 0, 'grammar_ms': 0, 'punctuation_ms': 0, 'total_ms': 0}
|
| 722 |
|
| 723 |
def map_range_to_original(start, end):
|
| 724 |
curr_start, curr_end = start, end
|
|
|
|
| 750 |
result = [best_correction] + alts + [original_word]
|
| 751 |
return result[:max_alts + 1] # cap at max_alts + keep-as-is
|
| 752 |
|
| 753 |
+
# ── Smart Text Processing Strategy ──
|
| 754 |
+
# Short (0-300 chars): full pipeline (Spelling + Grammar + Punctuation)
|
| 755 |
+
# Medium (300-1000 chars): Grammar + Punctuation only (skip AraSpell)
|
| 756 |
+
# Large (1000+ chars): Grammar + Punctuation only
|
| 757 |
+
text_len = len(current_text)
|
| 758 |
+
run_spelling = text_len <= 300
|
| 759 |
+
if not run_spelling:
|
| 760 |
+
logger.info(f"[ANALYZE] Text length {text_len} > 300 — skipping AraSpell for performance")
|
| 761 |
+
|
| 762 |
# 1. Spelling (with conservative post-filtering to avoid over-editing)
|
| 763 |
+
if run_spelling:
|
|
|
|
| 764 |
try:
|
| 765 |
t0 = time.time()
|
| 766 |
logger.info(f"[ANALYZE] Step 1: Spelling correction starting...")
|
| 767 |
from nlp.spelling.araspell_service import get_spelling_model
|
| 768 |
spell_checker = get_spelling_model()
|
| 769 |
raw_corrected = spell_checker.correct(current_text)
|
| 770 |
+
timing_ms['spelling_ms'] = int((time.time() - t0) * 1000)
|
| 771 |
+
logger.info(f"[ANALYZE] Step 1: Spelling done in {timing_ms['spelling_ms']}ms")
|
| 772 |
|
| 773 |
if raw_corrected != current_text:
|
| 774 |
orig_word_positions = get_word_positions(current_text)
|
|
|
|
| 900 |
from nlp.grammar.grammar_service import get_grammar_model
|
| 901 |
grammar_checker = get_grammar_model()
|
| 902 |
corrected_grammar = grammar_checker.correct(current_text)
|
| 903 |
+
timing_ms['grammar_ms'] = int((time.time() - t0) * 1000)
|
| 904 |
+
logger.info(f"[ANALYZE] Step 2: Grammar done in {timing_ms['grammar_ms']}ms")
|
| 905 |
if corrected_grammar != current_text:
|
| 906 |
diffs = get_word_diffs(current_text, corrected_grammar)
|
| 907 |
for d in diffs:
|
|
|
|
| 925 |
from nlp.punctuation.punctuation_service import get_punctuation_model
|
| 926 |
punc_checker = get_punctuation_model()
|
| 927 |
corrected_punc = punc_checker.correct(current_text)
|
| 928 |
+
timing_ms['punctuation_ms'] = int((time.time() - t0) * 1000)
|
| 929 |
+
logger.info(f"[ANALYZE] Step 3: Punctuation done in {timing_ms['punctuation_ms']}ms")
|
| 930 |
if corrected_punc != current_text:
|
| 931 |
diffs = get_word_diffs(current_text, corrected_punc)
|
| 932 |
for d in diffs:
|
|
|
|
| 944 |
logger.error(f"[ANALYZE] Punctuation failed: {e}")
|
| 945 |
|
| 946 |
total_time = time.time() - total_start
|
| 947 |
+
timing_ms['total_ms'] = int(total_time * 1000)
|
| 948 |
|
| 949 |
+
# ══════════════════════════════════════════════════════════
|
| 950 |
+
# GLOBAL OVERLAP RESOLVER — NLP-3.5 Hardening
|
| 951 |
+
# ══════════════════════════════════════════════════════════
|
| 952 |
+
# Priority: grammar(3) > punctuation(2) > spelling(1) > autocomplete(0)
|
| 953 |
+
# Rules:
|
| 954 |
+
# - Higher priority ALWAYS wins
|
| 955 |
+
# - One span = one highlight (no stacking)
|
| 956 |
+
# - Partial overlaps: higher priority span kept, lower dropped
|
| 957 |
+
PRIORITY = {'grammar': 3, 'punctuation': 2, 'spelling': 1, 'autocomplete': 0}
|
| 958 |
|
| 959 |
+
# Sort suggestions by priority (highest first)
|
| 960 |
+
suggestions.sort(key=lambda s: PRIORITY.get(s['type'], 0), reverse=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 961 |
|
| 962 |
+
# Build a set of "claimed" character positions
|
| 963 |
+
claimed_ranges = [] # list of (start, end, type)
|
| 964 |
+
resolved = []
|
| 965 |
+
|
| 966 |
+
for s in suggestions:
|
| 967 |
+
s_start, s_end = s['start'], s['end']
|
| 968 |
+
s_priority = PRIORITY.get(s['type'], 0)
|
| 969 |
+
|
| 970 |
+
# Check if this span overlaps with any already-claimed higher-priority span
|
| 971 |
+
overlaps = False
|
| 972 |
+
for (c_start, c_end, c_type) in claimed_ranges:
|
| 973 |
+
# Two spans overlap if one starts before the other ends
|
| 974 |
+
if s_start < c_end and s_end > c_start:
|
| 975 |
+
overlaps = True
|
| 976 |
+
break
|
| 977 |
+
|
| 978 |
+
if not overlaps:
|
| 979 |
+
resolved.append(s)
|
| 980 |
+
claimed_ranges.append((s_start, s_end, s['type']))
|
| 981 |
+
else:
|
| 982 |
+
logger.info(f"[OVERLAP] Dropped {s['type']} [{s_start}:{s_end}] "
|
| 983 |
+
f"'{s.get('original','')}' — conflicts with higher-priority span")
|
| 984 |
+
|
| 985 |
+
dropped = len(suggestions) - len(resolved)
|
| 986 |
+
if dropped > 0:
|
| 987 |
+
logger.info(f"[OVERLAP] Resolved {dropped} overlapping suggestions")
|
| 988 |
+
suggestions = resolved
|
| 989 |
+
|
| 990 |
+
logger.info(f"[ANALYZE] Total: {timing_ms['total_ms']}ms | "
|
| 991 |
+
f"Spelling: {timing_ms['spelling_ms']}ms | "
|
| 992 |
+
f"Grammar: {timing_ms['grammar_ms']}ms | "
|
| 993 |
+
f"Punctuation: {timing_ms['punctuation_ms']}ms | "
|
| 994 |
+
f"Suggestions: {len(suggestions)}")
|
| 995 |
|
| 996 |
return jsonify({
|
| 997 |
'original': text,
|
| 998 |
'corrected': current_text,
|
| 999 |
'suggestions': suggestions,
|
| 1000 |
+
'timing_ms': timing_ms,
|
| 1001 |
'status': 'success'
|
| 1002 |
})
|
| 1003 |
|