youssefreda9 commited on
Commit
920c24b
·
1 Parent(s): 00d416d

feat: NLP-3.5 Hardening Sprint — Performance + Overlap Resolver TASK 2: Per-stage timing instrumentation (spelling_ms, grammar_ms, punctuation_ms, total_ms) TASK 3: Smart text processing — skip AraSpell for texts > 300 chars TASK 4+5: Global overlap resolver with priority system grammar(3) > punctuation(2) > spelling(1) > autocomplete(0) Full span collision detection (exact + partial overlaps) TASK 6: AutoComplete priority hook prepared (priority=0)

Browse files
Files changed (1) hide show
  1. src/app.py +63 -23
src/app.py CHANGED
@@ -718,6 +718,7 @@ def analyze_text():
718
  suggestions = []
719
  mappers = []
720
  total_start = time.time()
 
721
 
722
  def map_range_to_original(start, end):
723
  curr_start, curr_end = start, end
@@ -749,16 +750,25 @@ def analyze_text():
749
  result = [best_correction] + alts + [original_word]
750
  return result[:max_alts + 1] # cap at max_alts + keep-as-is
751
 
 
 
 
 
 
 
 
 
 
752
  # 1. Spelling (with conservative post-filtering to avoid over-editing)
753
- has_spelling = True # Always available via lazy-loaded araspell_service
754
- if has_spelling:
755
  try:
756
  t0 = time.time()
757
  logger.info(f"[ANALYZE] Step 1: Spelling correction starting...")
758
  from nlp.spelling.araspell_service import get_spelling_model
759
  spell_checker = get_spelling_model()
760
  raw_corrected = spell_checker.correct(current_text)
761
- logger.info(f"[ANALYZE] Step 1: Spelling done in {time.time()-t0:.2f}s")
 
762
 
763
  if raw_corrected != current_text:
764
  orig_word_positions = get_word_positions(current_text)
@@ -890,7 +900,8 @@ def analyze_text():
890
  from nlp.grammar.grammar_service import get_grammar_model
891
  grammar_checker = get_grammar_model()
892
  corrected_grammar = grammar_checker.correct(current_text)
893
- logger.info(f"[ANALYZE] Step 2: Grammar done in {time.time()-t0:.2f}s")
 
894
  if corrected_grammar != current_text:
895
  diffs = get_word_diffs(current_text, corrected_grammar)
896
  for d in diffs:
@@ -914,7 +925,8 @@ def analyze_text():
914
  from nlp.punctuation.punctuation_service import get_punctuation_model
915
  punc_checker = get_punctuation_model()
916
  corrected_punc = punc_checker.correct(current_text)
917
- logger.info(f"[ANALYZE] Step 3: Punctuation done in {time.time()-t0:.2f}s")
 
918
  if corrected_punc != current_text:
919
  diffs = get_word_diffs(current_text, corrected_punc)
920
  for d in diffs:
@@ -932,32 +944,60 @@ def analyze_text():
932
  logger.error(f"[ANALYZE] Punctuation failed: {e}")
933
 
934
  total_time = time.time() - total_start
 
935
 
936
- # ── Deduplicate overlapping suggestions ──
937
- # If grammar and spelling both flag the same position, grammar wins.
938
- # This prevents double-highlighting on words like "يعملوا" which are
939
- # grammar errors (not spelling errors).
940
- grammar_positions = set()
941
- for s in suggestions:
942
- if s['type'] == 'grammar':
943
- grammar_positions.add((s['start'], s['end']))
 
944
 
945
- if grammar_positions:
946
- before_count = len(suggestions)
947
- suggestions = [
948
- s for s in suggestions
949
- if s['type'] != 'spelling' or (s['start'], s['end']) not in grammar_positions
950
- ]
951
- removed = before_count - len(suggestions)
952
- if removed > 0:
953
- logger.info(f"[ANALYZE] Removed {removed} spelling suggestions that overlap with grammar")
954
 
955
- logger.info(f"[ANALYZE] Total analysis time: {total_time:.2f}s | Suggestions: {len(suggestions)}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
956
 
957
  return jsonify({
958
  'original': text,
959
  'corrected': current_text,
960
  'suggestions': suggestions,
 
961
  'status': 'success'
962
  })
963
 
 
718
  suggestions = []
719
  mappers = []
720
  total_start = time.time()
721
+ timing_ms = {'spelling_ms': 0, 'grammar_ms': 0, 'punctuation_ms': 0, 'total_ms': 0}
722
 
723
  def map_range_to_original(start, end):
724
  curr_start, curr_end = start, end
 
750
  result = [best_correction] + alts + [original_word]
751
  return result[:max_alts + 1] # cap at max_alts + keep-as-is
752
 
753
+ # ── Smart Text Processing Strategy ──
754
+ # Short (0-300 chars): full pipeline (Spelling + Grammar + Punctuation)
755
+ # Medium (300-1000 chars): Grammar + Punctuation only (skip AraSpell)
756
+ # Large (1000+ chars): Grammar + Punctuation only
757
+ text_len = len(current_text)
758
+ run_spelling = text_len <= 300
759
+ if not run_spelling:
760
+ logger.info(f"[ANALYZE] Text length {text_len} > 300 — skipping AraSpell for performance")
761
+
762
  # 1. Spelling (with conservative post-filtering to avoid over-editing)
763
+ if run_spelling:
 
764
  try:
765
  t0 = time.time()
766
  logger.info(f"[ANALYZE] Step 1: Spelling correction starting...")
767
  from nlp.spelling.araspell_service import get_spelling_model
768
  spell_checker = get_spelling_model()
769
  raw_corrected = spell_checker.correct(current_text)
770
+ timing_ms['spelling_ms'] = int((time.time() - t0) * 1000)
771
+ logger.info(f"[ANALYZE] Step 1: Spelling done in {timing_ms['spelling_ms']}ms")
772
 
773
  if raw_corrected != current_text:
774
  orig_word_positions = get_word_positions(current_text)
 
900
  from nlp.grammar.grammar_service import get_grammar_model
901
  grammar_checker = get_grammar_model()
902
  corrected_grammar = grammar_checker.correct(current_text)
903
+ timing_ms['grammar_ms'] = int((time.time() - t0) * 1000)
904
+ logger.info(f"[ANALYZE] Step 2: Grammar done in {timing_ms['grammar_ms']}ms")
905
  if corrected_grammar != current_text:
906
  diffs = get_word_diffs(current_text, corrected_grammar)
907
  for d in diffs:
 
925
  from nlp.punctuation.punctuation_service import get_punctuation_model
926
  punc_checker = get_punctuation_model()
927
  corrected_punc = punc_checker.correct(current_text)
928
+ timing_ms['punctuation_ms'] = int((time.time() - t0) * 1000)
929
+ logger.info(f"[ANALYZE] Step 3: Punctuation done in {timing_ms['punctuation_ms']}ms")
930
  if corrected_punc != current_text:
931
  diffs = get_word_diffs(current_text, corrected_punc)
932
  for d in diffs:
 
944
  logger.error(f"[ANALYZE] Punctuation failed: {e}")
945
 
946
  total_time = time.time() - total_start
947
+ timing_ms['total_ms'] = int(total_time * 1000)
948
 
949
+ # ══════════════════════════════════════════════════════════
950
+ # GLOBAL OVERLAP RESOLVER — NLP-3.5 Hardening
951
+ # ══════════════════════════════════════════════════════════
952
+ # Priority: grammar(3) > punctuation(2) > spelling(1) > autocomplete(0)
953
+ # Rules:
954
+ # - Higher priority ALWAYS wins
955
+ # - One span = one highlight (no stacking)
956
+ # - Partial overlaps: higher priority span kept, lower dropped
957
+ PRIORITY = {'grammar': 3, 'punctuation': 2, 'spelling': 1, 'autocomplete': 0}
958
 
959
+ # Sort suggestions by priority (highest first)
960
+ suggestions.sort(key=lambda s: PRIORITY.get(s['type'], 0), reverse=True)
 
 
 
 
 
 
 
961
 
962
+ # Build a set of "claimed" character positions
963
+ claimed_ranges = [] # list of (start, end, type)
964
+ resolved = []
965
+
966
+ for s in suggestions:
967
+ s_start, s_end = s['start'], s['end']
968
+ s_priority = PRIORITY.get(s['type'], 0)
969
+
970
+ # Check if this span overlaps with any already-claimed higher-priority span
971
+ overlaps = False
972
+ for (c_start, c_end, c_type) in claimed_ranges:
973
+ # Two spans overlap if one starts before the other ends
974
+ if s_start < c_end and s_end > c_start:
975
+ overlaps = True
976
+ break
977
+
978
+ if not overlaps:
979
+ resolved.append(s)
980
+ claimed_ranges.append((s_start, s_end, s['type']))
981
+ else:
982
+ logger.info(f"[OVERLAP] Dropped {s['type']} [{s_start}:{s_end}] "
983
+ f"'{s.get('original','')}' — conflicts with higher-priority span")
984
+
985
+ dropped = len(suggestions) - len(resolved)
986
+ if dropped > 0:
987
+ logger.info(f"[OVERLAP] Resolved {dropped} overlapping suggestions")
988
+ suggestions = resolved
989
+
990
+ logger.info(f"[ANALYZE] Total: {timing_ms['total_ms']}ms | "
991
+ f"Spelling: {timing_ms['spelling_ms']}ms | "
992
+ f"Grammar: {timing_ms['grammar_ms']}ms | "
993
+ f"Punctuation: {timing_ms['punctuation_ms']}ms | "
994
+ f"Suggestions: {len(suggestions)}")
995
 
996
  return jsonify({
997
  'original': text,
998
  'corrected': current_text,
999
  'suggestions': suggestions,
1000
+ 'timing_ms': timing_ms,
1001
  'status': 'success'
1002
  })
1003