youssefreda9 commited on
Commit
9c49fff
·
1 Parent(s): 33fed1a

Update extension UI, app pipeline and model loaders

Browse files
extension/popup.js CHANGED
@@ -55,6 +55,13 @@ document.addEventListener('DOMContentLoaded', () => {
55
  */
56
  let analyzedText = '';
57
 
 
 
 
 
 
 
 
58
  /**
59
  * MED-2: Whether the analysis results are stale.
60
  * Set to true when the user edits the textarea after analysis.
@@ -251,7 +258,7 @@ document.addEventListener('DOMContentLoaded', () => {
251
  btnApplyAll.classList.add('is-hidden');
252
  if (analyzedText) {
253
  suggestionsSection.classList.remove('is-hidden');
254
- suggestionsList.innerHTML = '<div class="bayan-empty-state"><svg class="bayan-empty-state-icon" width="48" height="48" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12l2 2 4-4m6 2a9 9 0 11-18 0 9 9 0 0118 0z"/></svg><p class="bayan-empty-state-text">نصك ممتاز! لم نجد أي أخطاء</p></div>';
255
  } else {
256
  suggestionsSection.classList.add('is-hidden');
257
  }
@@ -279,7 +286,9 @@ document.addEventListener('DOMContentLoaded', () => {
279
  if (!suggestion) return;
280
 
281
  if (altText === suggestion.original) {
282
- // Dismiss — remove from list, no text change, no rebase needed
 
 
283
  currentSuggestions = removeSuggestion(currentSuggestions, suggestion.id);
284
  } else {
285
  // ═══════════════════════════════════════════════════
@@ -304,9 +313,13 @@ document.addEventListener('DOMContentLoaded', () => {
304
  });
305
  });
306
 
307
- // Show apply-all with count
308
- btnApplyAll.textContent = 'تطبيق الكل (' + suggestions.length.toLocaleString('ar-EG') + ')';
309
- btnApplyAll.classList.remove('is-hidden');
 
 
 
 
310
  }
311
 
312
  // ══════════════════════════════════════════════════════════
@@ -349,7 +362,9 @@ document.addEventListener('DOMContentLoaded', () => {
349
  const data = await bayanAnalyze(text);
350
 
351
  if (data.status === 'success' || data.status === 'partial') {
352
- const suggestions = sortSuggestions(data.suggestions || []);
 
 
353
  currentSuggestions = suggestions;
354
 
355
  // MED-2: Snapshot the analyzed text — all offsets reference THIS string
 
55
  */
56
  let analyzedText = '';
57
 
58
+ const _dismissedWords = new Set(
59
+ JSON.parse(localStorage.getItem('bayan_dismissed_words') || '[]')
60
+ );
61
+ function _saveDismissedWords() {
62
+ try { localStorage.setItem('bayan_dismissed_words', JSON.stringify([..._dismissedWords])); } catch {}
63
+ }
64
+
65
  /**
66
  * MED-2: Whether the analysis results are stale.
67
  * Set to true when the user edits the textarea after analysis.
 
258
  btnApplyAll.classList.add('is-hidden');
259
  if (analyzedText) {
260
  suggestionsSection.classList.remove('is-hidden');
261
+ suggestionsList.innerHTML = '<div class="bayan-empty-state"><svg class="bayan-empty-state-icon" width="48" height="48" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12l2 2 4-4m6 2a9 9 0 11-18 0 9 9 0 0118 0z"/></svg><p class="bayan-empty-state-title">نصك ممتاز!</p><p class="bayan-empty-state-desc">لم نجد أي أخطاء — أحسنت! ✨</p></div>';
262
  } else {
263
  suggestionsSection.classList.add('is-hidden');
264
  }
 
286
  if (!suggestion) return;
287
 
288
  if (altText === suggestion.original) {
289
+ // Dismiss — remove from list, add to permanent whitelist
290
+ _dismissedWords.add(suggestion.original);
291
+ _saveDismissedWords();
292
  currentSuggestions = removeSuggestion(currentSuggestions, suggestion.id);
293
  } else {
294
  // ═══════════════════════════════════════════════════
 
313
  });
314
  });
315
 
316
+ // Show apply-all only when more than 1 suggestion
317
+ if (suggestions.length > 1) {
318
+ btnApplyAll.textContent = 'تطبيق الكل (' + suggestions.length.toLocaleString('ar-EG') + ')';
319
+ btnApplyAll.classList.remove('is-hidden');
320
+ } else {
321
+ btnApplyAll.classList.add('is-hidden');
322
+ }
323
  }
324
 
325
  // ══════════════════════════════════════════════════════════
 
362
  const data = await bayanAnalyze(text);
363
 
364
  if (data.status === 'success' || data.status === 'partial') {
365
+ const suggestions = sortSuggestions(data.suggestions || []).filter(
366
+ s => !_dismissedWords.has(s.original)
367
+ );
368
  currentSuggestions = suggestions;
369
 
370
  // MED-2: Snapshot the analyzed text — all offsets reference THIS string
extension/sidepanel/sidepanel.js CHANGED
@@ -62,6 +62,13 @@ document.addEventListener('DOMContentLoaded', () => {
62
  let sourceSelectionText = '';
63
  let summaryMode = 'paragraph';
64
 
 
 
 
 
 
 
 
65
  const SCORE_CIRCUMFERENCE = 440;
66
 
67
  // ══════════════════════════════════════════════════════════
@@ -206,7 +213,7 @@ document.addEventListener('DOMContentLoaded', () => {
206
  btnApplyAll.classList.add('is-hidden');
207
  if (analyzedText) {
208
  suggestionsSection.classList.remove('is-hidden');
209
- suggestionsList.innerHTML = '<div class="sp-empty-state"><svg class="sp-empty-state-icon" width="48" height="48" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12l2 2 4-4m6 2a9 9 0 11-18 0 9 9 0 0118 0z"/></svg><p class="sp-empty-state-text">نصك ممتاز! لم نجد أي أخطاء</p></div>';
210
  } else {
211
  suggestionsSection.classList.add('is-hidden');
212
  }
@@ -231,6 +238,8 @@ document.addEventListener('DOMContentLoaded', () => {
231
  if (!suggestion) return;
232
 
233
  if (altText === suggestion.original) {
 
 
234
  currentSuggestions = removeSuggestion(currentSuggestions, suggestion.id);
235
  } else {
236
  const result = applyAndRebase(analyzedText, suggestion, altText, currentSuggestions);
@@ -248,9 +257,13 @@ document.addEventListener('DOMContentLoaded', () => {
248
  });
249
  });
250
 
251
- // Show apply-all with count
252
- btnApplyAll.textContent = 'تطبيق الكل (' + suggestions.length.toLocaleString('ar-EG') + ')';
253
- btnApplyAll.classList.remove('is-hidden');
 
 
 
 
254
  }
255
 
256
  // ══════════════════════════════════════════════════════════
@@ -274,7 +287,9 @@ document.addEventListener('DOMContentLoaded', () => {
274
  const data = await bayanAnalyze(text);
275
 
276
  if (data.status === 'success' || data.status === 'partial') {
277
- const suggestions = sortSuggestions(data.suggestions || []);
 
 
278
  currentSuggestions = suggestions;
279
  analyzedText = data.original;
280
 
 
62
  let sourceSelectionText = '';
63
  let summaryMode = 'paragraph';
64
 
65
+ const _dismissedWords = new Set(
66
+ JSON.parse(localStorage.getItem('bayan_dismissed_words') || '[]')
67
+ );
68
+ function _saveDismissedWords() {
69
+ try { localStorage.setItem('bayan_dismissed_words', JSON.stringify([..._dismissedWords])); } catch {}
70
+ }
71
+
72
  const SCORE_CIRCUMFERENCE = 440;
73
 
74
  // ══════════════════════════════════════════════════════════
 
213
  btnApplyAll.classList.add('is-hidden');
214
  if (analyzedText) {
215
  suggestionsSection.classList.remove('is-hidden');
216
+ suggestionsList.innerHTML = '<div class="sp-empty-state"><svg class="sp-empty-state-icon" width="48" height="48" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12l2 2 4-4m6 2a9 9 0 11-18 0 9 9 0 0118 0z"/></svg><p class="sp-empty-state-title">نصك ممتاز!</p><p class="sp-empty-state-desc">لم نجد أي أخطاء — أحسنت! ✨</p></div>';
217
  } else {
218
  suggestionsSection.classList.add('is-hidden');
219
  }
 
238
  if (!suggestion) return;
239
 
240
  if (altText === suggestion.original) {
241
+ _dismissedWords.add(suggestion.original);
242
+ _saveDismissedWords();
243
  currentSuggestions = removeSuggestion(currentSuggestions, suggestion.id);
244
  } else {
245
  const result = applyAndRebase(analyzedText, suggestion, altText, currentSuggestions);
 
257
  });
258
  });
259
 
260
+ // Show apply-all only when more than 1 suggestion
261
+ if (suggestions.length > 1) {
262
+ btnApplyAll.textContent = 'تطبيق الكل (' + suggestions.length.toLocaleString('ar-EG') + ')';
263
+ btnApplyAll.classList.remove('is-hidden');
264
+ } else {
265
+ btnApplyAll.classList.add('is-hidden');
266
+ }
267
  }
268
 
269
  // ══════════════════════════════════════════════════════════
 
287
  const data = await bayanAnalyze(text);
288
 
289
  if (data.status === 'success' || data.status === 'partial') {
290
+ const suggestions = sortSuggestions(data.suggestions || []).filter(
291
+ s => !_dismissedWords.has(s.original)
292
+ );
293
  currentSuggestions = suggestions;
294
  analyzedText = data.original;
295
 
src/app.py CHANGED
@@ -46,12 +46,10 @@ from model_loader import (
46
  SummarizationModel,
47
  SpellingModel,
48
  AutocompleteModel,
49
- GrammarModel,
50
  PunctuationModel,
51
  SUMMARIZATION_PATH,
52
  SPELLING_PATH,
53
  AUTOCOMPLETE_PATH,
54
- GRAMMAR_PATH,
55
  PUNCTUATION_PATH
56
  )
57
 
@@ -754,24 +752,34 @@ class OffsetMapper:
754
  inverted range (start > end) due to non-monotonic edits,
755
  the end is clamped to max(new_start, new_end).
756
  """
757
- new_start = self._forward_map_pos(start_in_before)
758
- new_end = self._forward_map_pos(end_in_before)
759
  # Monotonicity guard: prevent inverted ranges
760
  new_end = max(new_start, new_end)
761
  return new_start, new_end
762
 
763
- def _forward_map_pos(self, pos):
764
  """Map a single position text_before → text_after. PRIVATE."""
 
765
  for i1, i2, j1, j2 in self._opcodes:
766
  if i1 <= pos <= i2:
767
- if i2 == i1:
768
- return j1
 
 
 
 
 
 
 
 
 
 
 
769
  ratio = (pos - i1) / (i2 - i1)
770
- return int(j1 + ratio * (j2 - j1))
771
- if self._opcodes:
772
- last = self._opcodes[-1]
773
- return last[3] + (pos - last[1])
774
- return pos
775
 
776
 
777
 
@@ -1521,7 +1529,7 @@ def quran_verify():
1521
  if not logger_quran_ok:
1522
  return jsonify({'error': 'Quran search module not available'}), 503
1523
 
1524
- data = request.get_json(force=True)
1525
  text = data.get('text', '').strip()
1526
  language = data.get('language', 'تدقيق الايات').strip()
1527
 
@@ -2226,8 +2234,10 @@ def analyze_text():
2226
  # Replace structured content with Arabic placeholder tokens
2227
  if _structured_placeholders:
2228
  _structured_placeholders.sort(key=lambda x: x[0], reverse=True)
 
 
2229
  for _sp_idx, (_sp_start, _sp_end, _sp_text) in enumerate(_structured_placeholders):
2230
- _grammar_input_text = _grammar_input_text[:_sp_start] + f'بيان{_sp_idx}' + _grammar_input_text[_sp_end:]
2231
  logger.info(f"[ANALYZE] Protected {len(_structured_placeholders)} structured elements")
2232
 
2233
  # 2. Grammar (runs on spelling-corrected text — word-level dependency)
@@ -2249,7 +2259,7 @@ def analyze_text():
2249
  # FIX-03: Restore structured content in grammar output
2250
  if _structured_placeholders:
2251
  for _sp_idx, (_sp_start, _sp_end, _sp_text) in enumerate(_structured_placeholders):
2252
- corrected_grammar = corrected_grammar.replace(f'بيان{_sp_idx}', _sp_text)
2253
 
2254
  if corrected_grammar != ctx.current_text:
2255
  diffs = get_word_diffs(ctx.current_text, corrected_grammar)
@@ -2504,19 +2514,6 @@ def analyze_text():
2504
  )
2505
  continue
2506
 
2507
- # ── FIX-27a: Grammar structured data protection ──
2508
- # Block grammar diffs where the original contains digits.
2509
- # The grammar model corrupts dates/numbers/times/percentages.
2510
- # e.g., '2026-06-22' → 'عشرين 26-06-22ا'
2511
- if orig_text and any(c.isdigit() for c in orig_text):
2512
- logger.info(
2513
- f"[GRAMMAR] Blocked digit-containing diff: "
2514
- f"'{orig_text}'\u2192'{corr_text}'"
2515
- )
2516
- logger.info(f'[FILTER-TEL] {_tel_json.dumps({"event":"filter_reject","filter":"DigitGuard","original":orig_text[:80],"correction":corr_text[:80]})}')
2517
- _tel_events.append({"event":"filter_reject","filter":"DigitGuard","original":orig_text[:80],"correction":corr_text[:80]})
2518
- continue
2519
-
2520
  # ── FIX-27b: Grammar hallucination guard (Jaccard) ──
2521
  # Block grammar diffs where the correction is too different
2522
  # from the original (character-level Jaccard < 0.5).
@@ -2575,10 +2572,13 @@ def analyze_text():
2575
  # FIX-22: Protect tanween (preserve ً ٌ ٍ from original)
2576
  _TANWEEN_CHARS = set('ًٌٍ')
2577
  if any(c in _TANWEEN_CHARS for c in orig_text) and not any(c in _TANWEEN_CHARS for c in corr_text):
2578
- # Grammar stripped tanween — reattach it
2579
  for _tc in _TANWEEN_CHARS:
2580
  if _tc in orig_text and _tc not in corr_text:
2581
- corr_text = corr_text + _tc
 
 
 
2582
  d['correction'] = corr_text
2583
  break
2584
 
@@ -2609,14 +2609,25 @@ def analyze_text():
2609
  logger.info(
2610
  f"[GRAMMAR] Bracket-unbalanced output detected: "
2611
  f"orig=({orig_opens},{orig_closes}), corr=({corr_opens},{corr_closes}). "
2612
- f"Using individually-accepted diffs only."
2613
  )
 
2614
  if _grammar_accepted_diffs:
2615
  # FIX-04: Rebuild grammar text from ACCEPTED diffs only,
2616
  # not the full model output. Prevents phantom corrections.
 
 
 
 
 
 
 
 
 
 
 
2617
  _safe_grammar = ctx.current_text
2618
- # Apply accepted diffs in reverse order to build safe text
2619
- for _ad in sorted(_grammar_accepted_diffs, key=lambda x: x['start'], reverse=True):
2620
  _safe_grammar = (_safe_grammar[:_ad['start']] +
2621
  _ad['correction'] +
2622
  _safe_grammar[_ad['end']:])
@@ -2630,8 +2641,6 @@ def analyze_text():
2630
 
2631
  # 3. Punctuation (runs on grammar-corrected text — PuncAra-v1 local model)
2632
  # FIX-07: Skip punctuation for religious text
2633
- # FIX-51: Skip punctuation when spelling+grammar found no errors (clean text)
2634
- _has_real_corrections = any(p.stage in ('spelling', 'grammar') for p in ctx.patches.patches)
2635
  if not _skip_all_stages:
2636
  try:
2637
  t0 = time.time()
 
46
  SummarizationModel,
47
  SpellingModel,
48
  AutocompleteModel,
 
49
  PunctuationModel,
50
  SUMMARIZATION_PATH,
51
  SPELLING_PATH,
52
  AUTOCOMPLETE_PATH,
 
53
  PUNCTUATION_PATH
54
  )
55
 
 
752
  inverted range (start > end) due to non-monotonic edits,
753
  the end is clamped to max(new_start, new_end).
754
  """
755
+ new_start = self._forward_map_pos(start_in_before, is_end=False)
756
+ new_end = self._forward_map_pos(end_in_before, is_end=True)
757
  # Monotonicity guard: prevent inverted ranges
758
  new_end = max(new_start, new_end)
759
  return new_start, new_end
760
 
761
+ def _forward_map_pos(self, pos, is_end=False):
762
  """Map a single position text_before → text_after. PRIVATE."""
763
+ matches = []
764
  for i1, i2, j1, j2 in self._opcodes:
765
  if i1 <= pos <= i2:
766
+ matches.append((i1, i2, j1, j2))
767
+
768
+ if not matches:
769
+ if self._opcodes:
770
+ last = self._opcodes[-1]
771
+ return last[3] + (pos - last[1])
772
+ return pos
773
+
774
+ mapped = []
775
+ for i1, i2, j1, j2 in matches:
776
+ if i2 == i1: # insertion in text_after
777
+ mapped.append(j2 if is_end else j1)
778
+ else:
779
  ratio = (pos - i1) / (i2 - i1)
780
+ mapped.append(round(j1 + ratio * (j2 - j1)))
781
+
782
+ return max(mapped) if is_end else min(mapped)
 
 
783
 
784
 
785
 
 
1529
  if not logger_quran_ok:
1530
  return jsonify({'error': 'Quran search module not available'}), 503
1531
 
1532
+ data = request.get_json()
1533
  text = data.get('text', '').strip()
1534
  language = data.get('language', 'تدقيق الايات').strip()
1535
 
 
2234
  # Replace structured content with Arabic placeholder tokens
2235
  if _structured_placeholders:
2236
  _structured_placeholders.sort(key=lambda x: x[0], reverse=True)
2237
+ import uuid as _uuid_struct
2238
+ _ph_prefix = _uuid_struct.uuid4().hex[:8]
2239
  for _sp_idx, (_sp_start, _sp_end, _sp_text) in enumerate(_structured_placeholders):
2240
+ _grammar_input_text = _grammar_input_text[:_sp_start] + f'ـ{_ph_prefix}{_sp_idx}ـ' + _grammar_input_text[_sp_end:]
2241
  logger.info(f"[ANALYZE] Protected {len(_structured_placeholders)} structured elements")
2242
 
2243
  # 2. Grammar (runs on spelling-corrected text — word-level dependency)
 
2259
  # FIX-03: Restore structured content in grammar output
2260
  if _structured_placeholders:
2261
  for _sp_idx, (_sp_start, _sp_end, _sp_text) in enumerate(_structured_placeholders):
2262
+ corrected_grammar = corrected_grammar.replace(f'ـ{_ph_prefix}{_sp_idx}ـ', _sp_text)
2263
 
2264
  if corrected_grammar != ctx.current_text:
2265
  diffs = get_word_diffs(ctx.current_text, corrected_grammar)
 
2514
  )
2515
  continue
2516
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2517
  # ── FIX-27b: Grammar hallucination guard (Jaccard) ──
2518
  # Block grammar diffs where the correction is too different
2519
  # from the original (character-level Jaccard < 0.5).
 
2572
  # FIX-22: Protect tanween (preserve ً ٌ ٍ from original)
2573
  _TANWEEN_CHARS = set('ًٌٍ')
2574
  if any(c in _TANWEEN_CHARS for c in orig_text) and not any(c in _TANWEEN_CHARS for c in corr_text):
2575
+ # Grammar stripped tanween — reattach before trailing punctuation
2576
  for _tc in _TANWEEN_CHARS:
2577
  if _tc in orig_text and _tc not in corr_text:
2578
+ _tn_punct_idx = len(corr_text)
2579
+ while _tn_punct_idx > 0 and corr_text[_tn_punct_idx - 1] in '.،؛؟!?!':
2580
+ _tn_punct_idx -= 1
2581
+ corr_text = corr_text[:_tn_punct_idx] + _tc + corr_text[_tn_punct_idx:]
2582
  d['correction'] = corr_text
2583
  break
2584
 
 
2609
  logger.info(
2610
  f"[GRAMMAR] Bracket-unbalanced output detected: "
2611
  f"orig=({orig_opens},{orig_closes}), corr=({corr_opens},{corr_closes}). "
2612
+ f"Rejecting all grammar diffs."
2613
  )
2614
+ _grammar_accepted_diffs = []
2615
  if _grammar_accepted_diffs:
2616
  # FIX-04: Rebuild grammar text from ACCEPTED diffs only,
2617
  # not the full model output. Prevents phantom corrections.
2618
+ # Remove intra-stage overlaps: keep higher-start diff on conflict
2619
+ _grammar_accepted_diffs.sort(key=lambda x: x['start'])
2620
+ _non_overlapping = []
2621
+ for _ad in _grammar_accepted_diffs:
2622
+ if _non_overlapping and _ad['start'] < _non_overlapping[-1]['end']:
2623
+ logger.info(
2624
+ f"[GRAMMAR] Dropped overlapping diff [{_ad['start']}:{_ad['end']}] "
2625
+ f"(overlaps [{_non_overlapping[-1]['start']}:{_non_overlapping[-1]['end']}])"
2626
+ )
2627
+ continue
2628
+ _non_overlapping.append(_ad)
2629
  _safe_grammar = ctx.current_text
2630
+ for _ad in reversed(_non_overlapping):
 
2631
  _safe_grammar = (_safe_grammar[:_ad['start']] +
2632
  _ad['correction'] +
2633
  _safe_grammar[_ad['end']:])
 
2641
 
2642
  # 3. Punctuation (runs on grammar-corrected text — PuncAra-v1 local model)
2643
  # FIX-07: Skip punctuation for religious text
 
 
2644
  if not _skip_all_stages:
2645
  try:
2646
  t0 = time.time()
src/js/editor.js CHANGED
@@ -300,7 +300,7 @@ async function analyzeText() {
300
 
301
  const data = await response.json();
302
 
303
- if (data.status !== 'success' || !data.suggestions) {
304
  renderWithoutSuggestions(text);
305
  return;
306
  }
 
300
 
301
  const data = await response.json();
302
 
303
+ if ((data.status !== 'success' && data.status !== 'partial') || !data.suggestions) {
304
  renderWithoutSuggestions(text);
305
  return;
306
  }
src/js/ui.js CHANGED
@@ -26,7 +26,9 @@ function updateWritingScore(spelling, grammar, punctuation) {
26
  const hintEl = document.getElementById('score-hint');
27
 
28
  if (valueEl) {
29
- valueEl.textContent = score > 0 || (spelling + grammar + punctuation) > 0
 
 
30
  ? score.toLocaleString('ar-EG')
31
  : '--';
32
  }
@@ -38,7 +40,11 @@ function updateWritingScore(spelling, grammar, punctuation) {
38
 
39
  if (hintEl) {
40
  const total = spelling + grammar + punctuation;
41
- if (total === 0) {
 
 
 
 
42
  hintEl.innerHTML = 'ابدأ الكتابة لرؤية تقييمك<br><span class="text-xs">تحسين القواعد يرفع التقييم</span>';
43
  } else if (score >= 90) {
44
  hintEl.textContent = 'كتابة ممتازة! استمر.';
 
26
  const hintEl = document.getElementById('score-hint');
27
 
28
  if (valueEl) {
29
+ const editorText = typeof getEditorText === 'function' ? getEditorText() : '';
30
+ const hasText = editorText.trim().length > 0;
31
+ valueEl.textContent = (hasText || (spelling + grammar + punctuation) > 0)
32
  ? score.toLocaleString('ar-EG')
33
  : '--';
34
  }
 
40
 
41
  if (hintEl) {
42
  const total = spelling + grammar + punctuation;
43
+ const editorText = typeof getEditorText === 'function' ? getEditorText() : '';
44
+ const hasText = editorText.trim().length > 0;
45
+ if (total === 0 && hasText) {
46
+ hintEl.textContent = 'كتابة ممتازة! استمر.';
47
+ } else if (total === 0) {
48
  hintEl.innerHTML = 'ابدأ الكتابة لرؤية تقييمك<br><span class="text-xs">تحسين القواعد يرفع التقييم</span>';
49
  } else if (score >= 90) {
50
  hintEl.textContent = 'كتابة ممتازة! استمر.';
src/model_loader.py CHANGED
@@ -7,16 +7,14 @@ import os
7
  import logging
8
  from pathlib import Path
9
  import json
10
- import pickle
11
  import difflib
12
  import torch
13
  import importlib.util
14
  from transformers import (
15
- MBartForConditionalGeneration,
16
- AutoTokenizer,
17
  AutoConfig,
18
  AutoModelForSeq2SeqLM,
19
- AutoModelForCausalLM,
20
  EncoderDecoderModel,
21
  BertConfig,
22
  EncoderDecoderConfig
@@ -33,7 +31,6 @@ MODEL_BASE_PATH = Path(__file__).parent.parent / "models"
33
  SUMMARIZATION_PATH = MODEL_BASE_PATH / "Summarization" / "Model"
34
  SPELLING_PATH = MODEL_BASE_PATH / "Spelling" / "Model"
35
  AUTOCOMPLETE_PATH = MODEL_BASE_PATH / "Autocomplete" / "Model"
36
- GRAMMAR_PATH = MODEL_BASE_PATH / "Grammrar" / "Model"
37
  PUNCTUATION_PATH = MODEL_BASE_PATH / "Punctuation" / "Model"
38
 
39
 
@@ -91,43 +88,29 @@ class SummarizationModel:
91
  logger.info(f"Model path validated: {self.model_path}")
92
 
93
  def _fix_generation_config(self):
94
- """Fix generation_config.json and config.json if early_stopping is None/null."""
 
95
  if self._is_remote_source:
96
  return
97
 
98
- gen_config_path = self.model_path / "generation_config.json"
99
  config_path = self.model_path / "config.json"
100
-
 
101
  try:
102
- # Fix config.json first (this is the main issue)
103
  if config_path.exists():
104
  with open(config_path, 'r', encoding='utf-8') as f:
105
  config = json.load(f)
106
-
107
- # Check if early_stopping exists in config and is None
108
  if 'early_stopping' in config and config['early_stopping'] is None:
109
- logger.info("Fixing early_stopping in config.json (was None)")
110
- config['early_stopping'] = True
111
- with open(config_path, 'w', encoding='utf-8') as f:
112
- json.dump(config, f, indent=2, ensure_ascii=False)
113
- logger.info("Fixed config.json - set early_stopping to True")
114
-
115
- # Fix generation_config.json
116
  if gen_config_path.exists():
117
  with open(gen_config_path, 'r', encoding='utf-8') as f:
118
  gen_config = json.load(f)
119
-
120
- # Fix early_stopping if it's None/null
121
  if gen_config.get('early_stopping') is None:
122
- logger.info("Fixing early_stopping in generation_config.json (was None)")
123
- gen_config['early_stopping'] = True
124
- with open(gen_config_path, 'w', encoding='utf-8') as f:
125
- json.dump(gen_config, f, indent=2, ensure_ascii=False)
126
- logger.info("Fixed generation_config.json - set early_stopping to True")
127
-
128
  except Exception as e:
129
- logger.warning(f"Could not fix generation config files: {str(e)}")
130
- # Continue anyway, we'll try to load with workaround
131
 
132
  def _load_model(self):
133
  """Load the model and tokenizer."""
@@ -145,14 +128,14 @@ class SummarizationModel:
145
  try:
146
  self.tokenizer = AutoTokenizer.from_pretrained(
147
  self.model_source,
148
- local_files_only=True,
149
  trust_remote_code=False
150
  )
151
  logger.info("Tokenizer loaded successfully")
152
  except Exception as e:
153
- logger.error(f"Failed to load tokenizer: {str(e)}")
154
- # Try without local_files_only as fallback
155
- logger.info("Retrying tokenizer load without local_files_only...")
156
  self.tokenizer = AutoTokenizer.from_pretrained(
157
  self.model_source,
158
  trust_remote_code=False
@@ -169,7 +152,7 @@ class SummarizationModel:
169
  try:
170
  config = AutoConfig.from_pretrained(
171
  self.model_source,
172
- local_files_only=True,
173
  trust_remote_code=False
174
  )
175
 
@@ -384,20 +367,27 @@ class SummarizationModel:
384
 
385
  class SpellingModel:
386
  """Wrapper class for the Arabic spelling correction model."""
387
-
388
  def __init__(self, model_path=None):
389
  """
390
  Initialize the spelling model.
391
-
392
  Args:
393
  model_path: Path to the model directory (defaults to SPELLING_PATH)
394
  """
395
  self.model_path = Path(model_path) if model_path else SPELLING_PATH
396
  self.model = None
397
  self.device = None
398
-
399
  self._validate_path()
400
  self._load_model()
 
 
 
 
 
 
 
401
 
402
  def _validate_path(self):
403
  """Validate that the model path exists."""
@@ -464,7 +454,7 @@ class SpellingModel:
464
  # Load state dict
465
  pt_file = list(self.model_path.glob("*.pt"))[0]
466
  logger.info(f"Loading weights from {pt_file}...")
467
- checkpoint = torch.load(pt_file, map_location=self.device)
468
 
469
  if "model_state_dict" in checkpoint:
470
  state_dict = checkpoint["model_state_dict"]
@@ -657,161 +647,13 @@ class AutocompleteModel:
657
  raise RuntimeError(f"Autocomplete prediction failed: {str(e)}")
658
 
659
 
660
- class GrammarModel:
661
- """Wrapper class for the Arabic grammar correction model."""
662
-
663
- def __init__(self, model_path=None):
664
- """
665
- Initialize the grammar model.
666
-
667
- Args:
668
- model_path: Path to the model directory (defaults to GRAMMAR_PATH)
669
- """
670
- self.model_path = Path(model_path) if model_path else GRAMMAR_PATH
671
- self.model = None
672
- self.tokenizer = None
673
- self.device = None
674
-
675
- self._validate_path()
676
- self._load_model()
677
-
678
- def _validate_path(self):
679
- """Validate that the model path exists and contains required files."""
680
- if not self.model_path.exists():
681
- raise FileNotFoundError(f"Model path does not exist: {self.model_path}")
682
-
683
- required_files = ['config.json', 'tokenizer.json', 'model.safetensors']
684
- missing_files = [f for f in required_files if not (self.model_path / f).exists()]
685
-
686
- if missing_files:
687
- raise FileNotFoundError(f"Missing required files: {', '.join(missing_files)}")
688
-
689
- logger.info(f"Grammar model path validated: {self.model_path}")
690
-
691
- def _load_model(self):
692
- """Load the grammar model and tokenizer."""
693
- try:
694
- # Force CPU-only to avoid GPU OOM / system freezes
695
- self.device = torch.device('cpu')
696
- logger.info("Loading grammar model on CPU (GPU disabled by design)...")
697
-
698
- self.tokenizer = AutoTokenizer.from_pretrained(
699
- str(self.model_path),
700
- local_files_only=True
701
- )
702
-
703
- self.model = AutoModelForCausalLM.from_pretrained(
704
- str(self.model_path),
705
- local_files_only=True,
706
- trust_remote_code=True,
707
- torch_dtype=torch.float32 # safe default for CPU inference
708
- )
709
-
710
- self.model.to(self.device)
711
- self.model.eval()
712
-
713
- logger.info("Grammar model loaded successfully")
714
- except Exception as e:
715
- logger.error(f"Error loading grammar model: {str(e)}")
716
- raise RuntimeError(f"Failed to load grammar model: {str(e)}")
717
-
718
- def correct(self, text):
719
- """
720
- Correct grammar in Arabic text with a timeout.
721
-
722
- Args:
723
- text: Input Arabic text
724
-
725
- Returns:
726
- str: Grammar-corrected text
727
- """
728
- if self.model is None or self.tokenizer is None:
729
- raise RuntimeError("Model not loaded")
730
-
731
- import threading
732
- result = [text] # default to original
733
- error = [None]
734
-
735
- def _generate():
736
- try:
737
- # Use Gemma 3 chat template — pass text only (model is GEC-trained)
738
- messages = [{"role": "user", "content": text}]
739
- prompt = self.tokenizer.apply_chat_template(
740
- messages,
741
- tokenize=False,
742
- add_generation_prompt=True,
743
- )
744
-
745
- inputs = self.tokenizer(
746
- prompt,
747
- max_length=256,
748
- truncation=True,
749
- padding=True,
750
- return_tensors="pt",
751
- add_special_tokens=False,
752
- )
753
-
754
- inputs = {k: v.to(self.device) for k, v in inputs.items()}
755
- input_length = inputs['input_ids'].shape[1]
756
-
757
- with torch.no_grad():
758
- outputs = self.model.generate(
759
- **inputs,
760
- max_new_tokens=64, # enough for corrected sentence
761
- do_sample=False,
762
- )
763
-
764
- # Decode only the NEW tokens (skip the prompt)
765
- new_tokens = outputs[0][input_length:]
766
- corrected = self.tokenizer.decode(new_tokens, skip_special_tokens=True).strip()
767
-
768
- # If model returned empty or nonsense, keep original
769
- if not corrected or len(corrected) < 2:
770
- return
771
-
772
- # Reject generic instruction phrases (model fallback, not actual correction)
773
- generic_phrases = (
774
- "أعد كتابتها", "أعد كتابته", "أعد كتابتها.", "أعد كتابته.",
775
- "اعيد كتابتها", "اعيد كتابته", "أعد كتابة", "اعيد كتابة",
776
- "أعد كتابتها فقط", "أعد كتابته فقط",
777
- )
778
- corrected_lower = corrected.strip()
779
- for phrase in generic_phrases:
780
- if phrase in corrected_lower or corrected_lower.startswith(phrase):
781
- return
782
-
783
- # Take only the first non-empty line as the corrected sentence
784
- corrected_lines = [l.strip() for l in corrected.split('\n') if l.strip()]
785
- if corrected_lines:
786
- first = corrected_lines[0]
787
- if first and len(first) >= 2 and first not in generic_phrases:
788
- result[0] = first
789
- except Exception as e:
790
- error[0] = e
791
-
792
- # Run generation in a thread with a 30s timeout
793
- thread = threading.Thread(target=_generate)
794
- thread.start()
795
- thread.join(timeout=30)
796
-
797
- if thread.is_alive():
798
- logger.warning("[Grammar] Timed out after 30s — returning original text")
799
- return text
800
-
801
- if error[0]:
802
- logger.error(f"Error during grammar correction: {str(error[0])}")
803
- logger.warning("Returning original text due to grammar error.")
804
-
805
- return result[0]
806
-
807
-
808
  class PunctuationModel:
809
  """Wrapper class for the Arabic punctuation model."""
810
-
811
  def __init__(self, model_path=None):
812
  """
813
  Initialize the punctuation model.
814
-
815
  Args:
816
  model_path: Path to the model directory (defaults to PUNCTUATION_PATH)
817
  """
@@ -819,9 +661,19 @@ class PunctuationModel:
819
  self.model = None
820
  self.tokenizer = None
821
  self.device = None
822
-
823
  self._validate_path()
824
  self._load_model()
 
 
 
 
 
 
 
 
 
 
825
 
826
  def _validate_path(self):
827
  """Validate that the model path exists and contains required files."""
 
7
  import logging
8
  from pathlib import Path
9
  import json
 
10
  import difflib
11
  import torch
12
  import importlib.util
13
  from transformers import (
14
+ MBartForConditionalGeneration,
15
+ AutoTokenizer,
16
  AutoConfig,
17
  AutoModelForSeq2SeqLM,
 
18
  EncoderDecoderModel,
19
  BertConfig,
20
  EncoderDecoderConfig
 
31
  SUMMARIZATION_PATH = MODEL_BASE_PATH / "Summarization" / "Model"
32
  SPELLING_PATH = MODEL_BASE_PATH / "Spelling" / "Model"
33
  AUTOCOMPLETE_PATH = MODEL_BASE_PATH / "Autocomplete" / "Model"
 
34
  PUNCTUATION_PATH = MODEL_BASE_PATH / "Punctuation" / "Model"
35
 
36
 
 
88
  logger.info(f"Model path validated: {self.model_path}")
89
 
90
  def _fix_generation_config(self):
91
+ """Check generation configs for early_stopping=None and log a warning.
92
+ In-memory fix is applied after loading at line ~177."""
93
  if self._is_remote_source:
94
  return
95
 
 
96
  config_path = self.model_path / "config.json"
97
+ gen_config_path = self.model_path / "generation_config.json"
98
+
99
  try:
 
100
  if config_path.exists():
101
  with open(config_path, 'r', encoding='utf-8') as f:
102
  config = json.load(f)
 
 
103
  if 'early_stopping' in config and config['early_stopping'] is None:
104
+ logger.info("early_stopping is None in config.json — will fix in-memory after load")
105
+
 
 
 
 
 
106
  if gen_config_path.exists():
107
  with open(gen_config_path, 'r', encoding='utf-8') as f:
108
  gen_config = json.load(f)
 
 
109
  if gen_config.get('early_stopping') is None:
110
+ logger.info("early_stopping is None in generation_config.json — will fix in-memory after load")
111
+
 
 
 
 
112
  except Exception as e:
113
+ logger.warning(f"Could not read generation config files: {str(e)}")
 
114
 
115
  def _load_model(self):
116
  """Load the model and tokenizer."""
 
128
  try:
129
  self.tokenizer = AutoTokenizer.from_pretrained(
130
  self.model_source,
131
+ local_files_only=not self._is_remote_source,
132
  trust_remote_code=False
133
  )
134
  logger.info("Tokenizer loaded successfully")
135
  except Exception as e:
136
+ if not self._is_remote_source:
137
+ logger.error(f"Failed to load tokenizer: {str(e)}")
138
+ logger.info("Retrying tokenizer load without local_files_only...")
139
  self.tokenizer = AutoTokenizer.from_pretrained(
140
  self.model_source,
141
  trust_remote_code=False
 
152
  try:
153
  config = AutoConfig.from_pretrained(
154
  self.model_source,
155
+ local_files_only=not self._is_remote_source,
156
  trust_remote_code=False
157
  )
158
 
 
367
 
368
  class SpellingModel:
369
  """Wrapper class for the Arabic spelling correction model."""
370
+
371
  def __init__(self, model_path=None):
372
  """
373
  Initialize the spelling model.
374
+
375
  Args:
376
  model_path: Path to the model directory (defaults to SPELLING_PATH)
377
  """
378
  self.model_path = Path(model_path) if model_path else SPELLING_PATH
379
  self.model = None
380
  self.device = None
381
+
382
  self._validate_path()
383
  self._load_model()
384
+
385
+ def __del__(self):
386
+ if self.model is not None:
387
+ del self.model
388
+ self.model = None
389
+ if torch.cuda.is_available():
390
+ torch.cuda.empty_cache()
391
 
392
  def _validate_path(self):
393
  """Validate that the model path exists."""
 
454
  # Load state dict
455
  pt_file = list(self.model_path.glob("*.pt"))[0]
456
  logger.info(f"Loading weights from {pt_file}...")
457
+ checkpoint = torch.load(pt_file, map_location=self.device, weights_only=True)
458
 
459
  if "model_state_dict" in checkpoint:
460
  state_dict = checkpoint["model_state_dict"]
 
647
  raise RuntimeError(f"Autocomplete prediction failed: {str(e)}")
648
 
649
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
650
  class PunctuationModel:
651
  """Wrapper class for the Arabic punctuation model."""
652
+
653
  def __init__(self, model_path=None):
654
  """
655
  Initialize the punctuation model.
656
+
657
  Args:
658
  model_path: Path to the model directory (defaults to PUNCTUATION_PATH)
659
  """
 
661
  self.model = None
662
  self.tokenizer = None
663
  self.device = None
664
+
665
  self._validate_path()
666
  self._load_model()
667
+
668
+ def __del__(self):
669
+ if self.model is not None:
670
+ del self.model
671
+ self.model = None
672
+ if self.tokenizer is not None:
673
+ del self.tokenizer
674
+ self.tokenizer = None
675
+ if torch.cuda.is_available():
676
+ torch.cuda.empty_cache()
677
 
678
  def _validate_path(self):
679
  """Validate that the model path exists and contains required files."""
src/nlp/spelling/contextual_corrector.py CHANGED
@@ -12,6 +12,7 @@
12
  import logging
13
  import torch
14
  import threading
 
15
  from typing import List, Tuple, Optional, Dict
16
 
17
  logger = logging.getLogger(__name__)
@@ -19,7 +20,7 @@ logger = logging.getLogger(__name__)
19
  # Singleton instance
20
  _instance = None
21
  _loading = False
22
- _lock = threading.Lock()
23
 
24
 
25
  class ContextualCorrector:
@@ -44,7 +45,7 @@ class ContextualCorrector:
44
  self.model.eval()
45
 
46
  # Simple cache for scores
47
- self._cache: Dict[str, float] = {}
48
  self._cache_max = 5000
49
 
50
  # Vocab for candidate filtering
@@ -63,8 +64,9 @@ class ContextualCorrector:
63
  Returns:
64
  Probability score (0.0 to 1.0) — higher = better fit
65
  """
66
- cache_key = f"{text[:200]}|{len(text)}|{position}|{word}"
67
  if cache_key in self._cache:
 
68
  return self._cache[cache_key]
69
 
70
  words = text.split()
@@ -99,13 +101,10 @@ class ContextualCorrector:
99
  if not word_tokens:
100
  return 0.0
101
 
102
- if len(word_tokens) == 1:
103
- score = probs[word_tokens[0]].item()
104
- else:
105
- score = probs[word_tokens[0]].item()
106
- for t_id in word_tokens[1:]:
107
- score *= probs[t_id].item()
108
- score = score ** (1.0 / len(word_tokens))
109
 
110
  except Exception as e:
111
  logger.warning(f"[MLM] Score error for '{word}': {e}")
@@ -113,10 +112,8 @@ class ContextualCorrector:
113
 
114
  # Cache management
115
  if len(self._cache) >= self._cache_max:
116
- # Remove oldest 20% of entries
117
- keys_to_remove = list(self._cache.keys())[:self._cache_max // 5]
118
- for k in keys_to_remove:
119
- del self._cache[k]
120
  self._cache[cache_key] = score
121
 
122
  return score
@@ -175,8 +172,8 @@ class ContextualCorrector:
175
  if corr_score > confidence_threshold:
176
  continue
177
 
178
- # Score the original word in the corrected context
179
- orig_score = self.score_word_in_context(corrected_text, i, orig_w)
180
 
181
  # If original scores better, revert
182
  if orig_score > corr_score * 10 and orig_score > 0.01:
@@ -271,9 +268,6 @@ class ContextualCorrector:
271
  if not a or not b:
272
  return 0.0
273
  max_len = max(len(a), len(b))
274
- if max_len == 0:
275
- return 1.0
276
- # Inline Levenshtein to avoid extra dependency
277
  m, n = len(a), len(b)
278
  dp = list(range(n + 1))
279
  for i in range(1, m + 1):
 
12
  import logging
13
  import torch
14
  import threading
15
+ from collections import OrderedDict
16
  from typing import List, Tuple, Optional, Dict
17
 
18
  logger = logging.getLogger(__name__)
 
20
  # Singleton instance
21
  _instance = None
22
  _loading = False
23
+ _lock = threading.RLock()
24
 
25
 
26
  class ContextualCorrector:
 
45
  self.model.eval()
46
 
47
  # Simple cache for scores
48
+ self._cache: OrderedDict[str, float] = OrderedDict()
49
  self._cache_max = 5000
50
 
51
  # Vocab for candidate filtering
 
64
  Returns:
65
  Probability score (0.0 to 1.0) — higher = better fit
66
  """
67
+ cache_key = f"{hash(text)}|{position}|{word}"
68
  if cache_key in self._cache:
69
+ self._cache.move_to_end(cache_key)
70
  return self._cache[cache_key]
71
 
72
  words = text.split()
 
101
  if not word_tokens:
102
  return 0.0
103
 
104
+ # Use first subword token probability only. For multi-subword
105
+ # words, the single [MASK] distribution only predicts the first
106
+ # subword — multiplying unrelated token probs is nonsensical.
107
+ score = probs[word_tokens[0]].item()
 
 
 
108
 
109
  except Exception as e:
110
  logger.warning(f"[MLM] Score error for '{word}': {e}")
 
112
 
113
  # Cache management
114
  if len(self._cache) >= self._cache_max:
115
+ for _ in range(self._cache_max // 5):
116
+ self._cache.popitem(last=False)
 
 
117
  self._cache[cache_key] = score
118
 
119
  return score
 
172
  if corr_score > confidence_threshold:
173
  continue
174
 
175
+ # Score the original word in its own context (not contaminated by other corrections)
176
+ orig_score = self.score_word_in_context(original_text, i, orig_w)
177
 
178
  # If original scores better, revert
179
  if orig_score > corr_score * 10 and orig_score > 0.01:
 
268
  if not a or not b:
269
  return 0.0
270
  max_len = max(len(a), len(b))
 
 
 
271
  m, n = len(a), len(b)
272
  dp = list(range(n + 1))
273
  for i in range(1, m + 1):
src/nlp/stage_locker.py CHANGED
@@ -94,6 +94,8 @@ class StageLocker:
94
  req_priority = STAGE_PRIORITY.get(requesting_stage, 0)
95
  for ls, le, owner in self.locked_spans:
96
  if start < le and end > ls:
 
 
97
  owner_priority = STAGE_PRIORITY.get(owner, 0)
98
  if owner_priority >= req_priority:
99
  if PIPELINE_DEBUG:
@@ -161,6 +163,8 @@ class StageLocker:
161
  req_priority = STAGE_PRIORITY.get(requesting_stage, 0)
162
  for ls, le, owner in self.locked_spans:
163
  if start < le and end > ls:
 
 
164
  owner_priority = STAGE_PRIORITY.get(owner, 0)
165
  if owner_priority >= req_priority:
166
  return (ls, le, owner)
 
94
  req_priority = STAGE_PRIORITY.get(requesting_stage, 0)
95
  for ls, le, owner in self.locked_spans:
96
  if start < le and end > ls:
97
+ if owner == 'protection':
98
+ return True
99
  owner_priority = STAGE_PRIORITY.get(owner, 0)
100
  if owner_priority >= req_priority:
101
  if PIPELINE_DEBUG:
 
163
  req_priority = STAGE_PRIORITY.get(requesting_stage, 0)
164
  for ls, le, owner in self.locked_spans:
165
  if start < le and end > ls:
166
+ if owner == 'protection':
167
+ return (ls, le, owner)
168
  owner_priority = STAGE_PRIORITY.get(owner, 0)
169
  if owner_priority >= req_priority:
170
  return (ls, le, owner)