youssefreda9 commited on
Commit
b33660c
·
1 Parent(s): f8cef7c

Phase 11.5: Adversarial diagnostic + 4 critical pipeline fixes

Browse files

FIX-28a: Digit guard in bidirectional spelling path
- Numbers (2020, 150) were OOV→IV hallucinated into Arabic words
- Root cause: bidirectional fix had no digit protection

FIX-28b: Prefix-swap guard in bidirectional path
- فبالتالي→وبالتالي leaked through (same stem, prefix swap)

FIX-29: Exclamation mark guard in punctuation rules
- PuncAra added ! to declarative sentences (التاريخ→التاريخ!)
- Now only allows !/؟ when text has exclamatory cue words

FIX-30: Prefix-stripping protection in spelling filter
- وبالمستشفيات→والمستشفيات (stripped ب from compound prefix)
- Blocks corrections that strip compound prefixes (وبال, فبال, etc.)

Adversarial diagnostic: 83.3% pass (30/36), 6 failures identified
Full benchmark: 320 tests across 8 datasets, 55% overall pass rate

src/app.py CHANGED
@@ -852,6 +852,26 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
852
  if _corr_stem in _DIRECTIONAL_BLOCKS.get(_orig_stem, set()):
853
  return 0.0
854
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
855
  # Ignore tokens that contain non-letters (numbers / punctuation)
856
  # Arabic letters range plus basic Latin letters.
857
  if re.search(r'[^ء-يآأإىa-zA-Z]', orig_word):
@@ -1707,6 +1727,31 @@ def analyze_text():
1707
  _rw_iv = spell_checker.vocab_manager.is_iv(_raw_words[_bi])
1708
  # Our word is OOV but model's word is IV → take model's
1709
  if not _sw_iv and _rw_iv:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1710
  logger.info(
1711
  f"[SPELLING] Bidirectional fix: "
1712
  f"'{_safe_words[_bi]}'(OOV)→'{_raw_words[_bi]}'(IV)"
 
852
  if _corr_stem in _DIRECTIONAL_BLOCKS.get(_orig_stem, set()):
853
  return 0.0
854
 
855
+ # ── FIX-30: Prefix-stripping protection ──
856
+ # Block corrections that strip a clitic prefix from a valid compound:
857
+ # وبالمستشفيات → والمستشفيات (stripped ب from وب prefix chain)
858
+ # فبالتالي → وبالتالي (swapped ف→و)
859
+ # These destroy the meaning of the prefix (بال = by the, و = and, ف = so/then)
860
+ _COMPOUND_PREFIXES = ['وبال', 'فبال', 'وال', 'فال', 'بال', 'كال', 'ول', 'فل',
861
+ 'وب', 'فب', 'وك', 'فك']
862
+ for _cpfx in _COMPOUND_PREFIXES:
863
+ if orig_word.startswith(_cpfx) and len(orig_word) > len(_cpfx) + 2:
864
+ if not corr_word.startswith(_cpfx):
865
+ # Original has compound prefix but correction doesn't — check if
866
+ # the stem is the same (meaning only the prefix was stripped)
867
+ _stem = orig_word[len(_cpfx):]
868
+ for _alt_pfx in _COMPOUND_PREFIXES + list(_CLITIC_PREFIXES) + ['ال', '']:
869
+ if corr_word.startswith(_alt_pfx):
870
+ _corr_stem2 = corr_word[len(_alt_pfx):]
871
+ if _stem == _corr_stem2 or _levenshtein(_stem, _corr_stem2) <= 1:
872
+ return 0.0
873
+ break # Only check the longest matching prefix
874
+
875
  # Ignore tokens that contain non-letters (numbers / punctuation)
876
  # Arabic letters range plus basic Latin letters.
877
  if re.search(r'[^ء-يآأإىa-zA-Z]', orig_word):
 
1727
  _rw_iv = spell_checker.vocab_manager.is_iv(_raw_words[_bi])
1728
  # Our word is OOV but model's word is IV → take model's
1729
  if not _sw_iv and _rw_iv:
1730
+ # ── FIX-28a: Digit guard for bidirectional path ──
1731
+ # Numbers (2020, 150, etc.) are OOV but must NOT be
1732
+ # replaced with Arabic words (يناير, خمسين).
1733
+ _BIDI_DIGITS = set('0123456789٠١٢٣٤٥٦٧٨٩')
1734
+ if any(c in _BIDI_DIGITS for c in _safe_words[_bi]):
1735
+ logger.info(
1736
+ f"[SPELLING] Bidirectional blocked (digit): "
1737
+ f"'{_safe_words[_bi]}'→'{_raw_words[_bi]}'"
1738
+ )
1739
+ continue
1740
+ # ── FIX-28b: Prefix-change guard ──
1741
+ # Prevent changing leading clitics: فبالتالي→وبالتالي
1742
+ # If the words share the same stem but differ only in
1743
+ # the leading prefix (و↔ف↔ب↔ل↔ك), reject.
1744
+ _CLITIC_PFX = ('و', 'ف', 'ب', 'ل', 'ك')
1745
+ _sw = _safe_words[_bi]
1746
+ _rw = _raw_words[_bi]
1747
+ if (len(_sw) > 3 and len(_rw) > 3
1748
+ and _sw[0] in _CLITIC_PFX and _rw[0] in _CLITIC_PFX
1749
+ and _sw[0] != _rw[0] and _sw[1:] == _rw[1:]):
1750
+ logger.info(
1751
+ f"[SPELLING] Bidirectional blocked (prefix swap): "
1752
+ f"'{_sw}'→'{_rw}'"
1753
+ )
1754
+ continue
1755
  logger.info(
1756
  f"[SPELLING] Bidirectional fix: "
1757
  f"'{_safe_words[_bi]}'(OOV)→'{_raw_words[_bi]}'(IV)"
src/nlp/punctuation/punctuation_rules.py CHANGED
@@ -150,6 +150,21 @@ def validate_punctuation_diff(diff: dict, full_text: str = '') -> bool:
150
  _full_has_ellipsis = full_text.rstrip().endswith('...') if full_text else False
151
 
152
  if _full_word_count >= 3 and not _full_already_has_terminal and not _full_has_ellipsis:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
153
  # Multi-word sentence without terminal punct → ALLOW
154
  logger.info(
155
  f"[PUNC-SAFETY] Allowed terminal punct for sentence "
 
150
  _full_has_ellipsis = full_text.rstrip().endswith('...') if full_text else False
151
 
152
  if _full_word_count >= 3 and not _full_already_has_terminal and not _full_has_ellipsis:
153
+ # ── FIX-29: Exclamation mark guard ──
154
+ # PuncAra sometimes adds ! to declarative sentences.
155
+ # Only allow ! if text contains exclamatory cues.
156
+ _added_punct = correction[len(orig_stripped):]
157
+ if '!' in _added_punct or '؟' in _added_punct:
158
+ # Check for question/exclamation words
159
+ _EXCL_CUES = {'يا', 'ما', 'كم', 'لا', 'هل', 'أين', 'متى',
160
+ 'كيف', 'لماذا', 'ماذا', 'أي', 'لعل', 'ليت'}
161
+ _has_cue = any(w in _EXCL_CUES for w in full_text.split())
162
+ if not _has_cue:
163
+ logger.info(
164
+ f"[PUNC-SAFETY] Blocked !/? on declarative sentence: "
165
+ f"'{original}' → '{correction}'"
166
+ )
167
+ return False
168
  # Multi-word sentence without terminal punct → ALLOW
169
  logger.info(
170
  f"[PUNC-SAFETY] Allowed terminal punct for sentence "
tests/phase10/reports/phase10_results.json CHANGED
The diff for this file is too large to render. See raw diff
 
tests/phase11/deep_dive_adversarial.py ADDED
@@ -0,0 +1,687 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Phase 11 — Deep-Dive Adversarial Diagnostic Script
3
+ ====================================================
4
+ Senior ML Engineer adversarial testing for the BAYAN pipeline.
5
+ Tests raw models SEPARATELY then the INTEGRATED pipeline.
6
+ Forcefully triggers 5 known edge-case classes and provides root cause analysis.
7
+
8
+ Usage:
9
+ python tests/phase11/deep_dive_adversarial.py [--url URL]
10
+ """
11
+ import argparse
12
+ import json
13
+ import re
14
+ import sys
15
+ import time
16
+ import difflib
17
+ from pathlib import Path
18
+ from dataclasses import dataclass, field, asdict
19
+ from typing import List, Dict, Optional, Tuple
20
+ import requests
21
+
22
+ DEFAULT_URL = "https://bayan10-bayan-api.hf.space"
23
+ REPORT_DIR = Path(__file__).parent / "reports"
24
+
25
+
26
+ # ═══════════════════════════════════════════════════════════════════════════════
27
+ # API CLIENT
28
+ # ═══════════════════════════════════════════════════════════════════════════════
29
+ class API:
30
+ def __init__(self, base):
31
+ self.base = base.rstrip('/')
32
+ self.s = requests.Session()
33
+ self.s.headers['Content-Type'] = 'application/json'
34
+
35
+ def _post(self, ep, payload, timeout=180):
36
+ t0 = time.time()
37
+ try:
38
+ r = self.s.post(f"{self.base}{ep}", json=payload, timeout=timeout)
39
+ ms = int((time.time() - t0) * 1000)
40
+ d = r.json()
41
+ d['_ms'] = ms
42
+ d['_status'] = r.status_code
43
+ return d
44
+ except requests.Timeout:
45
+ return {'error': 'TIMEOUT', '_ms': int((time.time() - t0) * 1000), '_status': 0}
46
+ except Exception as e:
47
+ return {'error': str(e), '_ms': int((time.time() - t0) * 1000), '_status': 0}
48
+
49
+ def analyze(self, text):
50
+ """Full pipeline: /api/analyze"""
51
+ return self._post('/api/analyze', {'text': text})
52
+
53
+ def grammar(self, text):
54
+ """Raw grammar only: /api/grammar"""
55
+ return self._post('/api/grammar', {'text': text})
56
+
57
+ def punctuation(self, text):
58
+ """Raw punctuation only: /api/punctuation"""
59
+ return self._post('/api/punctuation', {'text': text})
60
+
61
+
62
+ # ═══════════════════════════════════════════════════════════════════════════════
63
+ # RESULT TYPES
64
+ # ═══════════════════════════════════════════════════════════════════════════════
65
+ @dataclass
66
+ class TestVector:
67
+ """A single adversarial test case."""
68
+ id: str
69
+ category: str # Edge-case category (1-5)
70
+ subcategory: str # Specific sub-type
71
+ input_text: str
72
+ expected_behavior: str # What SHOULD happen
73
+ severity: str = "critical"
74
+
75
+ @dataclass
76
+ class IsolationResult:
77
+ """Results from testing each model in isolation + pipeline."""
78
+ vector_id: str
79
+ category: str
80
+ subcategory: str
81
+ input_text: str
82
+ expected_behavior: str
83
+ # Raw model outputs
84
+ grammar_raw_output: str = ""
85
+ grammar_raw_ms: int = 0
86
+ punctuation_raw_output: str = ""
87
+ punctuation_raw_ms: int = 0
88
+ # Pipeline output
89
+ pipeline_output: str = ""
90
+ pipeline_corrected: str = ""
91
+ pipeline_suggestions: list = field(default_factory=list)
92
+ pipeline_ms: int = 0
93
+ pipeline_timing: dict = field(default_factory=dict)
94
+ # Analysis
95
+ verdict: str = "" # PASS, FAIL_MODEL, FAIL_PIPELINE, FAIL_FILTER, ERROR
96
+ root_cause: str = ""
97
+ failure_detail: str = ""
98
+ # Integration delta
99
+ grammar_raw_changed: bool = False
100
+ pipeline_changed: bool = False
101
+ raw_vs_pipeline_match: bool = False
102
+ # Coordinate validation
103
+ span_errors: list = field(default_factory=list)
104
+ offset_drift: int = 0
105
+
106
+
107
+ # ═══════════════════════════════════════════════════════════════════════════════
108
+ # STEP 1: ADVERSARIAL TEST SUITE
109
+ # ═══════════════════════════════════════════════════════════════════════════════
110
+
111
+ ADVERSARIAL_VECTORS: List[TestVector] = [
112
+ # ──────────────────────────────────────────────────────────
113
+ # CATEGORY 1: Punctuation Coordinate Shift (OffsetMapper bug)
114
+ # ─────────────��────────────────────────────────────────────
115
+ TestVector("OFS-001", "offset_shift", "punct_after_spelling_fix",
116
+ "ذهبت الي المدرسه وقابلت صديقتي وتحدثنا كثيرا",
117
+ "Punct marks should align with corrected text, not shift 1-2 words"),
118
+ TestVector("OFS-002", "offset_shift", "punct_after_multi_word_fix",
119
+ "الطلاب ذهبو الي المكتبه لقراءة الكتب",
120
+ "Punctuation positions should not drift after spelling corrects الي→إلى and المكتبه→المكتبة"),
121
+ TestVector("OFS-003", "offset_shift", "length_changing_fix",
122
+ "انا احب القراءه كثيرا وخصوصا الكتب العلميه",
123
+ "أنا → 3 chars vs انا → 3 chars (same length), but القراءه→القراءة changes char positions"),
124
+ TestVector("OFS-004", "offset_shift", "insertion_diffs",
125
+ "كتبت رساله لصديقي واخبرته بالاخبار",
126
+ "When spelling inserts أ (اخبرته→أخبرته), downstream coords must shift +1"),
127
+ TestVector("OFS-005", "offset_shift", "multi_stage_coord_chain",
128
+ "المهندسون ذهبو الي الموقعو وبدأو العمل",
129
+ "3+ fixes in same sentence: verify offset chain doesn't accumulate drift"),
130
+
131
+ # ──────────────────────────────────────────────────────────
132
+ # CATEGORY 2: Prefixes and Prepositions (السوابق وحروف الجر)
133
+ # ──────────────────────────────────────────────────────────
134
+ TestVector("PFX-001", "prefix_handling", "wa_prefix",
135
+ "وبالمستشفيات الكبيره يعالجون المرضي",
136
+ "وبالمستشفيات should NOT be split or hallucinated — it's a valid prefixed word"),
137
+ TestVector("PFX-002", "prefix_handling", "ba_prefix",
138
+ "بالمدرسه الجديده تعلمت كثيرا",
139
+ "بالمدرسه → بالمدرسة (only ه→ة fix, keep ب prefix)"),
140
+ TestVector("PFX-003", "prefix_handling", "li_prefix",
141
+ "للطالبات المجتهدات جوائز قيمه",
142
+ "للطالبات should be preserved intact"),
143
+ TestVector("PFX-004", "prefix_handling", "ka_prefix",
144
+ "كالعاده ذهبت الي العمل",
145
+ "كالعادة — ه→ة fix should work through ك+ال prefix"),
146
+ TestVector("PFX-005", "prefix_handling", "fa_prefix",
147
+ "فبالتالي يجب ان نعمل بجد",
148
+ "فبالتالي is valid, should not be split or changed"),
149
+ TestVector("PFX-006", "prefix_handling", "complex_prefix_chain",
150
+ "وبالاضافه الي ذلك فان المشكله كبيره",
151
+ "وبالإضافة إلى — hamza fix through وب prefix, plus ه→ة fixes"),
152
+ TestVector("PFX-007", "prefix_handling", "wa_attached_to_error",
153
+ "والمهندسين بنو المبني بسرعه",
154
+ "والمهندسين should stay, بنو→بنوا (grammar), المبني→المبنى, بسرعه→بسرعة"),
155
+ TestVector("PFX-008", "prefix_handling", "attached_pronoun",
156
+ "كتابه قديم وصفحاته ممزقه",
157
+ "كتابه (his book) — ه is pronoun, NOT ta marbuta. Must NOT become كتابة"),
158
+ TestVector("PFX-009", "prefix_handling", "pronoun_vs_tamarbuta",
159
+ "قرأته بعنايه فتأملته جيدا",
160
+ "قرأته and فتأملته — ته is pronoun suffix, NOT ta marbuta"),
161
+ TestVector("PFX-010", "prefix_handling", "merged_preposition",
162
+ "ذهبتللمدرسه صباحا",
163
+ "Missing space: ذهبت+ل+المدرسه — should split and fix"),
164
+
165
+ # ──────────────────────────────────────────────────────────
166
+ # CATEGORY 3: StageLocker / Directional Flow
167
+ # ──────────────────────────────────────────────────────────
168
+ TestVector("SLK-001", "stagelocker", "spelling_lock_grammar_override",
169
+ "السياره جميل جدا",
170
+ "Spelling fixes السياره→السيارة, Grammar should still fix جميل→جميلة (Phase 11 hierarchical)"),
171
+ TestVector("SLK-002", "stagelocker", "grammar_not_overwrite_spelling",
172
+ "المدرسه كبير وجميل",
173
+ "Spelling: المدرسه→المدرسة. Grammar should NOT revert to المدرسه. Should add ة to كبير/جميل"),
174
+ TestVector("SLK-003", "stagelocker", "punct_respects_grammar_lock",
175
+ "الطلاب ذهبوا الى المكتبة لقراءة الكتب الجديدة",
176
+ "Grammar unchanged (correct text). Punctuation should NOT move/corrupt words"),
177
+ TestVector("SLK-004", "stagelocker", "overlapping_patches",
178
+ "الاطفال يلعبون بالكره في الحديقه الكبيره",
179
+ "Spelling: الاطفال→الأطفال, بالكره→بالكرة, الحديقه→الحديقة, الكبيره→الكبيرة — multiple overlapping spans"),
180
+ TestVector("SLK-005", "stagelocker", "backward_overwrite",
181
+ "هذه المدينه جميله وهادئه",
182
+ "Three ه→ة fixes. Grammar must NOT overwrite any of them back to ه"),
183
+ TestVector("SLK-006", "stagelocker", "grammar_overrides_spelling_adjacent",
184
+ "الطالبه ذكي في المدرسه",
185
+ "Spelling fixes ه→ة, Grammar must fix gender (ذكي→ذكية) on ADJACENT word"),
186
+
187
+ # ──────────────────────────────────────────────────────────
188
+ # CATEGORY 4: Overcorrection & Safety Filter Analysis
189
+ # ──────────────────────────────────────────────────────────
190
+ TestVector("OVR-001", "overcorrection", "correct_text_unchanged",
191
+ "ذهب الطالب إلى المكتبة وقرأ كتاباً مفيداً",
192
+ "Perfectly correct text — should return UNCHANGED"),
193
+ TestVector("OVR-002", "overcorrection", "correct_grammar_unchanged",
194
+ "المهندسون يعملون في المصنع الكبير بجدٍ واجتهاد",
195
+ "Correct SV agreement + tanween — must not be modified"),
196
+ TestVector("OVR-003", "overcorrection", "jaccard_threshold_test",
197
+ "جالس في البيت يقرأ الصحيفة",
198
+ "Correct text. Grammar should not hallucinate جالس→جاكسون (Jaccard < 0.3)"),
199
+ TestVector("OVR-004", "overcorrection", "meaning_change_block",
200
+ "وكان الجو صحواً اليوم",
201
+ "وكان must NOT become وكأن (directional block). Correct as-is"),
202
+ TestVector("OVR-005", "overcorrection", "rare_word_dampening",
203
+ "الفيلسوف سقراط تأمل الحقيقة",
204
+ "سقراط is a rare valid name — spelling should not 'correct' it"),
205
+ TestVector("OVR-006", "overcorrection", "filter_over_blocking",
206
+ "انا ذاهب الي المدرسه",
207
+ "IV-IV guard should NOT block انا→أنا or الي→إلى (these are whitelist entries)"),
208
+ TestVector("OVR-007", "overcorrection", "tanween_preservation",
209
+ "قرأت كتاباً جميلاً عن التاريخ",
210
+ "Tanween (ً) must be preserved — grammar should not strip it"),
211
+ TestVector("OVR-008", "overcorrection", "correct_feminine_form",
212
+ "المعلمة الجديدة شرحت الدرس",
213
+ "Already correct — must NOT change المعلمة to المعلمه"),
214
+ TestVector("OVR-009", "overcorrection", "brackets_protection",
215
+ "الفصل الأول (المقدمة) يتناول الموضوع",
216
+ "Brackets must be preserved. Grammar should not unbalance them"),
217
+ TestVector("OVR-010", "overcorrection", "digits_protection",
218
+ "تأسست الشركة عام 2020 وبلغ عدد موظفيها 150 شخصاً",
219
+ "Digits must NOT be corrupted by grammar model"),
220
+
221
+ # ──────────────────────────────────────────────────────────
222
+ # CATEGORY 5: Model Mismatch (Spelling→Grammar friction)
223
+ # ──────────────────────────────────────────────────────────
224
+ TestVector("MIS-001", "model_mismatch", "spelling_output_breaks_grammar",
225
+ "الطالبه المجتهده حصلت علي درجات عاليه",
226
+ "Spelling: 4x ه→ة. Grammar should NOT break on the corrected output"),
227
+ TestVector("MIS-002", "model_mismatch", "grammar_undoes_spelling",
228
+ "ذهبنا الي المكتبه القريبه",
229
+ "Spelling fixes الي→إلى, المكتبه→المكتبة. Grammar must NOT revert these"),
230
+ TestVector("MIS-003", "model_mismatch", "cascading_length_change",
231
+ "اريد ان اذهب الي المدرسه لانني احب القراءه",
232
+ "Many hamza + ه→ة fixes. Total text length changes significantly — grammar must handle shifted text"),
233
+ TestVector("MIS-004", "model_mismatch", "grammar_hallucination_on_corrected",
234
+ "الاولاد يلعبون والبنات يذهبن",
235
+ "Spelling: الاولاد→الأولاد. Grammar should NOT hallucinate on already-correct يلعبون/يذهبن"),
236
+ TestVector("MIS-005", "model_mismatch", "punc_on_grammar_corrected",
237
+ "السيارات سريعة والطرق واسعه",
238
+ "After spelling (واسعه→واسعة), punctuation should add period at end, not corrupt words"),
239
+ ]
240
+
241
+
242
+ # ═══════════════════════════════════════════════════════════════════════════════
243
+ # STEP 2: ISOLATION vs INTEGRATION MATRIX
244
+ # ═══════════════════════════════════════════════════════════════════════════════
245
+
246
+ def strip_diacritics(text: str) -> str:
247
+ return re.sub(r'[\u064B-\u065F\u0670]', '', text)
248
+
249
+ def normalize(text: str) -> str:
250
+ t = strip_diacritics(text)
251
+ t = re.sub(r'\s+', ' ', t).strip()
252
+ return t
253
+
254
+ def validate_spans(original: str, suggestions: list) -> List[str]:
255
+ """Validate all suggestion spans point to correct original text."""
256
+ errors = []
257
+ for sg in suggestions:
258
+ start = sg.get('start', 0)
259
+ end = sg.get('end', 0)
260
+ expected_orig = sg.get('original', '')
261
+ actual_slice = original[start:end]
262
+ if actual_slice != expected_orig:
263
+ errors.append(
264
+ f"SPAN[{start}:{end}] expected='{expected_orig}' actual='{actual_slice}' "
265
+ f"(drift={len(actual_slice) - len(expected_orig)})"
266
+ )
267
+ return errors
268
+
269
+ def compute_offset_drift(original: str, suggestions: list) -> int:
270
+ """Compute max drift between expected and actual span content."""
271
+ max_drift = 0
272
+ for sg in suggestions:
273
+ start = sg.get('start', 0)
274
+ end = sg.get('end', 0)
275
+ expected = sg.get('original', '')
276
+ actual = original[start:end]
277
+ if actual != expected:
278
+ # Try to find the expected text nearby
279
+ for delta in range(-5, 6):
280
+ try_start = max(0, start + delta)
281
+ try_end = min(len(original), end + delta)
282
+ if original[try_start:try_end] == expected:
283
+ max_drift = max(max_drift, abs(delta))
284
+ break
285
+ else:
286
+ max_drift = max(max_drift, 99) # Not found at all
287
+ return max_drift
288
+
289
+
290
+ def run_isolation_matrix(api: API, vectors: List[TestVector]) -> List[IsolationResult]:
291
+ """Run each vector through raw models + pipeline and compare."""
292
+ results = []
293
+
294
+ for i, v in enumerate(vectors):
295
+ print(f"\n [{i+1}/{len(vectors)}] {v.id} ({v.category}/{v.subcategory})")
296
+ print(f" Input: {v.input_text[:80]}...")
297
+
298
+ r = IsolationResult(
299
+ vector_id=v.id,
300
+ category=v.category,
301
+ subcategory=v.subcategory,
302
+ input_text=v.input_text,
303
+ expected_behavior=v.expected_behavior,
304
+ )
305
+
306
+ # ── A) Raw Grammar Only ──
307
+ try:
308
+ resp_g = api.grammar(v.input_text)
309
+ r.grammar_raw_output = resp_g.get('corrected_text', resp_g.get('corrected', ''))
310
+ r.grammar_raw_ms = resp_g.get('_ms', 0)
311
+ r.grammar_raw_changed = (r.grammar_raw_output != v.input_text)
312
+ print(f" [A] Grammar raw: {'CHANGED' if r.grammar_raw_changed else 'unchanged'} ({r.grammar_raw_ms}ms)")
313
+ if r.grammar_raw_changed:
314
+ print(f" → {r.grammar_raw_output[:80]}")
315
+ except Exception as e:
316
+ print(f" [A] Grammar raw: ERROR ({e})")
317
+
318
+ # ── B) Raw Punctuation Only ──
319
+ try:
320
+ resp_p = api.punctuation(v.input_text)
321
+ r.punctuation_raw_output = resp_p.get('corrected_text', resp_p.get('corrected', ''))
322
+ r.punctuation_raw_ms = resp_p.get('_ms', 0)
323
+ punct_changed = (r.punctuation_raw_output != v.input_text)
324
+ print(f" [B] Punct raw: {'CHANGED' if punct_changed else 'unchanged'} ({r.punctuation_raw_ms}ms)")
325
+ if punct_changed:
326
+ print(f" → {r.punctuation_raw_output[:80]}")
327
+ except Exception as e:
328
+ print(f" [B] Punct raw: ERROR ({e})")
329
+
330
+ # ── C) Full Pipeline ──
331
+ try:
332
+ resp = api.analyze(v.input_text)
333
+ r.pipeline_ms = resp.get('_ms', 0)
334
+ r.pipeline_timing = resp.get('timing_ms', {})
335
+ r.pipeline_corrected = resp.get('corrected', '')
336
+ r.pipeline_output = r.pipeline_corrected
337
+ r.pipeline_suggestions = resp.get('suggestions', [])
338
+ r.pipeline_changed = (r.pipeline_corrected != resp.get('original', v.input_text))
339
+ print(f" [C] Pipeline: {'CHANGED' if r.pipeline_changed else 'unchanged'} ({r.pipeline_ms}ms) [{len(r.pipeline_suggestions)} suggestions]")
340
+ if r.pipeline_changed:
341
+ print(f" → {r.pipeline_corrected[:80]}")
342
+
343
+ # ── Span validation ──
344
+ original = resp.get('original', v.input_text)
345
+ r.span_errors = validate_spans(original, r.pipeline_suggestions)
346
+ r.offset_drift = compute_offset_drift(original, r.pipeline_suggestions)
347
+ if r.span_errors:
348
+ print(f" ⚠️ SPAN ERRORS: {r.span_errors}")
349
+ if r.offset_drift > 0:
350
+ print(f" ⚠️ OFFSET DRIFT: {r.offset_drift} chars")
351
+
352
+ # ── Raw vs Pipeline comparison ──
353
+ norm_grammar = normalize(r.grammar_raw_output)
354
+ norm_pipeline = normalize(r.pipeline_corrected)
355
+ r.raw_vs_pipeline_match = (norm_grammar == norm_pipeline)
356
+
357
+ except Exception as e:
358
+ r.verdict = "ERROR"
359
+ r.failure_detail = str(e)
360
+ print(f" [C] Pipeline: ERROR ({e})")
361
+
362
+ # ── Verdict determination ──
363
+ _classify_result(v, r)
364
+
365
+ icon = {"PASS": "✅", "FAIL_MODEL": "🔴", "FAIL_PIPELINE": "🟡",
366
+ "FAIL_FILTER": "🟠", "FAIL_OFFSET": "🔵", "ERROR": "💥"}.get(r.verdict, "❓")
367
+ print(f" {icon} {r.verdict}: {r.root_cause}")
368
+
369
+ results.append(r)
370
+
371
+ return results
372
+
373
+
374
+ def _classify_result(v: TestVector, r: IsolationResult):
375
+ """Classify the test result into a verdict + root cause."""
376
+ if r.verdict == "ERROR":
377
+ return
378
+
379
+ cat = v.category
380
+
381
+ # ── Category 1: Offset shift ──
382
+ if cat == "offset_shift":
383
+ if r.span_errors:
384
+ r.verdict = "FAIL_OFFSET"
385
+ r.root_cause = f"OffsetMapper coordinate drift ({r.offset_drift} chars): {r.span_errors[0]}"
386
+ r.failure_detail = "; ".join(r.span_errors)
387
+ elif r.offset_drift > 0:
388
+ r.verdict = "FAIL_OFFSET"
389
+ r.root_cause = f"Offset drift detected: {r.offset_drift} chars"
390
+ else:
391
+ r.verdict = "PASS"
392
+ r.root_cause = "Spans aligned correctly"
393
+ return
394
+
395
+ # ── Category 2: Prefix handling ──
396
+ if cat == "prefix_handling":
397
+ output = r.pipeline_corrected
398
+ inp = v.input_text
399
+ # Check for catastrophic prefix destruction
400
+ if _check_prefix_destruction(inp, output):
401
+ r.verdict = "FAIL_MODEL"
402
+ r.root_cause = "Prefix word was split, hallucinated, or destroyed"
403
+ r.failure_detail = f"Input had prefixed words that were corrupted in output"
404
+ elif r.span_errors:
405
+ r.verdict = "FAIL_OFFSET"
406
+ r.root_cause = f"Span errors in prefix test: {r.span_errors[0]}"
407
+ else:
408
+ r.verdict = "PASS"
409
+ r.root_cause = "Prefixed words handled correctly"
410
+ return
411
+
412
+ # ── Category 3: StageLocker ──
413
+ if cat == "stagelocker":
414
+ # Check if grammar fix was blocked by spelling lock
415
+ sugg_stages = [s.get('type', '') for s in r.pipeline_suggestions]
416
+ if v.subcategory == "spelling_lock_grammar_override":
417
+ has_grammar = 'grammar' in sugg_stages
418
+ has_spelling = 'spelling' in sugg_stages
419
+ if has_spelling and not has_grammar:
420
+ r.verdict = "FAIL_PIPELINE"
421
+ r.root_cause = "StageLocker STILL blocking grammar override of spelling (Phase 11 fix not deployed?)"
422
+ elif has_spelling and has_grammar:
423
+ r.verdict = "PASS"
424
+ r.root_cause = "Grammar successfully overrode spelling lock"
425
+ else:
426
+ r.verdict = "PASS"
427
+ r.root_cause = "Pipeline handled correctly"
428
+ elif v.subcategory == "backward_overwrite":
429
+ # Check that spelling fixes were preserved
430
+ if 'ه' in r.pipeline_corrected and 'المدين' in r.pipeline_corrected:
431
+ # Check if any ه→ة fix was reverted
432
+ for sg in r.pipeline_suggestions:
433
+ if sg.get('type') == 'grammar':
434
+ orig_sg = sg.get('original', '')
435
+ corr_sg = sg.get('correction', '')
436
+ if orig_sg.endswith('ة') and corr_sg.endswith('ه'):
437
+ r.verdict = "FAIL_PIPELINE"
438
+ r.root_cause = f"Grammar reverted spelling fix: '{orig_sg}'→'{corr_sg}'"
439
+ return
440
+ r.verdict = "PASS"
441
+ r.root_cause = "No backward overwrite detected"
442
+ else:
443
+ r.verdict = "PASS"
444
+ r.root_cause = "StageLocker operating correctly"
445
+ return
446
+
447
+ # ── Category 4: Overcorrection ──
448
+ if cat == "overcorrection":
449
+ if v.subcategory in ("correct_text_unchanged", "correct_grammar_unchanged",
450
+ "correct_feminine_form", "tanween_preservation"):
451
+ if r.pipeline_changed:
452
+ changes = [f"{s.get('original','')}→{s.get('correction','')}"
453
+ for s in r.pipeline_suggestions]
454
+ r.verdict = "FAIL_FILTER"
455
+ r.root_cause = f"Correct text was modified: {changes[:3]}"
456
+ r.failure_detail = f"All changes: {changes}"
457
+ else:
458
+ r.verdict = "PASS"
459
+ r.root_cause = "Correct text preserved"
460
+ elif v.subcategory == "meaning_change_block":
461
+ # Check if وكان was changed to وكأن
462
+ if 'وكأن' in r.pipeline_corrected:
463
+ r.verdict = "FAIL_FILTER"
464
+ r.root_cause = "Directional block failed: وكان→وكأن leaked through"
465
+ else:
466
+ r.verdict = "PASS"
467
+ r.root_cause = "Meaning preserved"
468
+ elif v.subcategory == "digits_protection":
469
+ if not re.search(r'2020', r.pipeline_corrected):
470
+ r.verdict = "FAIL_FILTER"
471
+ r.root_cause = "Digits were corrupted"
472
+ elif not re.search(r'150', r.pipeline_corrected):
473
+ r.verdict = "FAIL_FILTER"
474
+ r.root_cause = "Digits were corrupted"
475
+ else:
476
+ r.verdict = "PASS"
477
+ r.root_cause = "Digits preserved"
478
+ elif v.subcategory == "brackets_protection":
479
+ if '(' in v.input_text and '(' not in r.pipeline_corrected:
480
+ r.verdict = "FAIL_FILTER"
481
+ r.root_cause = "Brackets were removed"
482
+ else:
483
+ r.verdict = "PASS"
484
+ r.root_cause = "Brackets preserved"
485
+ else:
486
+ if r.pipeline_changed and not r.pipeline_suggestions:
487
+ r.verdict = "FAIL_FILTER"
488
+ r.root_cause = "Text changed without suggestions"
489
+ else:
490
+ r.verdict = "PASS"
491
+ r.root_cause = "No overcorrection detected"
492
+ return
493
+
494
+ # ── Category 5: Model mismatch ──
495
+ if cat == "model_mismatch":
496
+ if v.subcategory == "grammar_undoes_spelling":
497
+ # Check if spelling corrections were preserved
498
+ if 'الي' in r.pipeline_corrected or 'المكتبه' in r.pipeline_corrected:
499
+ r.verdict = "FAIL_PIPELINE"
500
+ r.root_cause = "Grammar undid spelling corrections"
501
+ else:
502
+ r.verdict = "PASS"
503
+ r.root_cause = "Spelling corrections preserved through grammar"
504
+ elif v.subcategory == "spelling_output_breaks_grammar":
505
+ if r.verdict != "ERROR" and r.pipeline_corrected:
506
+ r.verdict = "PASS"
507
+ r.root_cause = "Grammar handled spelling-corrected input"
508
+ else:
509
+ r.verdict = "FAIL_PIPELINE"
510
+ r.root_cause = "Grammar broke on spelling output"
511
+ else:
512
+ r.verdict = "PASS"
513
+ r.root_cause = "No model mismatch detected"
514
+ return
515
+
516
+ r.verdict = "PASS"
517
+ r.root_cause = "No issues detected"
518
+
519
+
520
+ def _check_prefix_destruction(input_text: str, output: str) -> bool:
521
+ """Check if prefixed words were catastrophically destroyed."""
522
+ PREFIXED_PATTERNS = [
523
+ (r'\bوبال\w+', 'وبال'),
524
+ (r'\bبال\w+', 'بال'),
525
+ (r'\bكال\w+', 'كال'),
526
+ (r'\bفبال\w+', 'فبال'),
527
+ (r'\bلل\w+', 'لل'),
528
+ ]
529
+ input_words = set(input_text.split())
530
+ output_words = set(output.split())
531
+
532
+ for word in input_words:
533
+ for pattern, prefix in PREFIXED_PATTERNS:
534
+ if re.match(pattern, word):
535
+ # This word has a prefix — check if it survived
536
+ # Allow ه→ة change but not complete destruction
537
+ stripped_orig = re.sub(r'[ةه]$', '', word)
538
+ found = False
539
+ for ow in output_words:
540
+ stripped_out = re.sub(r'[ةه]$', '', ow)
541
+ if stripped_orig == stripped_out or ow.startswith(prefix):
542
+ found = True
543
+ break
544
+ if not found:
545
+ return True # Prefixed word was destroyed
546
+ return False
547
+
548
+
549
+ # ═══════════════════════════════════════════════════════════════════════════════
550
+ # STEP 3: ANALYSIS & REPORTING
551
+ # ═══════════════════════════════════════════════════════════════════════════════
552
+
553
+ def generate_report(results: List[IsolationResult], url: str) -> dict:
554
+ """Generate comprehensive analysis report."""
555
+
556
+ # Aggregate
557
+ total = len(results)
558
+ by_verdict = {}
559
+ by_category = {}
560
+ offset_issues = []
561
+ filter_issues = []
562
+ pipeline_issues = []
563
+ model_issues = []
564
+
565
+ for r in results:
566
+ by_verdict[r.verdict] = by_verdict.get(r.verdict, 0) + 1
567
+ cat = r.category
568
+ if cat not in by_category:
569
+ by_category[cat] = {"total": 0, "pass": 0, "fail": 0}
570
+ by_category[cat]["total"] += 1
571
+ if r.verdict == "PASS":
572
+ by_category[cat]["pass"] += 1
573
+ else:
574
+ by_category[cat]["fail"] += 1
575
+
576
+ if r.verdict == "FAIL_OFFSET":
577
+ offset_issues.append(r)
578
+ elif r.verdict == "FAIL_FILTER":
579
+ filter_issues.append(r)
580
+ elif r.verdict == "FAIL_PIPELINE":
581
+ pipeline_issues.append(r)
582
+ elif r.verdict == "FAIL_MODEL":
583
+ model_issues.append(r)
584
+
585
+ pass_count = by_verdict.get("PASS", 0)
586
+ pass_rate = pass_count / total if total > 0 else 0
587
+
588
+ report = {
589
+ "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
590
+ "target": url,
591
+ "total_vectors": total,
592
+ "pass_rate": round(pass_rate * 100, 1),
593
+ "by_verdict": by_verdict,
594
+ "by_category": by_category,
595
+ "offset_issues": [asdict(r) for r in offset_issues],
596
+ "filter_issues": [asdict(r) for r in filter_issues],
597
+ "pipeline_issues": [asdict(r) for r in pipeline_issues],
598
+ "model_issues": [asdict(r) for r in model_issues],
599
+ "results": [asdict(r) for r in results],
600
+ }
601
+ return report
602
+
603
+
604
+ def print_report(results: List[IsolationResult], report: dict):
605
+ """Print detailed report to console."""
606
+ print(f"\n{'='*70}")
607
+ print("ADVERSARIAL DIAGNOSTIC REPORT")
608
+ print(f"{'='*70}")
609
+ print(f"\n## Overall: {report['pass_rate']}% pass rate ({report['total_vectors']} vectors)")
610
+ print(f"\n| Verdict | Count |")
611
+ print(f"|-----------------|-------|")
612
+ for verdict, count in sorted(report['by_verdict'].items()):
613
+ icon = {"PASS": "✅", "FAIL_MODEL": "🔴", "FAIL_PIPELINE": "🟡",
614
+ "FAIL_FILTER": "🟠", "FAIL_OFFSET": "🔵", "ERROR": "💥"}.get(verdict, "❓")
615
+ print(f"| {icon} {verdict:<14} | {count:>5} |")
616
+
617
+ print(f"\n## By Edge-Case Category")
618
+ print(f"| Category | Total | Pass | Fail | Rate |")
619
+ print(f"|----------------------|-------|------|------|------|")
620
+ for cat, data in sorted(report['by_category'].items()):
621
+ rate = data['pass'] / data['total'] * 100 if data['total'] > 0 else 0
622
+ print(f"| {cat:<20} | {data['total']:>5} | {data['pass']:>4} | {data['fail']:>4} | {rate:>3.0f}% |")
623
+
624
+ # ── Failure details ──
625
+ failures = [r for r in results if r.verdict != "PASS"]
626
+ if failures:
627
+ print(f"\n## Failure Details ({len(failures)} issues)")
628
+ print(f"{'─'*70}")
629
+ for r in failures:
630
+ icon = {"FAIL_MODEL": "🔴", "FAIL_PIPELINE": "🟡",
631
+ "FAIL_FILTER": "🟠", "FAIL_OFFSET": "🔵", "ERROR": "💥"}.get(r.verdict, "❓")
632
+ print(f"\n {icon} [{r.vector_id}] {r.category}/{r.subcategory}")
633
+ print(f" Input: {r.input_text[:70]}")
634
+ print(f" Pipeline: {r.pipeline_corrected[:70]}")
635
+ print(f" Cause: {r.root_cause}")
636
+ if r.span_errors:
637
+ for se in r.span_errors[:3]:
638
+ print(f" Span: {se}")
639
+
640
+ # ── OffsetMapper analysis ──
641
+ all_drifts = [r.offset_drift for r in results if r.offset_drift > 0]
642
+ if all_drifts:
643
+ print(f"\n## OffsetMapper Drift Analysis")
644
+ print(f" Vectors with drift: {len(all_drifts)}/{len(results)}")
645
+ print(f" Max drift: {max(all_drifts)} chars")
646
+ print(f" Affected vectors: {[r.vector_id for r in results if r.offset_drift > 0]}")
647
+
648
+
649
+ # ═══════════════════════════════════════════════════════════════════════════════
650
+ # MAIN
651
+ # ═══════════════════════════════════════════════════════════════════════════════
652
+
653
+ def main():
654
+ parser = argparse.ArgumentParser(description="Phase 11 Adversarial Diagnostic")
655
+ parser.add_argument("--url", default=DEFAULT_URL)
656
+ parser.add_argument("--category", default=None, help="Filter by category (offset_shift, prefix_handling, etc.)")
657
+ args = parser.parse_args()
658
+
659
+ api = API(args.url)
660
+
661
+ # Filter vectors if category specified
662
+ vectors = ADVERSARIAL_VECTORS
663
+ if args.category:
664
+ vectors = [v for v in vectors if v.category == args.category]
665
+
666
+ print(f"\n{'='*70}")
667
+ print(f"BAYAN ADVERSARIAL DIAGNOSTIC — Phase 11")
668
+ print(f"{'='*70}")
669
+ print(f" Target: {args.url}")
670
+ print(f" Vectors: {len(vectors)}")
671
+ print(f" Categories: {sorted(set(v.category for v in vectors))}")
672
+ print(f"{'='*70}")
673
+
674
+ results = run_isolation_matrix(api, vectors)
675
+ report = generate_report(results, args.url)
676
+ print_report(results, report)
677
+
678
+ # Save JSON report
679
+ REPORT_DIR.mkdir(parents=True, exist_ok=True)
680
+ out_path = REPORT_DIR / "adversarial_diagnostic.json"
681
+ with open(out_path, 'w', encoding='utf-8') as f:
682
+ json.dump(report, f, ensure_ascii=False, indent=2)
683
+ print(f"\n[DIAG] Full report → {out_path}")
684
+
685
+
686
+ if __name__ == "__main__":
687
+ main()
tests/phase11/reports/adversarial_diagnostic.json ADDED
@@ -0,0 +1,2654 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "timestamp": "2026-06-23T17:47:48Z",
3
+ "target": "https://bayan10-bayan-api.hf.space",
4
+ "total_vectors": 36,
5
+ "pass_rate": 83.3,
6
+ "by_verdict": {
7
+ "PASS": 30,
8
+ "FAIL_MODEL": 2,
9
+ "FAIL_FILTER": 4
10
+ },
11
+ "by_category": {
12
+ "offset_shift": {
13
+ "total": 5,
14
+ "pass": 5,
15
+ "fail": 0
16
+ },
17
+ "prefix_handling": {
18
+ "total": 10,
19
+ "pass": 8,
20
+ "fail": 2
21
+ },
22
+ "stagelocker": {
23
+ "total": 6,
24
+ "pass": 6,
25
+ "fail": 0
26
+ },
27
+ "overcorrection": {
28
+ "total": 10,
29
+ "pass": 6,
30
+ "fail": 4
31
+ },
32
+ "model_mismatch": {
33
+ "total": 5,
34
+ "pass": 5,
35
+ "fail": 0
36
+ }
37
+ },
38
+ "offset_issues": [],
39
+ "filter_issues": [
40
+ {
41
+ "vector_id": "OVR-002",
42
+ "category": "overcorrection",
43
+ "subcategory": "correct_grammar_unchanged",
44
+ "input_text": "المهندسون يعملون في المصنع الكبير بجدٍ واجتهاد",
45
+ "expected_behavior": "Correct SV agreement + tanween — must not be modified",
46
+ "grammar_raw_output": "المهندسون يعملون في المصنع الكبير بجد واجتهاد",
47
+ "grammar_raw_ms": 4220,
48
+ "punctuation_raw_output": "المهندسون يعملون في المصنع الكبير بجدٍ واجتهاد.",
49
+ "punctuation_raw_ms": 3964,
50
+ "pipeline_output": "المهندسون يعملون في المصنع الكبير بجدٍ واجتهاد.",
51
+ "pipeline_corrected": "المهندسون يعملون في المصنع الكبير بجدٍ واجتهاد.",
52
+ "pipeline_suggestions": [
53
+ {
54
+ "alternatives": [],
55
+ "confidence": 0.8,
56
+ "correction": "واجتهاد.",
57
+ "end": 46,
58
+ "id": "2ee09f18-187f-42a4-83bc-e81d696ffa80",
59
+ "locked": true,
60
+ "original": "واجتهاد",
61
+ "priority": 2,
62
+ "start": 39,
63
+ "type": "punctuation"
64
+ }
65
+ ],
66
+ "pipeline_ms": 10058,
67
+ "pipeline_timing": {
68
+ "grammar_ms": 948,
69
+ "punctuation_ms": 833,
70
+ "spelling_ms": 5345,
71
+ "total_ms": 7132
72
+ },
73
+ "verdict": "FAIL_FILTER",
74
+ "root_cause": "Correct text was modified: ['واجتهاد→واجتهاد.']",
75
+ "failure_detail": "All changes: ['واجتهاد→واجتهاد.']",
76
+ "grammar_raw_changed": true,
77
+ "pipeline_changed": true,
78
+ "raw_vs_pipeline_match": false,
79
+ "span_errors": [],
80
+ "offset_drift": 0
81
+ },
82
+ {
83
+ "vector_id": "OVR-007",
84
+ "category": "overcorrection",
85
+ "subcategory": "tanween_preservation",
86
+ "input_text": "قرأت كتاباً جميلاً عن التاريخ",
87
+ "expected_behavior": "Tanween (ً) must be preserved — grammar should not strip it",
88
+ "grammar_raw_output": "قرأت كتابا جميلا عن التاريخ",
89
+ "grammar_raw_ms": 5403,
90
+ "punctuation_raw_output": "قرأت كتاباً جميلاً عن التاريخ!",
91
+ "punctuation_raw_ms": 4864,
92
+ "pipeline_output": "قرأت كتاباً جميلاً عن التاريخ!",
93
+ "pipeline_corrected": "قرأت كتاباً جميلاً عن التاريخ!",
94
+ "pipeline_suggestions": [
95
+ {
96
+ "alternatives": [],
97
+ "confidence": 0.8,
98
+ "correction": "التاريخ!",
99
+ "end": 29,
100
+ "id": "a516b100-8451-4611-a618-a3c914e05928",
101
+ "locked": true,
102
+ "original": "التاريخ",
103
+ "priority": 2,
104
+ "start": 22,
105
+ "type": "punctuation"
106
+ }
107
+ ],
108
+ "pipeline_ms": 5350,
109
+ "pipeline_timing": {
110
+ "grammar_ms": 1185,
111
+ "punctuation_ms": 730,
112
+ "spelling_ms": 2609,
113
+ "total_ms": 4529
114
+ },
115
+ "verdict": "FAIL_FILTER",
116
+ "root_cause": "Correct text was modified: ['التاريخ→التاريخ!']",
117
+ "failure_detail": "All changes: ['التاريخ→التاريخ!']",
118
+ "grammar_raw_changed": true,
119
+ "pipeline_changed": true,
120
+ "raw_vs_pipeline_match": false,
121
+ "span_errors": [],
122
+ "offset_drift": 0
123
+ },
124
+ {
125
+ "vector_id": "OVR-008",
126
+ "category": "overcorrection",
127
+ "subcategory": "correct_feminine_form",
128
+ "input_text": "المعلمة الجديدة شرحت الدرس",
129
+ "expected_behavior": "Already correct — must NOT change المعلمة to المعلمه",
130
+ "grammar_raw_output": "المعلمة الجديدة شرحت الدرس",
131
+ "grammar_raw_ms": 4415,
132
+ "punctuation_raw_output": "المعلمة الجديدة شرحت الدرس.",
133
+ "punctuation_raw_ms": 1481,
134
+ "pipeline_output": "المعلمة الجديدة شرحت الدرس.",
135
+ "pipeline_corrected": "المعلمة الجديدة شرحت الدرس.",
136
+ "pipeline_suggestions": [
137
+ {
138
+ "alternatives": [],
139
+ "confidence": 0.8,
140
+ "correction": "الدرس.",
141
+ "end": 26,
142
+ "id": "31693a38-f540-437b-977b-f541204fa564",
143
+ "locked": true,
144
+ "original": "الدرس",
145
+ "priority": 2,
146
+ "start": 21,
147
+ "type": "punctuation"
148
+ }
149
+ ],
150
+ "pipeline_ms": 6679,
151
+ "pipeline_timing": {
152
+ "grammar_ms": 852,
153
+ "punctuation_ms": 643,
154
+ "spelling_ms": 1817,
155
+ "total_ms": 3314
156
+ },
157
+ "verdict": "FAIL_FILTER",
158
+ "root_cause": "Correct text was modified: ['الدرس→الدرس.']",
159
+ "failure_detail": "All changes: ['الدرس→الدرس.']",
160
+ "grammar_raw_changed": false,
161
+ "pipeline_changed": true,
162
+ "raw_vs_pipeline_match": false,
163
+ "span_errors": [],
164
+ "offset_drift": 0
165
+ },
166
+ {
167
+ "vector_id": "OVR-010",
168
+ "category": "overcorrection",
169
+ "subcategory": "digits_protection",
170
+ "input_text": "تأسست الشركة عام 2020 وبلغ عدد موظفيها 150 شخصاً",
171
+ "expected_behavior": "Digits must NOT be corrupted by grammar model",
172
+ "grammar_raw_output": "تأسست الشركة عام 2020 وبلغ عدد موظفيها 150 شخصا",
173
+ "grammar_raw_ms": 4709,
174
+ "punctuation_raw_output": "تأسست الشركة عام 2020 وبلغ عدد موظفيها 150 شخصاً.",
175
+ "punctuation_raw_ms": 1835,
176
+ "pipeline_output": "تأسست الشركة عام يناير وبلغ عدد موظفيها خمسين شخصاً.",
177
+ "pipeline_corrected": "تأسست الشركة عام يناير وبلغ عدد موظفيها خمسين شخصاً.",
178
+ "pipeline_suggestions": [
179
+ {
180
+ "alternatives": [],
181
+ "confidence": 0.8,
182
+ "correction": "شخصاً.",
183
+ "end": 48,
184
+ "id": "3905f603-2a35-4786-a1cd-8d44c1995ba9",
185
+ "locked": true,
186
+ "original": "شخصاً",
187
+ "priority": 2,
188
+ "start": 43,
189
+ "type": "punctuation"
190
+ },
191
+ {
192
+ "alternatives": [
193
+ "يناير",
194
+ "2020"
195
+ ],
196
+ "confidence": 0.6,
197
+ "correction": "يناير",
198
+ "end": 21,
199
+ "id": "99bdcd3f-20bf-4319-be00-cf116e7163a6",
200
+ "locked": true,
201
+ "original": "2020",
202
+ "priority": 1,
203
+ "start": 17,
204
+ "type": "spelling"
205
+ },
206
+ {
207
+ "alternatives": [
208
+ "خمسين",
209
+ "150"
210
+ ],
211
+ "confidence": 0.6,
212
+ "correction": "خمسين",
213
+ "end": 42,
214
+ "id": "91023a54-9a4f-415c-83cc-746e12f64dc3",
215
+ "locked": true,
216
+ "original": "150",
217
+ "priority": 1,
218
+ "start": 39,
219
+ "type": "spelling"
220
+ }
221
+ ],
222
+ "pipeline_ms": 15379,
223
+ "pipeline_timing": {
224
+ "grammar_ms": 1106,
225
+ "punctuation_ms": 905,
226
+ "spelling_ms": 9998,
227
+ "total_ms": 12016
228
+ },
229
+ "verdict": "FAIL_FILTER",
230
+ "root_cause": "Digits were corrupted",
231
+ "failure_detail": "",
232
+ "grammar_raw_changed": true,
233
+ "pipeline_changed": true,
234
+ "raw_vs_pipeline_match": false,
235
+ "span_errors": [],
236
+ "offset_drift": 0
237
+ }
238
+ ],
239
+ "pipeline_issues": [],
240
+ "model_issues": [
241
+ {
242
+ "vector_id": "PFX-001",
243
+ "category": "prefix_handling",
244
+ "subcategory": "wa_prefix",
245
+ "input_text": "وبالمستشفيات الكبيره يعالجون المرضي",
246
+ "expected_behavior": "وبالمستشفيات should NOT be split or hallucinated — it's a valid prefixed word",
247
+ "grammar_raw_output": "وبالمستشفيات الكبيرة يعالج المرضي",
248
+ "grammar_raw_ms": 2971,
249
+ "punctuation_raw_output": "وبالمستشفيات الكبيره يعالجون المرضي.",
250
+ "punctuation_raw_ms": 3292,
251
+ "pipeline_output": "والمستشفيات الكبيرة تعالج المرضى",
252
+ "pipeline_corrected": "والمستشفيات الكبيرة تعالج المرضى",
253
+ "pipeline_suggestions": [
254
+ {
255
+ "alternatives": [],
256
+ "confidence": 1.0,
257
+ "correction": "الكبيرة تعالج المرضى",
258
+ "end": 35,
259
+ "id": "864ec032-739e-4cdd-8c0e-ddaba704a15d",
260
+ "locked": true,
261
+ "original": "الكبيره يعالجون المرضي",
262
+ "priority": 3,
263
+ "start": 13,
264
+ "type": "grammar"
265
+ },
266
+ {
267
+ "alternatives": [
268
+ "والمستشفيات",
269
+ "بالمستشفيات",
270
+ "وبالمستشفيات"
271
+ ],
272
+ "confidence": 0.7,
273
+ "correction": "والمستشفيات",
274
+ "end": 12,
275
+ "id": "f22611a6-17ce-49d8-a3e1-fe5f182496f2",
276
+ "locked": true,
277
+ "original": "وبالمستشفيات",
278
+ "priority": 1,
279
+ "start": 0,
280
+ "type": "spelling"
281
+ }
282
+ ],
283
+ "pipeline_ms": 8651,
284
+ "pipeline_timing": {
285
+ "grammar_ms": 900,
286
+ "punctuation_ms": 670,
287
+ "spelling_ms": 3928,
288
+ "total_ms": 5504
289
+ },
290
+ "verdict": "FAIL_MODEL",
291
+ "root_cause": "Prefix word was split, hallucinated, or destroyed",
292
+ "failure_detail": "Input had prefixed words that were corrupted in output",
293
+ "grammar_raw_changed": true,
294
+ "pipeline_changed": true,
295
+ "raw_vs_pipeline_match": false,
296
+ "span_errors": [],
297
+ "offset_drift": 0
298
+ },
299
+ {
300
+ "vector_id": "PFX-005",
301
+ "category": "prefix_handling",
302
+ "subcategory": "fa_prefix",
303
+ "input_text": "فبالتالي يجب ان نعمل بجد",
304
+ "expected_behavior": "فبالتالي is valid, should not be split or changed",
305
+ "grammar_raw_output": "فبالتالي يجب ان نعمل بجد",
306
+ "grammar_raw_ms": 4139,
307
+ "punctuation_raw_output": "فبالتالي يجب ان نعمل بجد.",
308
+ "punctuation_raw_ms": 3775,
309
+ "pipeline_output": "وبالتالي يجب أن نعمل بجد",
310
+ "pipeline_corrected": "وبالتالي يجب أن نعمل بجد",
311
+ "pipeline_suggestions": [
312
+ {
313
+ "alternatives": [
314
+ "أن",
315
+ "ال",
316
+ "من",
317
+ "ان"
318
+ ],
319
+ "confidence": 0.9,
320
+ "correction": "أن",
321
+ "end": 15,
322
+ "id": "6bad78e5-9590-4425-9f0d-a206aa9e494f",
323
+ "locked": true,
324
+ "original": "ان",
325
+ "priority": 1,
326
+ "start": 13,
327
+ "type": "spelling"
328
+ },
329
+ {
330
+ "alternatives": [
331
+ "وبالتالي",
332
+ "فبالتالي"
333
+ ],
334
+ "confidence": 0.6,
335
+ "correction": "وبالتالي",
336
+ "end": 8,
337
+ "id": "07473106-3459-4551-8c57-649eb7055f5d",
338
+ "locked": true,
339
+ "original": "فبالتالي",
340
+ "priority": 1,
341
+ "start": 0,
342
+ "type": "spelling"
343
+ }
344
+ ],
345
+ "pipeline_ms": 8459,
346
+ "pipeline_timing": {
347
+ "grammar_ms": 747,
348
+ "punctuation_ms": 452,
349
+ "spelling_ms": 4087,
350
+ "total_ms": 5292
351
+ },
352
+ "verdict": "FAIL_MODEL",
353
+ "root_cause": "Prefix word was split, hallucinated, or destroyed",
354
+ "failure_detail": "Input had prefixed words that were corrupted in output",
355
+ "grammar_raw_changed": false,
356
+ "pipeline_changed": true,
357
+ "raw_vs_pipeline_match": false,
358
+ "span_errors": [],
359
+ "offset_drift": 0
360
+ }
361
+ ],
362
+ "results": [
363
+ {
364
+ "vector_id": "OFS-001",
365
+ "category": "offset_shift",
366
+ "subcategory": "punct_after_spelling_fix",
367
+ "input_text": "ذهبت الي المدرسه وقابلت صديقتي وتحدثنا كثيرا",
368
+ "expected_behavior": "Punct marks should align with corrected text, not shift 1-2 words",
369
+ "grammar_raw_output": "ذهبت إلى المدرسة وقابلت صديقتي وتحدثنا كثيرا",
370
+ "grammar_raw_ms": 2192,
371
+ "punctuation_raw_output": "ذهبت الي المدرسه وقابلت صديقتي وتحدثنا. كثيرا",
372
+ "punctuation_raw_ms": 4539,
373
+ "pipeline_output": "ذهبت إلى المدرسة وقابلت صديقتي وتحدثنا كثيرا.",
374
+ "pipeline_corrected": "ذهبت إلى المدرسة وقابلت صديقتي وتحدثنا كثيرا.",
375
+ "pipeline_suggestions": [
376
+ {
377
+ "alternatives": [],
378
+ "confidence": 0.8,
379
+ "correction": "كثيرا.",
380
+ "end": 44,
381
+ "id": "a75e719f-d731-4b6b-a818-c6677d86e805",
382
+ "locked": true,
383
+ "original": "كثيرا",
384
+ "priority": 2,
385
+ "start": 39,
386
+ "type": "punctuation"
387
+ },
388
+ {
389
+ "alternatives": [
390
+ "إلى",
391
+ "ال",
392
+ "الم",
393
+ "الي"
394
+ ],
395
+ "confidence": 0.9,
396
+ "correction": "إلى",
397
+ "end": 8,
398
+ "id": "16414133-769e-4d51-97a4-99a08f72cfee",
399
+ "locked": true,
400
+ "original": "الي",
401
+ "priority": 1,
402
+ "start": 5,
403
+ "type": "spelling"
404
+ },
405
+ {
406
+ "alternatives": [
407
+ "المدرسة",
408
+ "المدرسي",
409
+ "المدرس",
410
+ "المدرسه"
411
+ ],
412
+ "confidence": 0.9,
413
+ "correction": "المدرسة",
414
+ "end": 16,
415
+ "id": "b07eb52f-656d-48de-b10e-fc1ad7ab3ba3",
416
+ "locked": true,
417
+ "original": "المدرسه",
418
+ "priority": 1,
419
+ "start": 9,
420
+ "type": "spelling"
421
+ }
422
+ ],
423
+ "pipeline_ms": 15129,
424
+ "pipeline_timing": {
425
+ "grammar_ms": 1100,
426
+ "punctuation_ms": 1028,
427
+ "spelling_ms": 12751,
428
+ "total_ms": 14915
429
+ },
430
+ "verdict": "PASS",
431
+ "root_cause": "Spans aligned correctly",
432
+ "failure_detail": "",
433
+ "grammar_raw_changed": true,
434
+ "pipeline_changed": true,
435
+ "raw_vs_pipeline_match": false,
436
+ "span_errors": [],
437
+ "offset_drift": 0
438
+ },
439
+ {
440
+ "vector_id": "OFS-002",
441
+ "category": "offset_shift",
442
+ "subcategory": "punct_after_multi_word_fix",
443
+ "input_text": "الطلاب ذهبو الي المكتبه لقراءة الكتب",
444
+ "expected_behavior": "Punctuation positions should not drift after spelling corrects الي→إلى and المكتبه→المكتبة",
445
+ "grammar_raw_output": "الطلاب ذهبوا إلى المكتبة لقراءة الكتب",
446
+ "grammar_raw_ms": 3641,
447
+ "punctuation_raw_output": "الطلاب ذهبو الي المكتبه لقراءة الكتب.",
448
+ "punctuation_raw_ms": 5669,
449
+ "pipeline_output": "الطلاب ذهبوا إلى المكتبة لقراءة الكتب.",
450
+ "pipeline_corrected": "الطلاب ذهبوا إلى المكتبة لقراءة الكتب.",
451
+ "pipeline_suggestions": [
452
+ {
453
+ "alternatives": [],
454
+ "confidence": 0.8,
455
+ "correction": "الكتب.",
456
+ "end": 36,
457
+ "id": "70e059c6-dc53-4d2a-8264-1d30b38dd374",
458
+ "locked": true,
459
+ "original": "الكتب",
460
+ "priority": 2,
461
+ "start": 31,
462
+ "type": "punctuation"
463
+ },
464
+ {
465
+ "alternatives": [
466
+ "إلى",
467
+ "ال",
468
+ "الم",
469
+ "الي"
470
+ ],
471
+ "confidence": 0.9,
472
+ "correction": "إلى",
473
+ "end": 15,
474
+ "id": "f3a2de67-9b80-48a5-bfe2-630f4a974d47",
475
+ "locked": true,
476
+ "original": "الي",
477
+ "priority": 1,
478
+ "start": 12,
479
+ "type": "spelling"
480
+ },
481
+ {
482
+ "alternatives": [
483
+ "المكتبة",
484
+ "المكتب",
485
+ "المشتبه",
486
+ "المكتبه"
487
+ ],
488
+ "confidence": 0.9,
489
+ "correction": "المكتبة",
490
+ "end": 23,
491
+ "id": "21cdcf2c-a116-4ad9-b4a2-408ffdc944d3",
492
+ "locked": true,
493
+ "original": "المكتبه",
494
+ "priority": 1,
495
+ "start": 16,
496
+ "type": "spelling"
497
+ },
498
+ {
499
+ "alternatives": [
500
+ "ذهبوا",
501
+ "ذهب",
502
+ "ذهبت",
503
+ "ذهبو"
504
+ ],
505
+ "confidence": 0.7,
506
+ "correction": "ذهبوا",
507
+ "end": 11,
508
+ "id": "b6a655e2-0947-424b-8583-18fa28425ed6",
509
+ "locked": true,
510
+ "original": "ذهبو",
511
+ "priority": 1,
512
+ "start": 7,
513
+ "type": "spelling"
514
+ }
515
+ ],
516
+ "pipeline_ms": 12183,
517
+ "pipeline_timing": {
518
+ "grammar_ms": 894,
519
+ "punctuation_ms": 861,
520
+ "spelling_ms": 5803,
521
+ "total_ms": 7565
522
+ },
523
+ "verdict": "PASS",
524
+ "root_cause": "Spans aligned correctly",
525
+ "failure_detail": "",
526
+ "grammar_raw_changed": true,
527
+ "pipeline_changed": true,
528
+ "raw_vs_pipeline_match": false,
529
+ "span_errors": [],
530
+ "offset_drift": 0
531
+ },
532
+ {
533
+ "vector_id": "OFS-003",
534
+ "category": "offset_shift",
535
+ "subcategory": "length_changing_fix",
536
+ "input_text": "انا احب القراءه كثيرا وخصوصا الكتب العلميه",
537
+ "expected_behavior": "أنا → 3 chars vs انا → 3 chars (same length), but القراءه→القراءة changes char positions",
538
+ "grammar_raw_output": "أنا أحب القراءة كثيرا وخصوصا الكتب العلمية",
539
+ "grammar_raw_ms": 5069,
540
+ "punctuation_raw_output": "انا احب القراءه كثيرا وخصوصا الكتب العلميه.",
541
+ "punctuation_raw_ms": 3695,
542
+ "pipeline_output": "أنا أحب القراءة كثيرا وخصوصا الكتب العلمية.",
543
+ "pipeline_corrected": "أنا أحب القراءة كثيرا وخصوصا الكتب العلمية.",
544
+ "pipeline_suggestions": [
545
+ {
546
+ "alternatives": [],
547
+ "confidence": 0.8,
548
+ "correction": "العلمية.",
549
+ "end": 42,
550
+ "id": "d425111e-86bd-408c-a730-abbf9b16d0d3",
551
+ "locked": true,
552
+ "original": "العلميه",
553
+ "priority": 2,
554
+ "start": 35,
555
+ "type": "punctuation"
556
+ },
557
+ {
558
+ "alternatives": [],
559
+ "confidence": 1.0,
560
+ "correction": "القراءة",
561
+ "end": 15,
562
+ "id": "1f15dcfb-98ee-4595-b9e4-5ff1ac1798b2",
563
+ "locked": true,
564
+ "original": "القراءه",
565
+ "priority": 1,
566
+ "start": 8,
567
+ "type": "spelling"
568
+ },
569
+ {
570
+ "alternatives": [
571
+ "أنا",
572
+ "الا",
573
+ "ان",
574
+ "انا"
575
+ ],
576
+ "confidence": 0.9,
577
+ "correction": "أنا",
578
+ "end": 3,
579
+ "id": "024c3155-f0b1-44eb-a893-1323375f3ca8",
580
+ "locked": true,
581
+ "original": "انا",
582
+ "priority": 1,
583
+ "start": 0,
584
+ "type": "spelling"
585
+ },
586
+ {
587
+ "alternatives": [
588
+ "أحب",
589
+ "الب",
590
+ "احت",
591
+ "احب"
592
+ ],
593
+ "confidence": 0.9,
594
+ "correction": "أحب",
595
+ "end": 7,
596
+ "id": "4617eaa9-d3ac-49d8-a6ed-44fe9399ae6b",
597
+ "locked": true,
598
+ "original": "احب",
599
+ "priority": 1,
600
+ "start": 4,
601
+ "type": "spelling"
602
+ }
603
+ ],
604
+ "pipeline_ms": 11427,
605
+ "pipeline_timing": {
606
+ "grammar_ms": 865,
607
+ "punctuation_ms": 859,
608
+ "spelling_ms": 5156,
609
+ "total_ms": 6892
610
+ },
611
+ "verdict": "PASS",
612
+ "root_cause": "Spans aligned correctly",
613
+ "failure_detail": "",
614
+ "grammar_raw_changed": true,
615
+ "pipeline_changed": true,
616
+ "raw_vs_pipeline_match": false,
617
+ "span_errors": [],
618
+ "offset_drift": 0
619
+ },
620
+ {
621
+ "vector_id": "OFS-004",
622
+ "category": "offset_shift",
623
+ "subcategory": "insertion_diffs",
624
+ "input_text": "كتبت رساله لصديقي واخبرته بالاخبار",
625
+ "expected_behavior": "When spelling inserts أ (اخبرته→أخبرته), downstream coords must shift +1",
626
+ "grammar_raw_output": "كتبت رسالة لصديقي وأخبرته بالأخبار",
627
+ "grammar_raw_ms": 5247,
628
+ "punctuation_raw_output": "كتبت رساله لصديقي واخبرته بالاخبار.",
629
+ "punctuation_raw_ms": 4047,
630
+ "pipeline_output": "كتبت رسالة لصديقي واخبرته بالاخبار.",
631
+ "pipeline_corrected": "كتبت رسالة لصديقي واخبرته بالاخبار.",
632
+ "pipeline_suggestions": [
633
+ {
634
+ "alternatives": [],
635
+ "confidence": 0.8,
636
+ "correction": "بالاخبار.",
637
+ "end": 34,
638
+ "id": "cf599833-50ed-4e29-a29a-e34015bc5c06",
639
+ "locked": true,
640
+ "original": "بالاخبار",
641
+ "priority": 2,
642
+ "start": 26,
643
+ "type": "punctuation"
644
+ },
645
+ {
646
+ "alternatives": [
647
+ "رسالة",
648
+ "رسالته",
649
+ "رسالت",
650
+ "رساله"
651
+ ],
652
+ "confidence": 0.9,
653
+ "correction": "رسالة",
654
+ "end": 10,
655
+ "id": "17395e5a-34d1-4900-8dda-d17334fc7232",
656
+ "locked": true,
657
+ "original": "رساله",
658
+ "priority": 1,
659
+ "start": 5,
660
+ "type": "spelling"
661
+ }
662
+ ],
663
+ "pipeline_ms": 10598,
664
+ "pipeline_timing": {
665
+ "grammar_ms": 954,
666
+ "punctuation_ms": 845,
667
+ "spelling_ms": 5722,
668
+ "total_ms": 7525
669
+ },
670
+ "verdict": "PASS",
671
+ "root_cause": "Spans aligned correctly",
672
+ "failure_detail": "",
673
+ "grammar_raw_changed": true,
674
+ "pipeline_changed": true,
675
+ "raw_vs_pipeline_match": false,
676
+ "span_errors": [],
677
+ "offset_drift": 0
678
+ },
679
+ {
680
+ "vector_id": "OFS-005",
681
+ "category": "offset_shift",
682
+ "subcategory": "multi_stage_coord_chain",
683
+ "input_text": "المهندسون ذهبو الي الموقعو وبدأو العمل",
684
+ "expected_behavior": "3+ fixes in same sentence: verify offset chain doesn't accumulate drift",
685
+ "grammar_raw_output": "المهندسون ذهبو إلى الموقع وبدأو العمل",
686
+ "grammar_raw_ms": 4050,
687
+ "punctuation_raw_output": "المهندسون ذهبو الي الموقعو وبدأو العمل.",
688
+ "punctuation_raw_ms": 3819,
689
+ "pipeline_output": "المهندسون ذهبوا إلى الموقع وبدأوا العمل.",
690
+ "pipeline_corrected": "المهندسون ذهبوا إلى الموقع وبدأوا العمل.",
691
+ "pipeline_suggestions": [
692
+ {
693
+ "alternatives": [],
694
+ "confidence": 0.8,
695
+ "correction": "العمل.",
696
+ "end": 38,
697
+ "id": "64709109-bee3-4899-9f03-c1518bea5c98",
698
+ "locked": true,
699
+ "original": "العمل",
700
+ "priority": 2,
701
+ "start": 33,
702
+ "type": "punctuation"
703
+ },
704
+ {
705
+ "alternatives": [
706
+ "إلى",
707
+ "ال",
708
+ "الم",
709
+ "الي"
710
+ ],
711
+ "confidence": 0.9,
712
+ "correction": "إلى",
713
+ "end": 18,
714
+ "id": "1ea64620-098c-43f1-b3ee-7697cc122318",
715
+ "locked": true,
716
+ "original": "الي",
717
+ "priority": 1,
718
+ "start": 15,
719
+ "type": "spelling"
720
+ },
721
+ {
722
+ "alternatives": [
723
+ "ذهبوا",
724
+ "ذهب",
725
+ "ذهبت",
726
+ "ذهبو"
727
+ ],
728
+ "confidence": 0.7,
729
+ "correction": "ذهبوا",
730
+ "end": 14,
731
+ "id": "6f7ea270-b62f-41f1-958d-329c34880996",
732
+ "locked": true,
733
+ "original": "ذهبو",
734
+ "priority": 1,
735
+ "start": 10,
736
+ "type": "spelling"
737
+ },
738
+ {
739
+ "alternatives": [
740
+ "الموقع",
741
+ "الموقعة",
742
+ "الموقعو"
743
+ ],
744
+ "confidence": 0.7,
745
+ "correction": "الموقع",
746
+ "end": 26,
747
+ "id": "7aca1cd0-c410-4dd0-bb2d-f6f862de374b",
748
+ "locked": true,
749
+ "original": "الموقعو",
750
+ "priority": 1,
751
+ "start": 19,
752
+ "type": "spelling"
753
+ },
754
+ {
755
+ "alternatives": [
756
+ "وبدأوا",
757
+ "وبدأ",
758
+ "وبدأت",
759
+ "وبدأو"
760
+ ],
761
+ "confidence": 0.7,
762
+ "correction": "وبدأوا",
763
+ "end": 32,
764
+ "id": "7b93e1e9-89c4-40a9-ac80-f23d0a938529",
765
+ "locked": true,
766
+ "original": "وبدأو",
767
+ "priority": 1,
768
+ "start": 27,
769
+ "type": "spelling"
770
+ }
771
+ ],
772
+ "pipeline_ms": 10684,
773
+ "pipeline_timing": {
774
+ "grammar_ms": 1091,
775
+ "punctuation_ms": 953,
776
+ "spelling_ms": 4027,
777
+ "total_ms": 6078
778
+ },
779
+ "verdict": "PASS",
780
+ "root_cause": "Spans aligned correctly",
781
+ "failure_detail": "",
782
+ "grammar_raw_changed": true,
783
+ "pipeline_changed": true,
784
+ "raw_vs_pipeline_match": false,
785
+ "span_errors": [],
786
+ "offset_drift": 0
787
+ },
788
+ {
789
+ "vector_id": "PFX-001",
790
+ "category": "prefix_handling",
791
+ "subcategory": "wa_prefix",
792
+ "input_text": "وبالمستشفيات الكبيره يعالجون المرضي",
793
+ "expected_behavior": "وبالمستشفيات should NOT be split or hallucinated — it's a valid prefixed word",
794
+ "grammar_raw_output": "وبالمستشفيات الكبيرة يعالج المرضي",
795
+ "grammar_raw_ms": 2971,
796
+ "punctuation_raw_output": "وبالمستشفيات الكبيره يعالجون المرضي.",
797
+ "punctuation_raw_ms": 3292,
798
+ "pipeline_output": "والمستشفيات الكبيرة تعالج المرضى",
799
+ "pipeline_corrected": "والمستشفيات الكبيرة تعالج المرضى",
800
+ "pipeline_suggestions": [
801
+ {
802
+ "alternatives": [],
803
+ "confidence": 1.0,
804
+ "correction": "الكبيرة تعالج المرضى",
805
+ "end": 35,
806
+ "id": "864ec032-739e-4cdd-8c0e-ddaba704a15d",
807
+ "locked": true,
808
+ "original": "الكبيره يعالجون المرضي",
809
+ "priority": 3,
810
+ "start": 13,
811
+ "type": "grammar"
812
+ },
813
+ {
814
+ "alternatives": [
815
+ "والمستشفيات",
816
+ "بالمستشفيات",
817
+ "وبالمستشفيات"
818
+ ],
819
+ "confidence": 0.7,
820
+ "correction": "والمستشفيات",
821
+ "end": 12,
822
+ "id": "f22611a6-17ce-49d8-a3e1-fe5f182496f2",
823
+ "locked": true,
824
+ "original": "وبالمستشفيات",
825
+ "priority": 1,
826
+ "start": 0,
827
+ "type": "spelling"
828
+ }
829
+ ],
830
+ "pipeline_ms": 8651,
831
+ "pipeline_timing": {
832
+ "grammar_ms": 900,
833
+ "punctuation_ms": 670,
834
+ "spelling_ms": 3928,
835
+ "total_ms": 5504
836
+ },
837
+ "verdict": "FAIL_MODEL",
838
+ "root_cause": "Prefix word was split, hallucinated, or destroyed",
839
+ "failure_detail": "Input had prefixed words that were corrupted in output",
840
+ "grammar_raw_changed": true,
841
+ "pipeline_changed": true,
842
+ "raw_vs_pipeline_match": false,
843
+ "span_errors": [],
844
+ "offset_drift": 0
845
+ },
846
+ {
847
+ "vector_id": "PFX-002",
848
+ "category": "prefix_handling",
849
+ "subcategory": "ba_prefix",
850
+ "input_text": "بالمدرسه الجديده تعلمت كثيرا",
851
+ "expected_behavior": "بالمدرسه → بالمدرسة (only ه→ة fix, keep ب prefix)",
852
+ "grammar_raw_output": "بالمدرسة الجديدة تعلمت كثيرا",
853
+ "grammar_raw_ms": 4857,
854
+ "punctuation_raw_output": "بالمدرسه الجديده تعلمت كثيرا!",
855
+ "punctuation_raw_ms": 3498,
856
+ "pipeline_output": "بالمدرسة الجديدة تعلمت كثيرا!",
857
+ "pipeline_corrected": "بالمدرسة الجديدة تعلمت كثيرا!",
858
+ "pipeline_suggestions": [
859
+ {
860
+ "alternatives": [],
861
+ "confidence": 0.8,
862
+ "correction": "كثيرا!",
863
+ "end": 28,
864
+ "id": "2ed528c2-3c7d-4e79-a44a-7403baea2498",
865
+ "locked": true,
866
+ "original": "كثيرا",
867
+ "priority": 2,
868
+ "start": 23,
869
+ "type": "punctuation"
870
+ },
871
+ {
872
+ "alternatives": [
873
+ "بالمدرسة",
874
+ "بالمدرسه"
875
+ ],
876
+ "confidence": 0.9,
877
+ "correction": "بالمدرسة",
878
+ "end": 8,
879
+ "id": "eb76dee8-b0a1-4229-a8c8-396b00503632",
880
+ "locked": true,
881
+ "original": "بالمدرسه",
882
+ "priority": 1,
883
+ "start": 0,
884
+ "type": "spelling"
885
+ },
886
+ {
887
+ "alternatives": [
888
+ "الجديدة",
889
+ "الجديد",
890
+ "الجديده"
891
+ ],
892
+ "confidence": 0.9,
893
+ "correction": "الجديدة",
894
+ "end": 16,
895
+ "id": "d0406447-2aab-40a8-8e67-f1be08eb147c",
896
+ "locked": true,
897
+ "original": "الجديده",
898
+ "priority": 1,
899
+ "start": 9,
900
+ "type": "spelling"
901
+ }
902
+ ],
903
+ "pipeline_ms": 6854,
904
+ "pipeline_timing": {
905
+ "grammar_ms": 683,
906
+ "punctuation_ms": 695,
907
+ "spelling_ms": 2462,
908
+ "total_ms": 3844
909
+ },
910
+ "verdict": "PASS",
911
+ "root_cause": "Prefixed words handled correctly",
912
+ "failure_detail": "",
913
+ "grammar_raw_changed": true,
914
+ "pipeline_changed": true,
915
+ "raw_vs_pipeline_match": false,
916
+ "span_errors": [],
917
+ "offset_drift": 0
918
+ },
919
+ {
920
+ "vector_id": "PFX-003",
921
+ "category": "prefix_handling",
922
+ "subcategory": "li_prefix",
923
+ "input_text": "للطالبات المجتهدات جوائز قيمه",
924
+ "expected_behavior": "للطالبات should be preserved intact",
925
+ "grammar_raw_output": "للطالبات المجتهدات جوائز قيمة",
926
+ "grammar_raw_ms": 3639,
927
+ "punctuation_raw_output": "للطالبات المجتهدات جوائز قيمه.",
928
+ "punctuation_raw_ms": 4755,
929
+ "pipeline_output": "للطالبات المجتهدات جوائز قيمة.",
930
+ "pipeline_corrected": "للطالبات المجتهدات جوائز قيمة.",
931
+ "pipeline_suggestions": [
932
+ {
933
+ "alternatives": [],
934
+ "confidence": 0.8,
935
+ "correction": "قيمة.",
936
+ "end": 29,
937
+ "id": "775e81e6-24fb-4e63-87e5-e24b05a6667d",
938
+ "locked": true,
939
+ "original": "قيمه",
940
+ "priority": 2,
941
+ "start": 25,
942
+ "type": "punctuation"
943
+ }
944
+ ],
945
+ "pipeline_ms": 7291,
946
+ "pipeline_timing": {
947
+ "grammar_ms": 695,
948
+ "punctuation_ms": 703,
949
+ "spelling_ms": 3010,
950
+ "total_ms": 4412
951
+ },
952
+ "verdict": "PASS",
953
+ "root_cause": "Prefixed words handled correctly",
954
+ "failure_detail": "",
955
+ "grammar_raw_changed": true,
956
+ "pipeline_changed": true,
957
+ "raw_vs_pipeline_match": false,
958
+ "span_errors": [],
959
+ "offset_drift": 0
960
+ },
961
+ {
962
+ "vector_id": "PFX-004",
963
+ "category": "prefix_handling",
964
+ "subcategory": "ka_prefix",
965
+ "input_text": "كالعاده ذهبت الي العمل",
966
+ "expected_behavior": "كالعادة — ه→ة fix should work through ك+ال prefix",
967
+ "grammar_raw_output": "كالعادة ذهبت إلى العمل",
968
+ "grammar_raw_ms": 3487,
969
+ "punctuation_raw_output": "كالعاده ذهبت الي. العمل",
970
+ "punctuation_raw_ms": 3577,
971
+ "pipeline_output": "كالعادة ذهبت إلى العمل.",
972
+ "pipeline_corrected": "كالعادة ذهبت إلى العمل.",
973
+ "pipeline_suggestions": [
974
+ {
975
+ "alternatives": [],
976
+ "confidence": 0.8,
977
+ "correction": "العمل.",
978
+ "end": 22,
979
+ "id": "2dec32ad-860f-4839-93d9-fe92a7089654",
980
+ "locked": true,
981
+ "original": "العمل",
982
+ "priority": 2,
983
+ "start": 17,
984
+ "type": "punctuation"
985
+ },
986
+ {
987
+ "alternatives": [
988
+ "كالعادة",
989
+ "كالعاده"
990
+ ],
991
+ "confidence": 0.9,
992
+ "correction": "كالعادة",
993
+ "end": 7,
994
+ "id": "fb09c4e9-3766-4db1-b859-43ceb3c8d25c",
995
+ "locked": true,
996
+ "original": "كالعاده",
997
+ "priority": 1,
998
+ "start": 0,
999
+ "type": "spelling"
1000
+ },
1001
+ {
1002
+ "alternatives": [
1003
+ "إلى",
1004
+ "ال",
1005
+ "الم",
1006
+ "الي"
1007
+ ],
1008
+ "confidence": 0.9,
1009
+ "correction": "إلى",
1010
+ "end": 16,
1011
+ "id": "4969eaee-2579-4a12-a8e0-eb87c6996647",
1012
+ "locked": true,
1013
+ "original": "الي",
1014
+ "priority": 1,
1015
+ "start": 13,
1016
+ "type": "spelling"
1017
+ }
1018
+ ],
1019
+ "pipeline_ms": 7188,
1020
+ "pipeline_timing": {
1021
+ "grammar_ms": 646,
1022
+ "punctuation_ms": 529,
1023
+ "spelling_ms": 2683,
1024
+ "total_ms": 3863
1025
+ },
1026
+ "verdict": "PASS",
1027
+ "root_cause": "Prefixed words handled correctly",
1028
+ "failure_detail": "",
1029
+ "grammar_raw_changed": true,
1030
+ "pipeline_changed": true,
1031
+ "raw_vs_pipeline_match": false,
1032
+ "span_errors": [],
1033
+ "offset_drift": 0
1034
+ },
1035
+ {
1036
+ "vector_id": "PFX-005",
1037
+ "category": "prefix_handling",
1038
+ "subcategory": "fa_prefix",
1039
+ "input_text": "فبالتالي يجب ان نعمل بجد",
1040
+ "expected_behavior": "فبالتالي is valid, should not be split or changed",
1041
+ "grammar_raw_output": "فبالتالي يجب ان نعمل بجد",
1042
+ "grammar_raw_ms": 4139,
1043
+ "punctuation_raw_output": "فبالتالي يجب ان نعمل بجد.",
1044
+ "punctuation_raw_ms": 3775,
1045
+ "pipeline_output": "وبالتالي يجب أن نعمل بجد",
1046
+ "pipeline_corrected": "وبالتالي يجب أن نعمل بجد",
1047
+ "pipeline_suggestions": [
1048
+ {
1049
+ "alternatives": [
1050
+ "أن",
1051
+ "ال",
1052
+ "من",
1053
+ "ان"
1054
+ ],
1055
+ "confidence": 0.9,
1056
+ "correction": "أن",
1057
+ "end": 15,
1058
+ "id": "6bad78e5-9590-4425-9f0d-a206aa9e494f",
1059
+ "locked": true,
1060
+ "original": "ان",
1061
+ "priority": 1,
1062
+ "start": 13,
1063
+ "type": "spelling"
1064
+ },
1065
+ {
1066
+ "alternatives": [
1067
+ "وبالتالي",
1068
+ "فبالتالي"
1069
+ ],
1070
+ "confidence": 0.6,
1071
+ "correction": "وبالتالي",
1072
+ "end": 8,
1073
+ "id": "07473106-3459-4551-8c57-649eb7055f5d",
1074
+ "locked": true,
1075
+ "original": "فبالتالي",
1076
+ "priority": 1,
1077
+ "start": 0,
1078
+ "type": "spelling"
1079
+ }
1080
+ ],
1081
+ "pipeline_ms": 8459,
1082
+ "pipeline_timing": {
1083
+ "grammar_ms": 747,
1084
+ "punctuation_ms": 452,
1085
+ "spelling_ms": 4087,
1086
+ "total_ms": 5292
1087
+ },
1088
+ "verdict": "FAIL_MODEL",
1089
+ "root_cause": "Prefix word was split, hallucinated, or destroyed",
1090
+ "failure_detail": "Input had prefixed words that were corrupted in output",
1091
+ "grammar_raw_changed": false,
1092
+ "pipeline_changed": true,
1093
+ "raw_vs_pipeline_match": false,
1094
+ "span_errors": [],
1095
+ "offset_drift": 0
1096
+ },
1097
+ {
1098
+ "vector_id": "PFX-006",
1099
+ "category": "prefix_handling",
1100
+ "subcategory": "complex_prefix_chain",
1101
+ "input_text": "وبالاضافه الي ذلك فان المشكله كبيره",
1102
+ "expected_behavior": "وبالإضافة إلى — hamza fix through وب prefix, plus ه→ة fixes",
1103
+ "grammar_raw_output": "بالإضافة إلى ذلك فإن المشكلة كبيرة",
1104
+ "grammar_raw_ms": 5673,
1105
+ "punctuation_raw_output": "وبالاضافه الي ذلك فان المشكله كبيره",
1106
+ "punctuation_raw_ms": 5848,
1107
+ "pipeline_output": "وبالاضافة إلى ذلك فأن المشكلة كبيرة",
1108
+ "pipeline_corrected": "وبالاضافة إلى ذلك فأن المشكلة كبيرة",
1109
+ "pipeline_suggestions": [
1110
+ {
1111
+ "alternatives": [
1112
+ "وبالاضافة",
1113
+ "وبالاضافه"
1114
+ ],
1115
+ "confidence": 0.9,
1116
+ "correction": "وبالاضافة",
1117
+ "end": 9,
1118
+ "id": "0e73b0d4-951b-4080-8ad0-0a676b744ff6",
1119
+ "locked": true,
1120
+ "original": "وبالاضافه",
1121
+ "priority": 1,
1122
+ "start": 0,
1123
+ "type": "spelling"
1124
+ },
1125
+ {
1126
+ "alternatives": [
1127
+ "إلى",
1128
+ "ال",
1129
+ "الم",
1130
+ "الي"
1131
+ ],
1132
+ "confidence": 0.9,
1133
+ "correction": "إلى",
1134
+ "end": 13,
1135
+ "id": "fc987f49-e21f-47e8-9982-da87eaba7719",
1136
+ "locked": true,
1137
+ "original": "الي",
1138
+ "priority": 1,
1139
+ "start": 10,
1140
+ "type": "spelling"
1141
+ },
1142
+ {
1143
+ "alternatives": [
1144
+ "فأن",
1145
+ "ان",
1146
+ "كان",
1147
+ "فان"
1148
+ ],
1149
+ "confidence": 0.9,
1150
+ "correction": "فأن",
1151
+ "end": 21,
1152
+ "id": "ef5eb7fb-d623-4a3c-aa72-22785ca8d920",
1153
+ "locked": true,
1154
+ "original": "فان",
1155
+ "priority": 1,
1156
+ "start": 18,
1157
+ "type": "spelling"
1158
+ },
1159
+ {
1160
+ "alternatives": [
1161
+ "المشكلة",
1162
+ "المشكل",
1163
+ "المشكله"
1164
+ ],
1165
+ "confidence": 0.9,
1166
+ "correction": "المشكلة",
1167
+ "end": 29,
1168
+ "id": "a2c44be1-a574-422a-8a08-253daa04072b",
1169
+ "locked": true,
1170
+ "original": "المشكله",
1171
+ "priority": 1,
1172
+ "start": 22,
1173
+ "type": "spelling"
1174
+ },
1175
+ {
1176
+ "alternatives": [
1177
+ "كبيرة",
1178
+ "كبير",
1179
+ "كبيرا",
1180
+ "كبيره"
1181
+ ],
1182
+ "confidence": 0.9,
1183
+ "correction": "كبيرة",
1184
+ "end": 35,
1185
+ "id": "6d53838b-635c-41a8-933f-5de600b28494",
1186
+ "locked": true,
1187
+ "original": "كبيره",
1188
+ "priority": 1,
1189
+ "start": 30,
1190
+ "type": "spelling"
1191
+ }
1192
+ ],
1193
+ "pipeline_ms": 8512,
1194
+ "pipeline_timing": {
1195
+ "grammar_ms": 795,
1196
+ "punctuation_ms": 664,
1197
+ "spelling_ms": 4027,
1198
+ "total_ms": 5496
1199
+ },
1200
+ "verdict": "PASS",
1201
+ "root_cause": "Prefixed words handled correctly",
1202
+ "failure_detail": "",
1203
+ "grammar_raw_changed": true,
1204
+ "pipeline_changed": true,
1205
+ "raw_vs_pipeline_match": false,
1206
+ "span_errors": [],
1207
+ "offset_drift": 0
1208
+ },
1209
+ {
1210
+ "vector_id": "PFX-007",
1211
+ "category": "prefix_handling",
1212
+ "subcategory": "wa_attached_to_error",
1213
+ "input_text": "والمهندسين بنو المبني بسرعه",
1214
+ "expected_behavior": "والمهندسين should stay, بنو→بنوا (grammar), المبني→المبنى, بسرعه→بسرعة",
1215
+ "grammar_raw_output": "والمهندسين بنو المبنى بسرعة",
1216
+ "grammar_raw_ms": 3892,
1217
+ "punctuation_raw_output": "والمهندسين بنو المبني بسرعه",
1218
+ "punctuation_raw_ms": 5865,
1219
+ "pipeline_output": "والمهندسين بنو المبنى بسرعة",
1220
+ "pipeline_corrected": "والمهندسين بنو المبنى بسرعة",
1221
+ "pipeline_suggestions": [
1222
+ {
1223
+ "alternatives": [],
1224
+ "confidence": 1.0,
1225
+ "correction": "المبنى",
1226
+ "end": 21,
1227
+ "id": "9525df7d-619d-4b89-9eef-52e2cfa11c27",
1228
+ "locked": true,
1229
+ "original": "المبني",
1230
+ "priority": 1,
1231
+ "start": 15,
1232
+ "type": "spelling"
1233
+ },
1234
+ {
1235
+ "alternatives": [
1236
+ "بسرعة",
1237
+ "بسرعه"
1238
+ ],
1239
+ "confidence": 0.9,
1240
+ "correction": "بسرعة",
1241
+ "end": 27,
1242
+ "id": "07f7347d-07ae-42fb-9bd3-7366b4df58e4",
1243
+ "locked": true,
1244
+ "original": "بسرعه",
1245
+ "priority": 1,
1246
+ "start": 22,
1247
+ "type": "spelling"
1248
+ }
1249
+ ],
1250
+ "pipeline_ms": 7327,
1251
+ "pipeline_timing": {
1252
+ "grammar_ms": 741,
1253
+ "punctuation_ms": 720,
1254
+ "spelling_ms": 2257,
1255
+ "total_ms": 3723
1256
+ },
1257
+ "verdict": "PASS",
1258
+ "root_cause": "Prefixed words handled correctly",
1259
+ "failure_detail": "",
1260
+ "grammar_raw_changed": true,
1261
+ "pipeline_changed": true,
1262
+ "raw_vs_pipeline_match": true,
1263
+ "span_errors": [],
1264
+ "offset_drift": 0
1265
+ },
1266
+ {
1267
+ "vector_id": "PFX-008",
1268
+ "category": "prefix_handling",
1269
+ "subcategory": "attached_pronoun",
1270
+ "input_text": "كتابه قديم وصفحاته ممزقه",
1271
+ "expected_behavior": "كتابه (his book) — ه is pronoun, NOT ta marbuta. Must NOT become كتابة",
1272
+ "grammar_raw_output": "كتابه قديم وصفحاته ممزقه",
1273
+ "grammar_raw_ms": 3990,
1274
+ "punctuation_raw_output": "كتابه قديم وصفحاته ممزقه.",
1275
+ "punctuation_raw_ms": 4414,
1276
+ "pipeline_output": "كتابه قديم وصفحاته ممزقة.",
1277
+ "pipeline_corrected": "كتابه قديم وصفحاته ممزقة.",
1278
+ "pipeline_suggestions": [
1279
+ {
1280
+ "alternatives": [],
1281
+ "confidence": 0.8,
1282
+ "correction": "ممزقة.",
1283
+ "end": 24,
1284
+ "id": "e043a926-dd17-4d48-a5c9-629f3e23c421",
1285
+ "locked": true,
1286
+ "original": "ممزقه",
1287
+ "priority": 2,
1288
+ "start": 19,
1289
+ "type": "punctuation"
1290
+ }
1291
+ ],
1292
+ "pipeline_ms": 8659,
1293
+ "pipeline_timing": {
1294
+ "grammar_ms": 898,
1295
+ "punctuation_ms": 569,
1296
+ "spelling_ms": 2573,
1297
+ "total_ms": 4043
1298
+ },
1299
+ "verdict": "PASS",
1300
+ "root_cause": "Prefixed words handled correctly",
1301
+ "failure_detail": "",
1302
+ "grammar_raw_changed": false,
1303
+ "pipeline_changed": true,
1304
+ "raw_vs_pipeline_match": false,
1305
+ "span_errors": [],
1306
+ "offset_drift": 0
1307
+ },
1308
+ {
1309
+ "vector_id": "PFX-009",
1310
+ "category": "prefix_handling",
1311
+ "subcategory": "pronoun_vs_tamarbuta",
1312
+ "input_text": "قرأته بعنايه فتأملته جيدا",
1313
+ "expected_behavior": "قرأته and فتأملته — ته is pronoun suffix, NOT ta marbuta",
1314
+ "grammar_raw_output": "قرأته بعناية فتأملته جيدا",
1315
+ "grammar_raw_ms": 5216,
1316
+ "punctuation_raw_output": "قرأته بعنايه فتأملته جيدا.",
1317
+ "punctuation_raw_ms": 4752,
1318
+ "pipeline_output": "قرأته بعناية فتأملته جيدا",
1319
+ "pipeline_corrected": "قرأته بعناية فتأملته جيدا",
1320
+ "pipeline_suggestions": [
1321
+ {
1322
+ "alternatives": [
1323
+ "بعناية",
1324
+ "بعنايه"
1325
+ ],
1326
+ "confidence": 0.9,
1327
+ "correction": "بعناية",
1328
+ "end": 12,
1329
+ "id": "e54cb82b-c7c6-46da-af00-c32a1e6e0ec0",
1330
+ "locked": true,
1331
+ "original": "بعنايه",
1332
+ "priority": 1,
1333
+ "start": 6,
1334
+ "type": "spelling"
1335
+ }
1336
+ ],
1337
+ "pipeline_ms": 9512,
1338
+ "pipeline_timing": {
1339
+ "grammar_ms": 886,
1340
+ "punctuation_ms": 680,
1341
+ "spelling_ms": 4227,
1342
+ "total_ms": 5796
1343
+ },
1344
+ "verdict": "PASS",
1345
+ "root_cause": "Prefixed words handled correctly",
1346
+ "failure_detail": "",
1347
+ "grammar_raw_changed": true,
1348
+ "pipeline_changed": true,
1349
+ "raw_vs_pipeline_match": true,
1350
+ "span_errors": [],
1351
+ "offset_drift": 0
1352
+ },
1353
+ {
1354
+ "vector_id": "PFX-010",
1355
+ "category": "prefix_handling",
1356
+ "subcategory": "merged_preposition",
1357
+ "input_text": "ذهبتللمدرسه صباحا",
1358
+ "expected_behavior": "Missing space: ذهبت+ل+المدرسه — should split and fix",
1359
+ "grammar_raw_output": "ذهبت للمدرسة صباحا",
1360
+ "grammar_raw_ms": 4748,
1361
+ "punctuation_raw_output": "ذهبتللمدرسه صباحا.",
1362
+ "punctuation_raw_ms": 4569,
1363
+ "pipeline_output": "ذهبتللمدرسة صباحا",
1364
+ "pipeline_corrected": "ذهبتللمدرسة صباحا",
1365
+ "pipeline_suggestions": [
1366
+ {
1367
+ "alternatives": [
1368
+ "ذهبتللمدرسة",
1369
+ "ذهبتللمدرسه"
1370
+ ],
1371
+ "confidence": 0.9,
1372
+ "correction": "ذهبتللمدرسة",
1373
+ "end": 11,
1374
+ "id": "89bbf955-1553-406d-be97-c419f91dd608",
1375
+ "locked": true,
1376
+ "original": "ذهبتللمدرسه",
1377
+ "priority": 1,
1378
+ "start": 0,
1379
+ "type": "spelling"
1380
+ }
1381
+ ],
1382
+ "pipeline_ms": 7768,
1383
+ "pipeline_timing": {
1384
+ "grammar_ms": 737,
1385
+ "punctuation_ms": 473,
1386
+ "spelling_ms": 3234,
1387
+ "total_ms": 4448
1388
+ },
1389
+ "verdict": "PASS",
1390
+ "root_cause": "Prefixed words handled correctly",
1391
+ "failure_detail": "",
1392
+ "grammar_raw_changed": true,
1393
+ "pipeline_changed": true,
1394
+ "raw_vs_pipeline_match": false,
1395
+ "span_errors": [],
1396
+ "offset_drift": 0
1397
+ },
1398
+ {
1399
+ "vector_id": "SLK-001",
1400
+ "category": "stagelocker",
1401
+ "subcategory": "spelling_lock_grammar_override",
1402
+ "input_text": "السياره جميل جدا",
1403
+ "expected_behavior": "Spelling fixes السياره→السيارة, Grammar should still fix جميل→جميلة (Phase 11 hierarchical)",
1404
+ "grammar_raw_output": "السيارة جميلة جدا",
1405
+ "grammar_raw_ms": 4482,
1406
+ "punctuation_raw_output": "السياره جميل جدا.",
1407
+ "punctuation_raw_ms": 4206,
1408
+ "pipeline_output": "السيارة جميلة جدا.",
1409
+ "pipeline_corrected": "السيارة جميلة جدا.",
1410
+ "pipeline_suggestions": [
1411
+ {
1412
+ "alternatives": [],
1413
+ "confidence": 1.0,
1414
+ "correction": "جميلة",
1415
+ "end": 12,
1416
+ "id": "577256fe-dbe7-48dd-b5de-0eb174522356",
1417
+ "locked": true,
1418
+ "original": "جميل",
1419
+ "priority": 3,
1420
+ "start": 8,
1421
+ "type": "grammar"
1422
+ },
1423
+ {
1424
+ "alternatives": [],
1425
+ "confidence": 0.8,
1426
+ "correction": "جدا.",
1427
+ "end": 16,
1428
+ "id": "f3f98848-5d81-4253-920c-5c8637f41976",
1429
+ "locked": true,
1430
+ "original": "جدا",
1431
+ "priority": 2,
1432
+ "start": 13,
1433
+ "type": "punctuation"
1434
+ },
1435
+ {
1436
+ "alternatives": [
1437
+ "السيارة",
1438
+ "السياره"
1439
+ ],
1440
+ "confidence": 0.9,
1441
+ "correction": "السيارة",
1442
+ "end": 7,
1443
+ "id": "64185d2f-cb8c-4fae-85b1-d8da8217ec7a",
1444
+ "locked": true,
1445
+ "original": "السياره",
1446
+ "priority": 1,
1447
+ "start": 0,
1448
+ "type": "spelling"
1449
+ }
1450
+ ],
1451
+ "pipeline_ms": 7050,
1452
+ "pipeline_timing": {
1453
+ "grammar_ms": 534,
1454
+ "punctuation_ms": 397,
1455
+ "spelling_ms": 2028,
1456
+ "total_ms": 2964
1457
+ },
1458
+ "verdict": "PASS",
1459
+ "root_cause": "Grammar successfully overrode spelling lock",
1460
+ "failure_detail": "",
1461
+ "grammar_raw_changed": true,
1462
+ "pipeline_changed": true,
1463
+ "raw_vs_pipeline_match": false,
1464
+ "span_errors": [],
1465
+ "offset_drift": 0
1466
+ },
1467
+ {
1468
+ "vector_id": "SLK-002",
1469
+ "category": "stagelocker",
1470
+ "subcategory": "grammar_not_overwrite_spelling",
1471
+ "input_text": "المدرسه كبير وجميل",
1472
+ "expected_behavior": "Spelling: المدرسه→المدرسة. Grammar should NOT revert to المدرسه. Should add ة to كبير/جميل",
1473
+ "grammar_raw_output": "المدرسة كبيرة وجميلة",
1474
+ "grammar_raw_ms": 3443,
1475
+ "punctuation_raw_output": "المدرسه كبير وجميل.",
1476
+ "punctuation_raw_ms": 2430,
1477
+ "pipeline_output": "المدرسة كبيرة وجميلة",
1478
+ "pipeline_corrected": "المدرسة كبيرة وجميلة",
1479
+ "pipeline_suggestions": [
1480
+ {
1481
+ "alternatives": [],
1482
+ "confidence": 1.0,
1483
+ "correction": "كبيرة وجميلة",
1484
+ "end": 18,
1485
+ "id": "992518de-2c09-4969-b6a1-6b04791b8f62",
1486
+ "locked": true,
1487
+ "original": "كبير وجميل",
1488
+ "priority": 3,
1489
+ "start": 8,
1490
+ "type": "grammar"
1491
+ },
1492
+ {
1493
+ "alternatives": [
1494
+ "المدرسة",
1495
+ "المدرسي",
1496
+ "المدرس",
1497
+ "المدرسه"
1498
+ ],
1499
+ "confidence": 0.9,
1500
+ "correction": "المدرسة",
1501
+ "end": 7,
1502
+ "id": "8eedd2a1-d3fb-4709-80c0-36a959b440c2",
1503
+ "locked": true,
1504
+ "original": "المدرسه",
1505
+ "priority": 1,
1506
+ "start": 0,
1507
+ "type": "spelling"
1508
+ }
1509
+ ],
1510
+ "pipeline_ms": 6426,
1511
+ "pipeline_timing": {
1512
+ "grammar_ms": 597,
1513
+ "punctuation_ms": 546,
1514
+ "spelling_ms": 2681,
1515
+ "total_ms": 3828
1516
+ },
1517
+ "verdict": "PASS",
1518
+ "root_cause": "StageLocker operating correctly",
1519
+ "failure_detail": "",
1520
+ "grammar_raw_changed": true,
1521
+ "pipeline_changed": true,
1522
+ "raw_vs_pipeline_match": true,
1523
+ "span_errors": [],
1524
+ "offset_drift": 0
1525
+ },
1526
+ {
1527
+ "vector_id": "SLK-003",
1528
+ "category": "stagelocker",
1529
+ "subcategory": "punct_respects_grammar_lock",
1530
+ "input_text": "الطلاب ذهبوا الى المكتبة لقراءة الكتب الجديدة",
1531
+ "expected_behavior": "Grammar unchanged (correct text). Punctuation should NOT move/corrupt words",
1532
+ "grammar_raw_output": "الطلاب ذهبوا الى المكتبة لقراءة الكتب الجديدة",
1533
+ "grammar_raw_ms": 3155,
1534
+ "punctuation_raw_output": "الطلاب ذهبوا الى المكتبة لقراءة الكتب الجديدة!",
1535
+ "punctuation_raw_ms": 3522,
1536
+ "pipeline_output": "الطلاب ذهبوا إلى المكتبة لقراءة الكتب الجديدة.",
1537
+ "pipeline_corrected": "الطلاب ذهبوا إلى المكتبة لقراءة الكتب الجديدة.",
1538
+ "pipeline_suggestions": [
1539
+ {
1540
+ "alternatives": [],
1541
+ "confidence": 0.8,
1542
+ "correction": "الجديدة.",
1543
+ "end": 45,
1544
+ "id": "3893b371-9b48-47db-b594-af7c91784cf0",
1545
+ "locked": true,
1546
+ "original": "الجديدة",
1547
+ "priority": 2,
1548
+ "start": 38,
1549
+ "type": "punctuation"
1550
+ },
1551
+ {
1552
+ "alternatives": [
1553
+ "إلى",
1554
+ "ال",
1555
+ "الم",
1556
+ "الى"
1557
+ ],
1558
+ "confidence": 0.9,
1559
+ "correction": "إلى",
1560
+ "end": 16,
1561
+ "id": "59e00d41-8699-4ae1-bc78-ac1d9af75ad7",
1562
+ "locked": true,
1563
+ "original": "الى",
1564
+ "priority": 1,
1565
+ "start": 13,
1566
+ "type": "spelling"
1567
+ }
1568
+ ],
1569
+ "pipeline_ms": 7703,
1570
+ "pipeline_timing": {
1571
+ "grammar_ms": 944,
1572
+ "punctuation_ms": 888,
1573
+ "spelling_ms": 2923,
1574
+ "total_ms": 4764
1575
+ },
1576
+ "verdict": "PASS",
1577
+ "root_cause": "StageLocker operating correctly",
1578
+ "failure_detail": "",
1579
+ "grammar_raw_changed": false,
1580
+ "pipeline_changed": true,
1581
+ "raw_vs_pipeline_match": false,
1582
+ "span_errors": [],
1583
+ "offset_drift": 0
1584
+ },
1585
+ {
1586
+ "vector_id": "SLK-004",
1587
+ "category": "stagelocker",
1588
+ "subcategory": "overlapping_patches",
1589
+ "input_text": "الاطفال يلعبون بالكره في الحديقه الكبيره",
1590
+ "expected_behavior": "Spelling: الاطفال→الأطفال, بالكره→بالكرة, الحديقه→الحديقة, الكبيره→الكبيرة — multiple overlapping spans",
1591
+ "grammar_raw_output": "الأطفال يلعبون بالكرة في الحديقة الكبيرة",
1592
+ "grammar_raw_ms": 3386,
1593
+ "punctuation_raw_output": "الاطفال يلعبون بالكره في الحديقه الكبيره!",
1594
+ "punctuation_raw_ms": 3407,
1595
+ "pipeline_output": "الأطفال يلعبون بالكرة في الحديقة الكبيرة.",
1596
+ "pipeline_corrected": "الأطفال يلعبون بالكرة في الحديقة الكبيرة.",
1597
+ "pipeline_suggestions": [
1598
+ {
1599
+ "alternatives": [],
1600
+ "confidence": 0.8,
1601
+ "correction": "الكبيرة.",
1602
+ "end": 40,
1603
+ "id": "3aa85f52-d09d-4b37-b09b-22b62182b700",
1604
+ "locked": true,
1605
+ "original": "الكبيره",
1606
+ "priority": 2,
1607
+ "start": 33,
1608
+ "type": "punctuation"
1609
+ },
1610
+ {
1611
+ "alternatives": [
1612
+ "الأطفال",
1613
+ "للاطفال",
1614
+ "الاطفاء",
1615
+ "الاطفال"
1616
+ ],
1617
+ "confidence": 0.9,
1618
+ "correction": "الأطفال",
1619
+ "end": 7,
1620
+ "id": "303beeea-6796-4336-a28b-d2d5765bbe38",
1621
+ "locked": true,
1622
+ "original": "الاطفال",
1623
+ "priority": 1,
1624
+ "start": 0,
1625
+ "type": "spelling"
1626
+ },
1627
+ {
1628
+ "alternatives": [
1629
+ "بالكرة",
1630
+ "��الكر",
1631
+ "الكره",
1632
+ "بالكره"
1633
+ ],
1634
+ "confidence": 0.9,
1635
+ "correction": "بالكرة",
1636
+ "end": 21,
1637
+ "id": "cca7efd9-8123-4380-a468-3d0faa4dec13",
1638
+ "locked": true,
1639
+ "original": "بالكره",
1640
+ "priority": 1,
1641
+ "start": 15,
1642
+ "type": "spelling"
1643
+ },
1644
+ {
1645
+ "alternatives": [
1646
+ "الحديقة",
1647
+ "الحديقه"
1648
+ ],
1649
+ "confidence": 0.9,
1650
+ "correction": "الحديقة",
1651
+ "end": 32,
1652
+ "id": "b9274c96-4e7e-4c45-8ed3-95d5fe5cd88b",
1653
+ "locked": true,
1654
+ "original": "الحديقه",
1655
+ "priority": 1,
1656
+ "start": 25,
1657
+ "type": "spelling"
1658
+ }
1659
+ ],
1660
+ "pipeline_ms": 8575,
1661
+ "pipeline_timing": {
1662
+ "grammar_ms": 893,
1663
+ "punctuation_ms": 860,
1664
+ "spelling_ms": 4372,
1665
+ "total_ms": 6132
1666
+ },
1667
+ "verdict": "PASS",
1668
+ "root_cause": "StageLocker operating correctly",
1669
+ "failure_detail": "",
1670
+ "grammar_raw_changed": true,
1671
+ "pipeline_changed": true,
1672
+ "raw_vs_pipeline_match": false,
1673
+ "span_errors": [],
1674
+ "offset_drift": 0
1675
+ },
1676
+ {
1677
+ "vector_id": "SLK-005",
1678
+ "category": "stagelocker",
1679
+ "subcategory": "backward_overwrite",
1680
+ "input_text": "هذه المدينه جميله وهادئه",
1681
+ "expected_behavior": "Three ه→ة fixes. Grammar must NOT overwrite any of them back to ه",
1682
+ "grammar_raw_output": "هذه المدينة جميلة وهادئة",
1683
+ "grammar_raw_ms": 4014,
1684
+ "punctuation_raw_output": "هذه المدينه جميله وهادئه!",
1685
+ "punctuation_raw_ms": 3050,
1686
+ "pipeline_output": "هذه المدينة جميلة وهادئة!",
1687
+ "pipeline_corrected": "هذه المدينة جميلة وهادئة!",
1688
+ "pipeline_suggestions": [
1689
+ {
1690
+ "alternatives": [],
1691
+ "confidence": 0.8,
1692
+ "correction": "وهادئة!",
1693
+ "end": 24,
1694
+ "id": "5171793c-ef57-4d42-8bf0-44bb0c4c3472",
1695
+ "locked": true,
1696
+ "original": "وهادئه",
1697
+ "priority": 2,
1698
+ "start": 18,
1699
+ "type": "punctuation"
1700
+ },
1701
+ {
1702
+ "alternatives": [
1703
+ "المدينة",
1704
+ "المدين",
1705
+ "المدينه"
1706
+ ],
1707
+ "confidence": 0.9,
1708
+ "correction": "المدينة",
1709
+ "end": 11,
1710
+ "id": "76b1dbc3-590d-4db0-a353-ba2eb0e7fa76",
1711
+ "locked": true,
1712
+ "original": "المدينه",
1713
+ "priority": 1,
1714
+ "start": 4,
1715
+ "type": "spelling"
1716
+ },
1717
+ {
1718
+ "alternatives": [
1719
+ "جميلة",
1720
+ "جميل",
1721
+ "زميله",
1722
+ "جميله"
1723
+ ],
1724
+ "confidence": 0.9,
1725
+ "correction": "جميلة",
1726
+ "end": 17,
1727
+ "id": "7834d0e8-f4d2-421d-8b46-4199afe493b9",
1728
+ "locked": true,
1729
+ "original": "جميله",
1730
+ "priority": 1,
1731
+ "start": 12,
1732
+ "type": "spelling"
1733
+ }
1734
+ ],
1735
+ "pipeline_ms": 6328,
1736
+ "pipeline_timing": {
1737
+ "grammar_ms": 711,
1738
+ "punctuation_ms": 503,
1739
+ "spelling_ms": 2463,
1740
+ "total_ms": 3682
1741
+ },
1742
+ "verdict": "PASS",
1743
+ "root_cause": "No backward overwrite detected",
1744
+ "failure_detail": "",
1745
+ "grammar_raw_changed": true,
1746
+ "pipeline_changed": true,
1747
+ "raw_vs_pipeline_match": false,
1748
+ "span_errors": [],
1749
+ "offset_drift": 0
1750
+ },
1751
+ {
1752
+ "vector_id": "SLK-006",
1753
+ "category": "stagelocker",
1754
+ "subcategory": "grammar_overrides_spelling_adjacent",
1755
+ "input_text": "الطالبه ذكي في المدرسه",
1756
+ "expected_behavior": "Spelling fixes ه→ة, Grammar must fix gender (ذكي→ذكية) on ADJACENT word",
1757
+ "grammar_raw_output": "الطالبة ذكية في المدرسة",
1758
+ "grammar_raw_ms": 3893,
1759
+ "punctuation_raw_output": "الطالبه ذكي في المدرسه.",
1760
+ "punctuation_raw_ms": 3501,
1761
+ "pipeline_output": "الطالبة ذكية في المدرسة.",
1762
+ "pipeline_corrected": "الطالبة ذكية في المدرسة.",
1763
+ "pipeline_suggestions": [
1764
+ {
1765
+ "alternatives": [],
1766
+ "confidence": 1.0,
1767
+ "correction": "الطالبة ذكية",
1768
+ "end": 11,
1769
+ "id": "d34734eb-f09f-47c4-9ee5-f16e474df973",
1770
+ "locked": true,
1771
+ "original": "الطالبه ذكي",
1772
+ "priority": 3,
1773
+ "start": 0,
1774
+ "type": "grammar"
1775
+ },
1776
+ {
1777
+ "alternatives": [],
1778
+ "confidence": 0.8,
1779
+ "correction": "المدرسة.",
1780
+ "end": 22,
1781
+ "id": "483c3a0a-1ac6-4c97-bfee-a7a514aef898",
1782
+ "locked": true,
1783
+ "original": "المدرسه",
1784
+ "priority": 2,
1785
+ "start": 15,
1786
+ "type": "punctuation"
1787
+ }
1788
+ ],
1789
+ "pipeline_ms": 7047,
1790
+ "pipeline_timing": {
1791
+ "grammar_ms": 646,
1792
+ "punctuation_ms": 555,
1793
+ "spelling_ms": 2952,
1794
+ "total_ms": 4157
1795
+ },
1796
+ "verdict": "PASS",
1797
+ "root_cause": "StageLocker operating correctly",
1798
+ "failure_detail": "",
1799
+ "grammar_raw_changed": true,
1800
+ "pipeline_changed": true,
1801
+ "raw_vs_pipeline_match": false,
1802
+ "span_errors": [],
1803
+ "offset_drift": 0
1804
+ },
1805
+ {
1806
+ "vector_id": "OVR-001",
1807
+ "category": "overcorrection",
1808
+ "subcategory": "correct_text_unchanged",
1809
+ "input_text": "ذهب الطالب إلى المكتبة وقرأ كتاباً مفيداً",
1810
+ "expected_behavior": "Perfectly correct text — should return UNCHANGED",
1811
+ "grammar_raw_output": "ذهب الطالب إلى المكتبة وقرأ كتابا مفيدا",
1812
+ "grammar_raw_ms": 3996,
1813
+ "punctuation_raw_output": "ذهب الطالب إلى المكتبة وقرأ، كتاباً مفيداً",
1814
+ "punctuation_raw_ms": 4282,
1815
+ "pipeline_output": "ذهب الطالب إلى المكتبة وقرأ كتاباً مفيداً",
1816
+ "pipeline_corrected": "ذهب الطالب إلى المكتبة وقرأ كتاباً مفيداً",
1817
+ "pipeline_suggestions": [],
1818
+ "pipeline_ms": 10781,
1819
+ "pipeline_timing": {
1820
+ "grammar_ms": 1207,
1821
+ "punctuation_ms": 799,
1822
+ "spelling_ms": 4593,
1823
+ "total_ms": 6605
1824
+ },
1825
+ "verdict": "PASS",
1826
+ "root_cause": "Correct text preserved",
1827
+ "failure_detail": "",
1828
+ "grammar_raw_changed": true,
1829
+ "pipeline_changed": false,
1830
+ "raw_vs_pipeline_match": true,
1831
+ "span_errors": [],
1832
+ "offset_drift": 0
1833
+ },
1834
+ {
1835
+ "vector_id": "OVR-002",
1836
+ "category": "overcorrection",
1837
+ "subcategory": "correct_grammar_unchanged",
1838
+ "input_text": "المهندسون يعملون في المصنع الكبير بجدٍ واجتهاد",
1839
+ "expected_behavior": "Correct SV agreement + tanween — must not be modified",
1840
+ "grammar_raw_output": "المهندسون يعملون في المصنع الكبير بجد واجتهاد",
1841
+ "grammar_raw_ms": 4220,
1842
+ "punctuation_raw_output": "المهندسون يعملون في المصنع الكبير بجدٍ واجتهاد.",
1843
+ "punctuation_raw_ms": 3964,
1844
+ "pipeline_output": "المهندسون يعملون في المصنع الكبير بجدٍ واجتهاد.",
1845
+ "pipeline_corrected": "المهندسون يعملون في المصنع الكبير بجدٍ واجتهاد.",
1846
+ "pipeline_suggestions": [
1847
+ {
1848
+ "alternatives": [],
1849
+ "confidence": 0.8,
1850
+ "correction": "واجتهاد.",
1851
+ "end": 46,
1852
+ "id": "2ee09f18-187f-42a4-83bc-e81d696ffa80",
1853
+ "locked": true,
1854
+ "original": "واجتهاد",
1855
+ "priority": 2,
1856
+ "start": 39,
1857
+ "type": "punctuation"
1858
+ }
1859
+ ],
1860
+ "pipeline_ms": 10058,
1861
+ "pipeline_timing": {
1862
+ "grammar_ms": 948,
1863
+ "punctuation_ms": 833,
1864
+ "spelling_ms": 5345,
1865
+ "total_ms": 7132
1866
+ },
1867
+ "verdict": "FAIL_FILTER",
1868
+ "root_cause": "Correct text was modified: ['واجتهاد→واجتهاد.']",
1869
+ "failure_detail": "All changes: ['واجتهاد→واجتهاد.']",
1870
+ "grammar_raw_changed": true,
1871
+ "pipeline_changed": true,
1872
+ "raw_vs_pipeline_match": false,
1873
+ "span_errors": [],
1874
+ "offset_drift": 0
1875
+ },
1876
+ {
1877
+ "vector_id": "OVR-003",
1878
+ "category": "overcorrection",
1879
+ "subcategory": "jaccard_threshold_test",
1880
+ "input_text": "جالس في البيت يقرأ الصحيفة",
1881
+ "expected_behavior": "Correct text. Grammar should not hallucinate جالس→جاكسون (Jaccard < 0.3)",
1882
+ "grammar_raw_output": "جالس في البيت يقرأ الصحيفة",
1883
+ "grammar_raw_ms": 4351,
1884
+ "punctuation_raw_output": "جالس في البيت يقرأ الصحيفة.",
1885
+ "punctuation_raw_ms": 5021,
1886
+ "pipeline_output": "جالس في البيت يقرأ الصحيفة.",
1887
+ "pipeline_corrected": "جالس في البيت يقرأ الصحيفة.",
1888
+ "pipeline_suggestions": [
1889
+ {
1890
+ "alternatives": [],
1891
+ "confidence": 0.8,
1892
+ "correction": "الصحيفة.",
1893
+ "end": 26,
1894
+ "id": "ec5b7d9a-fda2-4d2e-ab19-ffc202ef15f2",
1895
+ "locked": true,
1896
+ "original": "الصحيفة",
1897
+ "priority": 2,
1898
+ "start": 19,
1899
+ "type": "punctuation"
1900
+ }
1901
+ ],
1902
+ "pipeline_ms": 8821,
1903
+ "pipeline_timing": {
1904
+ "grammar_ms": 789,
1905
+ "punctuation_ms": 548,
1906
+ "spelling_ms": 3544,
1907
+ "total_ms": 4887
1908
+ },
1909
+ "verdict": "PASS",
1910
+ "root_cause": "No overcorrection detected",
1911
+ "failure_detail": "",
1912
+ "grammar_raw_changed": false,
1913
+ "pipeline_changed": true,
1914
+ "raw_vs_pipeline_match": false,
1915
+ "span_errors": [],
1916
+ "offset_drift": 0
1917
+ },
1918
+ {
1919
+ "vector_id": "OVR-004",
1920
+ "category": "overcorrection",
1921
+ "subcategory": "meaning_change_block",
1922
+ "input_text": "وكان الجو صحواً اليوم",
1923
+ "expected_behavior": "وكان must NOT become وكأن (directional block). Correct as-is",
1924
+ "grammar_raw_output": "وكان الجو صحوا اليوم",
1925
+ "grammar_raw_ms": 5367,
1926
+ "punctuation_raw_output": "وكان الجو صحواً اليوم.",
1927
+ "punctuation_raw_ms": 3066,
1928
+ "pipeline_output": "وكان الجو صحواً اليوم.",
1929
+ "pipeline_corrected": "وكان الجو صحواً اليوم.",
1930
+ "pipeline_suggestions": [
1931
+ {
1932
+ "alternatives": [],
1933
+ "confidence": 0.8,
1934
+ "correction": "اليوم.",
1935
+ "end": 21,
1936
+ "id": "e02e158e-9dc7-4876-9d9b-d618eb58ab9f",
1937
+ "locked": true,
1938
+ "original": "اليوم",
1939
+ "priority": 2,
1940
+ "start": 16,
1941
+ "type": "punctuation"
1942
+ }
1943
+ ],
1944
+ "pipeline_ms": 6751,
1945
+ "pipeline_timing": {
1946
+ "grammar_ms": 788,
1947
+ "punctuation_ms": 513,
1948
+ "spelling_ms": 2977,
1949
+ "total_ms": 4281
1950
+ },
1951
+ "verdict": "PASS",
1952
+ "root_cause": "Meaning preserved",
1953
+ "failure_detail": "",
1954
+ "grammar_raw_changed": true,
1955
+ "pipeline_changed": true,
1956
+ "raw_vs_pipeline_match": false,
1957
+ "span_errors": [],
1958
+ "offset_drift": 0
1959
+ },
1960
+ {
1961
+ "vector_id": "OVR-005",
1962
+ "category": "overcorrection",
1963
+ "subcategory": "rare_word_dampening",
1964
+ "input_text": "الفيلسوف سقراط تأمل الحقيقة",
1965
+ "expected_behavior": "سقراط is a rare valid name — spelling should not 'correct' it",
1966
+ "grammar_raw_output": "الفيلسوف سقراط تأمل الحقيقة",
1967
+ "grammar_raw_ms": 6077,
1968
+ "punctuation_raw_output": "الفيلسوف سقراط تأمل الحقيقة.",
1969
+ "punctuation_raw_ms": 4019,
1970
+ "pipeline_output": "الفيلسوف سقراط تأمل الحقيقة.",
1971
+ "pipeline_corrected": "الفيلسوف سقراط تأمل الحقيقة.",
1972
+ "pipeline_suggestions": [
1973
+ {
1974
+ "alternatives": [],
1975
+ "confidence": 0.8,
1976
+ "correction": "الحقيقة.",
1977
+ "end": 27,
1978
+ "id": "509344fd-27fc-4476-9f09-7eb0ac1a87da",
1979
+ "locked": true,
1980
+ "original": "الحقيقة",
1981
+ "priority": 2,
1982
+ "start": 20,
1983
+ "type": "punctuation"
1984
+ }
1985
+ ],
1986
+ "pipeline_ms": 7657,
1987
+ "pipeline_timing": {
1988
+ "grammar_ms": 597,
1989
+ "punctuation_ms": 580,
1990
+ "spelling_ms": 2414,
1991
+ "total_ms": 3593
1992
+ },
1993
+ "verdict": "PASS",
1994
+ "root_cause": "No overcorrection detected",
1995
+ "failure_detail": "",
1996
+ "grammar_raw_changed": false,
1997
+ "pipeline_changed": true,
1998
+ "raw_vs_pipeline_match": false,
1999
+ "span_errors": [],
2000
+ "offset_drift": 0
2001
+ },
2002
+ {
2003
+ "vector_id": "OVR-006",
2004
+ "category": "overcorrection",
2005
+ "subcategory": "filter_over_blocking",
2006
+ "input_text": "انا ذاهب الي المدرسه",
2007
+ "expected_behavior": "IV-IV guard should NOT block انا→أنا or الي→إلى (these are whitelist entries)",
2008
+ "grammar_raw_output": "أنا ذاهب إلى المدرسة",
2009
+ "grammar_raw_ms": 5475,
2010
+ "punctuation_raw_output": "انا ذاهب الي المدرسه.",
2011
+ "punctuation_raw_ms": 4105,
2012
+ "pipeline_output": "أنا ذاهب إلى المدرسة.",
2013
+ "pipeline_corrected": "أنا ذاهب إلى المدرسة.",
2014
+ "pipeline_suggestions": [
2015
+ {
2016
+ "alternatives": [],
2017
+ "confidence": 0.8,
2018
+ "correction": "المدرسة.",
2019
+ "end": 20,
2020
+ "id": "2c36a146-f2c7-4788-906e-5ccb09de7725",
2021
+ "locked": true,
2022
+ "original": "المدرسه",
2023
+ "priority": 2,
2024
+ "start": 13,
2025
+ "type": "punctuation"
2026
+ },
2027
+ {
2028
+ "alternatives": [
2029
+ "أنا",
2030
+ "الا",
2031
+ "ان",
2032
+ "انا"
2033
+ ],
2034
+ "confidence": 0.9,
2035
+ "correction": "أنا",
2036
+ "end": 3,
2037
+ "id": "1d5583a7-6781-4d91-a8d4-1c30d498ad40",
2038
+ "locked": true,
2039
+ "original": "انا",
2040
+ "priority": 1,
2041
+ "start": 0,
2042
+ "type": "spelling"
2043
+ },
2044
+ {
2045
+ "alternatives": [
2046
+ "إلى",
2047
+ "ال",
2048
+ "الم",
2049
+ "الي"
2050
+ ],
2051
+ "confidence": 0.9,
2052
+ "correction": "إلى",
2053
+ "end": 12,
2054
+ "id": "1ad036d9-d9de-489f-8664-f2b365602c1d",
2055
+ "locked": true,
2056
+ "original": "الي",
2057
+ "priority": 1,
2058
+ "start": 9,
2059
+ "type": "spelling"
2060
+ }
2061
+ ],
2062
+ "pipeline_ms": 8499,
2063
+ "pipeline_timing": {
2064
+ "grammar_ms": 638,
2065
+ "punctuation_ms": 399,
2066
+ "spelling_ms": 3465,
2067
+ "total_ms": 4506
2068
+ },
2069
+ "verdict": "PASS",
2070
+ "root_cause": "No overcorrection detected",
2071
+ "failure_detail": "",
2072
+ "grammar_raw_changed": true,
2073
+ "pipeline_changed": true,
2074
+ "raw_vs_pipeline_match": false,
2075
+ "span_errors": [],
2076
+ "offset_drift": 0
2077
+ },
2078
+ {
2079
+ "vector_id": "OVR-007",
2080
+ "category": "overcorrection",
2081
+ "subcategory": "tanween_preservation",
2082
+ "input_text": "قرأت كتاباً جميلاً عن التاريخ",
2083
+ "expected_behavior": "Tanween (ً) must be preserved — grammar should not strip it",
2084
+ "grammar_raw_output": "قرأت كتابا جميلا عن التاريخ",
2085
+ "grammar_raw_ms": 5403,
2086
+ "punctuation_raw_output": "قرأت كتاباً جميلاً عن التاريخ!",
2087
+ "punctuation_raw_ms": 4864,
2088
+ "pipeline_output": "قرأت كتاباً جميلاً عن التاريخ!",
2089
+ "pipeline_corrected": "قرأت كتاباً جميلاً عن التاريخ!",
2090
+ "pipeline_suggestions": [
2091
+ {
2092
+ "alternatives": [],
2093
+ "confidence": 0.8,
2094
+ "correction": "التاريخ!",
2095
+ "end": 29,
2096
+ "id": "a516b100-8451-4611-a618-a3c914e05928",
2097
+ "locked": true,
2098
+ "original": "التاريخ",
2099
+ "priority": 2,
2100
+ "start": 22,
2101
+ "type": "punctuation"
2102
+ }
2103
+ ],
2104
+ "pipeline_ms": 5350,
2105
+ "pipeline_timing": {
2106
+ "grammar_ms": 1185,
2107
+ "punctuation_ms": 730,
2108
+ "spelling_ms": 2609,
2109
+ "total_ms": 4529
2110
+ },
2111
+ "verdict": "FAIL_FILTER",
2112
+ "root_cause": "Correct text was modified: ['التاريخ→التاريخ!']",
2113
+ "failure_detail": "All changes: ['التاريخ→التاريخ!']",
2114
+ "grammar_raw_changed": true,
2115
+ "pipeline_changed": true,
2116
+ "raw_vs_pipeline_match": false,
2117
+ "span_errors": [],
2118
+ "offset_drift": 0
2119
+ },
2120
+ {
2121
+ "vector_id": "OVR-008",
2122
+ "category": "overcorrection",
2123
+ "subcategory": "correct_feminine_form",
2124
+ "input_text": "المعلمة الجديدة شرحت الدرس",
2125
+ "expected_behavior": "Already correct — must NOT change المعلمة to المعلمه",
2126
+ "grammar_raw_output": "المعلمة الجديدة شرحت الدرس",
2127
+ "grammar_raw_ms": 4415,
2128
+ "punctuation_raw_output": "المعلمة الجديدة شرحت الدرس.",
2129
+ "punctuation_raw_ms": 1481,
2130
+ "pipeline_output": "المعلمة الجديدة شرحت الدرس.",
2131
+ "pipeline_corrected": "المعلمة الجديدة شرحت الدرس.",
2132
+ "pipeline_suggestions": [
2133
+ {
2134
+ "alternatives": [],
2135
+ "confidence": 0.8,
2136
+ "correction": "الدرس.",
2137
+ "end": 26,
2138
+ "id": "31693a38-f540-437b-977b-f541204fa564",
2139
+ "locked": true,
2140
+ "original": "الدرس",
2141
+ "priority": 2,
2142
+ "start": 21,
2143
+ "type": "punctuation"
2144
+ }
2145
+ ],
2146
+ "pipeline_ms": 6679,
2147
+ "pipeline_timing": {
2148
+ "grammar_ms": 852,
2149
+ "punctuation_ms": 643,
2150
+ "spelling_ms": 1817,
2151
+ "total_ms": 3314
2152
+ },
2153
+ "verdict": "FAIL_FILTER",
2154
+ "root_cause": "Correct text was modified: ['الدرس→الدرس.']",
2155
+ "failure_detail": "All changes: ['الدرس→الدرس.']",
2156
+ "grammar_raw_changed": false,
2157
+ "pipeline_changed": true,
2158
+ "raw_vs_pipeline_match": false,
2159
+ "span_errors": [],
2160
+ "offset_drift": 0
2161
+ },
2162
+ {
2163
+ "vector_id": "OVR-009",
2164
+ "category": "overcorrection",
2165
+ "subcategory": "brackets_protection",
2166
+ "input_text": "الفصل الأول (المقدمة) يتناول الموضوع",
2167
+ "expected_behavior": "Brackets must be preserved. Grammar should not unbalance them",
2168
+ "grammar_raw_output": "الفصل الأول لمقدمة يتناول الموضوع",
2169
+ "grammar_raw_ms": 1685,
2170
+ "punctuation_raw_output": "الفصل الأول (المقدمة): يتناول الموضوع؛",
2171
+ "punctuation_raw_ms": 4323,
2172
+ "pipeline_output": "الفصل الأول (المقدمة) يتناول الموضوع",
2173
+ "pipeline_corrected": "الفصل الأول (المقدمة) يتناول الموضوع",
2174
+ "pipeline_suggestions": [],
2175
+ "pipeline_ms": 7948,
2176
+ "pipeline_timing": {
2177
+ "grammar_ms": 924,
2178
+ "punctuation_ms": 1380,
2179
+ "spelling_ms": 4723,
2180
+ "total_ms": 7030
2181
+ },
2182
+ "verdict": "PASS",
2183
+ "root_cause": "Brackets preserved",
2184
+ "failure_detail": "",
2185
+ "grammar_raw_changed": true,
2186
+ "pipeline_changed": false,
2187
+ "raw_vs_pipeline_match": false,
2188
+ "span_errors": [],
2189
+ "offset_drift": 0
2190
+ },
2191
+ {
2192
+ "vector_id": "OVR-010",
2193
+ "category": "overcorrection",
2194
+ "subcategory": "digits_protection",
2195
+ "input_text": "تأسست الشركة عام 2020 وبلغ عدد موظفيها 150 ��خصاً",
2196
+ "expected_behavior": "Digits must NOT be corrupted by grammar model",
2197
+ "grammar_raw_output": "تأسست الشركة عام 2020 وبلغ عدد موظفيها 150 شخصا",
2198
+ "grammar_raw_ms": 4709,
2199
+ "punctuation_raw_output": "تأسست الشركة عام 2020 وبلغ عدد موظفيها 150 شخصاً.",
2200
+ "punctuation_raw_ms": 1835,
2201
+ "pipeline_output": "تأسست الشركة عام يناير وبلغ عدد موظفيها خمسين شخصاً.",
2202
+ "pipeline_corrected": "تأسست الشركة عام يناير وبلغ عدد موظفيها خمسين شخصاً.",
2203
+ "pipeline_suggestions": [
2204
+ {
2205
+ "alternatives": [],
2206
+ "confidence": 0.8,
2207
+ "correction": "شخصاً.",
2208
+ "end": 48,
2209
+ "id": "3905f603-2a35-4786-a1cd-8d44c1995ba9",
2210
+ "locked": true,
2211
+ "original": "شخصاً",
2212
+ "priority": 2,
2213
+ "start": 43,
2214
+ "type": "punctuation"
2215
+ },
2216
+ {
2217
+ "alternatives": [
2218
+ "يناير",
2219
+ "2020"
2220
+ ],
2221
+ "confidence": 0.6,
2222
+ "correction": "يناير",
2223
+ "end": 21,
2224
+ "id": "99bdcd3f-20bf-4319-be00-cf116e7163a6",
2225
+ "locked": true,
2226
+ "original": "2020",
2227
+ "priority": 1,
2228
+ "start": 17,
2229
+ "type": "spelling"
2230
+ },
2231
+ {
2232
+ "alternatives": [
2233
+ "خمسين",
2234
+ "150"
2235
+ ],
2236
+ "confidence": 0.6,
2237
+ "correction": "خمسين",
2238
+ "end": 42,
2239
+ "id": "91023a54-9a4f-415c-83cc-746e12f64dc3",
2240
+ "locked": true,
2241
+ "original": "150",
2242
+ "priority": 1,
2243
+ "start": 39,
2244
+ "type": "spelling"
2245
+ }
2246
+ ],
2247
+ "pipeline_ms": 15379,
2248
+ "pipeline_timing": {
2249
+ "grammar_ms": 1106,
2250
+ "punctuation_ms": 905,
2251
+ "spelling_ms": 9998,
2252
+ "total_ms": 12016
2253
+ },
2254
+ "verdict": "FAIL_FILTER",
2255
+ "root_cause": "Digits were corrupted",
2256
+ "failure_detail": "",
2257
+ "grammar_raw_changed": true,
2258
+ "pipeline_changed": true,
2259
+ "raw_vs_pipeline_match": false,
2260
+ "span_errors": [],
2261
+ "offset_drift": 0
2262
+ },
2263
+ {
2264
+ "vector_id": "MIS-001",
2265
+ "category": "model_mismatch",
2266
+ "subcategory": "spelling_output_breaks_grammar",
2267
+ "input_text": "الطالبه المجتهده حصلت علي درجات عاليه",
2268
+ "expected_behavior": "Spelling: 4x ه→ة. Grammar should NOT break on the corrected output",
2269
+ "grammar_raw_output": "الطالبة المجتهدة حصلت على درجات عالية",
2270
+ "grammar_raw_ms": 1693,
2271
+ "punctuation_raw_output": "الطالبه المجتهده حصلت علي درجات عاليه.",
2272
+ "punctuation_raw_ms": 4126,
2273
+ "pipeline_output": "الطالبة المجتهدة حصلت علي درجات عالية.",
2274
+ "pipeline_corrected": "الطالبة المجتهدة حصلت علي درجات عالية.",
2275
+ "pipeline_suggestions": [
2276
+ {
2277
+ "alternatives": [],
2278
+ "confidence": 0.8,
2279
+ "correction": "عالية.",
2280
+ "end": 37,
2281
+ "id": "0405a474-1ea1-4d28-bca2-c3b8fc393b13",
2282
+ "locked": true,
2283
+ "original": "عاليه",
2284
+ "priority": 2,
2285
+ "start": 32,
2286
+ "type": "punctuation"
2287
+ },
2288
+ {
2289
+ "alternatives": [
2290
+ "الطالبة",
2291
+ "الطالب",
2292
+ "الطالبه"
2293
+ ],
2294
+ "confidence": 0.9,
2295
+ "correction": "الطالبة",
2296
+ "end": 7,
2297
+ "id": "64cd41c7-89da-46b2-b750-5555fdb7def0",
2298
+ "locked": true,
2299
+ "original": "الطالبه",
2300
+ "priority": 1,
2301
+ "start": 0,
2302
+ "type": "spelling"
2303
+ },
2304
+ {
2305
+ "alternatives": [
2306
+ "المجتهدة",
2307
+ "المجتهد",
2308
+ "المجتهده"
2309
+ ],
2310
+ "confidence": 0.9,
2311
+ "correction": "المجتهدة",
2312
+ "end": 16,
2313
+ "id": "c3b7cd46-ba07-4d6f-942e-fe1425e2b195",
2314
+ "locked": true,
2315
+ "original": "المجتهده",
2316
+ "priority": 1,
2317
+ "start": 8,
2318
+ "type": "spelling"
2319
+ }
2320
+ ],
2321
+ "pipeline_ms": 6675,
2322
+ "pipeline_timing": {
2323
+ "grammar_ms": 904,
2324
+ "punctuation_ms": 859,
2325
+ "spelling_ms": 4231,
2326
+ "total_ms": 6003
2327
+ },
2328
+ "verdict": "PASS",
2329
+ "root_cause": "Grammar handled spelling-corrected input",
2330
+ "failure_detail": "",
2331
+ "grammar_raw_changed": true,
2332
+ "pipeline_changed": true,
2333
+ "raw_vs_pipeline_match": false,
2334
+ "span_errors": [],
2335
+ "offset_drift": 0
2336
+ },
2337
+ {
2338
+ "vector_id": "MIS-002",
2339
+ "category": "model_mismatch",
2340
+ "subcategory": "grammar_undoes_spelling",
2341
+ "input_text": "ذهبنا الي المكتبه القريبه",
2342
+ "expected_behavior": "Spelling fixes الي��إلى, المكتبه→المكتبة. Grammar must NOT revert these",
2343
+ "grammar_raw_output": "ذهبنا إلى المكتبة القريبة",
2344
+ "grammar_raw_ms": 3975,
2345
+ "punctuation_raw_output": "ذهبنا الي المكتبه القريبه.",
2346
+ "punctuation_raw_ms": 1300,
2347
+ "pipeline_output": "ذهبنا إلى المكتبة القريبة.",
2348
+ "pipeline_corrected": "ذهبنا إلى المكتبة القريبة.",
2349
+ "pipeline_suggestions": [
2350
+ {
2351
+ "alternatives": [],
2352
+ "confidence": 0.8,
2353
+ "correction": "القريبة.",
2354
+ "end": 25,
2355
+ "id": "bc5dd39c-d7b1-4b9a-b7bd-f62a6d5d3f3d",
2356
+ "locked": true,
2357
+ "original": "القريبه",
2358
+ "priority": 2,
2359
+ "start": 18,
2360
+ "type": "punctuation"
2361
+ },
2362
+ {
2363
+ "alternatives": [
2364
+ "إلى",
2365
+ "ال",
2366
+ "الم",
2367
+ "الي"
2368
+ ],
2369
+ "confidence": 0.9,
2370
+ "correction": "إلى",
2371
+ "end": 9,
2372
+ "id": "076de2fe-4498-42bb-b3c8-4ace6cf17958",
2373
+ "locked": true,
2374
+ "original": "الي",
2375
+ "priority": 1,
2376
+ "start": 6,
2377
+ "type": "spelling"
2378
+ },
2379
+ {
2380
+ "alternatives": [
2381
+ "المكتبة",
2382
+ "المكتب",
2383
+ "المشتبه",
2384
+ "المكتبه"
2385
+ ],
2386
+ "confidence": 0.9,
2387
+ "correction": "المكتبة",
2388
+ "end": 17,
2389
+ "id": "6a5d280b-17d1-4122-aa8a-71371c304412",
2390
+ "locked": true,
2391
+ "original": "المكتبه",
2392
+ "priority": 1,
2393
+ "start": 10,
2394
+ "type": "spelling"
2395
+ }
2396
+ ],
2397
+ "pipeline_ms": 6505,
2398
+ "pipeline_timing": {
2399
+ "grammar_ms": 698,
2400
+ "punctuation_ms": 523,
2401
+ "spelling_ms": 2621,
2402
+ "total_ms": 3850
2403
+ },
2404
+ "verdict": "PASS",
2405
+ "root_cause": "Spelling corrections preserved through grammar",
2406
+ "failure_detail": "",
2407
+ "grammar_raw_changed": true,
2408
+ "pipeline_changed": true,
2409
+ "raw_vs_pipeline_match": false,
2410
+ "span_errors": [],
2411
+ "offset_drift": 0
2412
+ },
2413
+ {
2414
+ "vector_id": "MIS-003",
2415
+ "category": "model_mismatch",
2416
+ "subcategory": "cascading_length_change",
2417
+ "input_text": "اريد ان اذهب الي المدرسه لانني احب القراءه",
2418
+ "expected_behavior": "Many hamza + ه→ة fixes. Total text length changes significantly — grammar must handle shifted text",
2419
+ "grammar_raw_output": "أريد أن أذهب إلى المدرسة لأنني أحب القراءة",
2420
+ "grammar_raw_ms": 1581,
2421
+ "punctuation_raw_output": "اريد ان اذهب الي المدرسه لانني؛ احب القراءه",
2422
+ "punctuation_raw_ms": 3600,
2423
+ "pipeline_output": "أريد أن أذهب إلى المدرسة لأنني أحب القراءة.",
2424
+ "pipeline_corrected": "أريد أن أذهب إلى المدرسة لأنني أحب القراءة.",
2425
+ "pipeline_suggestions": [
2426
+ {
2427
+ "alternatives": [],
2428
+ "confidence": 0.8,
2429
+ "correction": "القراءة.",
2430
+ "end": 42,
2431
+ "id": "d2e08076-8ac6-4e91-8fc0-0eb1d7e7577a",
2432
+ "locked": true,
2433
+ "original": "القراءه",
2434
+ "priority": 2,
2435
+ "start": 35,
2436
+ "type": "punctuation"
2437
+ },
2438
+ {
2439
+ "alternatives": [],
2440
+ "confidence": 1.0,
2441
+ "correction": "لأنني",
2442
+ "end": 30,
2443
+ "id": "c15ca269-0b85-4795-880a-ddf56a205817",
2444
+ "locked": true,
2445
+ "original": "لانني",
2446
+ "priority": 1,
2447
+ "start": 25,
2448
+ "type": "spelling"
2449
+ },
2450
+ {
2451
+ "alternatives": [
2452
+ "أريد",
2453
+ "يريد",
2454
+ "تريد",
2455
+ "اريد"
2456
+ ],
2457
+ "confidence": 0.9,
2458
+ "correction": "أريد",
2459
+ "end": 4,
2460
+ "id": "b97dcb5a-12cf-464e-99e2-85de15de80cf",
2461
+ "locked": true,
2462
+ "original": "اريد",
2463
+ "priority": 1,
2464
+ "start": 0,
2465
+ "type": "spelling"
2466
+ },
2467
+ {
2468
+ "alternatives": [
2469
+ "أن",
2470
+ "ال",
2471
+ "من",
2472
+ "ان"
2473
+ ],
2474
+ "confidence": 0.9,
2475
+ "correction": "أن",
2476
+ "end": 7,
2477
+ "id": "9e37bf45-10d4-4114-b31f-011110310241",
2478
+ "locked": true,
2479
+ "original": "ان",
2480
+ "priority": 1,
2481
+ "start": 5,
2482
+ "type": "spelling"
2483
+ },
2484
+ {
2485
+ "alternatives": [
2486
+ "أذهب",
2487
+ "الذهب",
2488
+ "ذهب",
2489
+ "اذهب"
2490
+ ],
2491
+ "confidence": 0.9,
2492
+ "correction": "أذهب",
2493
+ "end": 12,
2494
+ "id": "0952785f-40cb-4860-82ef-18b3cf85f53f",
2495
+ "locked": true,
2496
+ "original": "اذهب",
2497
+ "priority": 1,
2498
+ "start": 8,
2499
+ "type": "spelling"
2500
+ },
2501
+ {
2502
+ "alternatives": [
2503
+ "إلى",
2504
+ "ال",
2505
+ "الم",
2506
+ "الي"
2507
+ ],
2508
+ "confidence": 0.9,
2509
+ "correction": "إلى",
2510
+ "end": 16,
2511
+ "id": "92973715-dfca-4a84-bf9a-2903b53478b2",
2512
+ "locked": true,
2513
+ "original": "الي",
2514
+ "priority": 1,
2515
+ "start": 13,
2516
+ "type": "spelling"
2517
+ },
2518
+ {
2519
+ "alternatives": [
2520
+ "المدرسة",
2521
+ "المدرسي",
2522
+ "المدرس",
2523
+ "المدرسه"
2524
+ ],
2525
+ "confidence": 0.9,
2526
+ "correction": "المدرسة",
2527
+ "end": 24,
2528
+ "id": "8cc179c2-89ea-4235-b0a3-05e00e1e6482",
2529
+ "locked": true,
2530
+ "original": "المدرسه",
2531
+ "priority": 1,
2532
+ "start": 17,
2533
+ "type": "spelling"
2534
+ },
2535
+ {
2536
+ "alternatives": [
2537
+ "أحب",
2538
+ "الب",
2539
+ "احت",
2540
+ "احب"
2541
+ ],
2542
+ "confidence": 0.9,
2543
+ "correction": "أحب",
2544
+ "end": 34,
2545
+ "id": "b0432045-ba7a-45fb-b538-efe62ff83026",
2546
+ "locked": true,
2547
+ "original": "احب",
2548
+ "priority": 1,
2549
+ "start": 31,
2550
+ "type": "spelling"
2551
+ }
2552
+ ],
2553
+ "pipeline_ms": 13520,
2554
+ "pipeline_timing": {
2555
+ "grammar_ms": 1105,
2556
+ "punctuation_ms": 836,
2557
+ "spelling_ms": 10929,
2558
+ "total_ms": 12888
2559
+ },
2560
+ "verdict": "PASS",
2561
+ "root_cause": "No model mismatch detected",
2562
+ "failure_detail": "",
2563
+ "grammar_raw_changed": true,
2564
+ "pipeline_changed": true,
2565
+ "raw_vs_pipeline_match": false,
2566
+ "span_errors": [],
2567
+ "offset_drift": 0
2568
+ },
2569
+ {
2570
+ "vector_id": "MIS-004",
2571
+ "category": "model_mismatch",
2572
+ "subcategory": "grammar_hallucination_on_corrected",
2573
+ "input_text": "الاولاد يلعبون والبنات يذهبن",
2574
+ "expected_behavior": "Spelling: الاولاد→الأولاد. Grammar should NOT hallucinate on already-correct يلعبون/يذهبن",
2575
+ "grammar_raw_output": "الاولاد يلعبون والبنات يذهبن",
2576
+ "grammar_raw_ms": 4826,
2577
+ "punctuation_raw_output": "الاولاد يلعبون والبنات يذهبن.",
2578
+ "punctuation_raw_ms": 1438,
2579
+ "pipeline_output": "الاولاد يلعبون والبنات يذهبن.",
2580
+ "pipeline_corrected": "الاولاد يلعبون والبنات يذهبن.",
2581
+ "pipeline_suggestions": [
2582
+ {
2583
+ "alternatives": [],
2584
+ "confidence": 0.8,
2585
+ "correction": "يذهبن.",
2586
+ "end": 28,
2587
+ "id": "1b722906-7c41-4041-966b-405f03698d6c",
2588
+ "locked": true,
2589
+ "original": "يذهبن",
2590
+ "priority": 2,
2591
+ "start": 23,
2592
+ "type": "punctuation"
2593
+ }
2594
+ ],
2595
+ "pipeline_ms": 6558,
2596
+ "pipeline_timing": {
2597
+ "grammar_ms": 683,
2598
+ "punctuation_ms": 632,
2599
+ "spelling_ms": 2822,
2600
+ "total_ms": 4140
2601
+ },
2602
+ "verdict": "PASS",
2603
+ "root_cause": "No model mismatch detected",
2604
+ "failure_detail": "",
2605
+ "grammar_raw_changed": false,
2606
+ "pipeline_changed": true,
2607
+ "raw_vs_pipeline_match": false,
2608
+ "span_errors": [],
2609
+ "offset_drift": 0
2610
+ },
2611
+ {
2612
+ "vector_id": "MIS-005",
2613
+ "category": "model_mismatch",
2614
+ "subcategory": "punc_on_grammar_corrected",
2615
+ "input_text": "السيارات سريعة والطرق واسعه",
2616
+ "expected_behavior": "After spelling (واسعه→واسعة), punctuation should add period at end, not corrupt words",
2617
+ "grammar_raw_output": "السيارات سريعة والطرق واسعة",
2618
+ "grammar_raw_ms": 1454,
2619
+ "punctuation_raw_output": "السيارات سريعة والطرق واسعه.",
2620
+ "punctuation_raw_ms": 3587,
2621
+ "pipeline_output": "السيارات سريعة والطرق واسعة.",
2622
+ "pipeline_corrected": "السيارات سريعة والطرق واسعة.",
2623
+ "pipeline_suggestions": [
2624
+ {
2625
+ "alternatives": [],
2626
+ "confidence": 0.8,
2627
+ "correction": "واسعة.",
2628
+ "end": 27,
2629
+ "id": "9780815c-9786-43c9-a81a-75437a0a9313",
2630
+ "locked": true,
2631
+ "original": "واسعه",
2632
+ "priority": 2,
2633
+ "start": 22,
2634
+ "type": "punctuation"
2635
+ }
2636
+ ],
2637
+ "pipeline_ms": 4889,
2638
+ "pipeline_timing": {
2639
+ "grammar_ms": 554,
2640
+ "punctuation_ms": 668,
2641
+ "spelling_ms": 2875,
2642
+ "total_ms": 4103
2643
+ },
2644
+ "verdict": "PASS",
2645
+ "root_cause": "No model mismatch detected",
2646
+ "failure_detail": "",
2647
+ "grammar_raw_changed": true,
2648
+ "pipeline_changed": true,
2649
+ "raw_vs_pipeline_match": false,
2650
+ "span_errors": [],
2651
+ "offset_drift": 0
2652
+ }
2653
+ ]
2654
+ }