Spaces:
Running
Running
Upload 31 files
Browse files- .gitattributes +1 -0
- README.md +129 -4
- README_AR.md +111 -0
- REFERENCES.md +121 -0
- REFERENCES_AR.md +37 -0
- alignment.py +180 -0
- app.js +158 -0
- app.py +154 -0
- benchmark.py +59 -0
- cache.js +25 -0
- camelbert.js +49 -0
- config.js +17 -0
- detector.py +140 -0
- examples.json +147 -0
- hadith.idx.gz +3 -0
- hadith.json +3 -0
- idgham.py +71 -0
- index.html +62 -17
- index_builder.py +163 -0
- islamic_unified_dataset.jsonl +0 -0
- islamiceval_dev_subset.jsonl +0 -0
- llm_client.py +104 -0
- normalization.py +117 -0
- quran.idx.gz +3 -0
- quran.json +0 -0
- retrieval.py +187 -0
- scanner.py +313 -0
- similarity.py +121 -0
- styles.css +193 -0
- ui.py +396 -0
- verifier.py +503 -0
- worker.js +72 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
hadith.json filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -1,10 +1,135 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
colorTo: yellow
|
| 6 |
sdk: static
|
|
|
|
| 7 |
pinned: false
|
|
|
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: التحقق من هلوسة القرآن والحديث وتصحيحها
|
| 3 |
+
emoji: 🕌
|
| 4 |
+
colorFrom: green
|
| 5 |
colorTo: yellow
|
| 6 |
sdk: static
|
| 7 |
+
app_file: index.html
|
| 8 |
pinned: false
|
| 9 |
+
license: mit
|
| 10 |
+
short_description: التحقق من هلوسة القرآن والحديث وتصحيحها
|
| 11 |
---
|
| 12 |
|
| 13 |
+
|
| 14 |
+
# التحقق من هلوسة القرآن والحديث وتصحيحها · Quran & Hadith Hallucination Verifier and Corrector
|
| 15 |
+
|
| 16 |
+
Arabic version: [README_AR.md](README_AR.md)
|
| 17 |
+
|
| 18 |
+
Detects Quran and Hadith quotations in any text (for example answers written by language models), verifies each one
|
| 19 |
+
against the sources, shows the evidence, and proposes a correction **taken verbatim from the source text**. When the
|
| 20 |
+
evidence is not sufficient the tool says so and refers the case to human review. The user interface is Arabic.
|
| 21 |
+
|
| 22 |
+
## Live Demo
|
| 23 |
+
|
| 24 |
+
**[LIVE_DEPLOYMENT_URL]** <!-- replace with the public Hugging Face Space URL after deployment -->
|
| 25 |
+
|
| 26 |
+
## Features
|
| 27 |
+
|
| 28 |
+
* **Direct verification:** paste any paragraph; every Quran / Hadith quotation is found, compared word by word and
|
| 29 |
+
highlighted, with the exact source text offered as the correction.
|
| 30 |
+
* **Ask then Verify:** type a question; a pre-configured ChatGPT (OpenAI) client answers and the same engine verifies the
|
| 31 |
+
answer and returns a corrected version. There is no model selection in this tab.
|
| 32 |
+
* **No trigger phrases required:** quotations are recognised by matching the text with the Quran and Hadith corpora, with or
|
| 33 |
+
without quotation marks and with or without openers such as "قال الله تعالى" or "قال رسول الله ﷺ". Such phrases are only a
|
| 34 |
+
weak fallback for altered quotations that the corpus cannot confirm.
|
| 35 |
+
* **Detection model selector (English):** *Standard · Rules + Corpus*, or *CAMeLBERT-MSA · Fine-tuned*. See below.
|
| 36 |
+
* **Benchmark export:** JSON and TSV in the IslamicEval 1A / 1B / 1C structure (see [Benchmark alignment](#benchmark-alignment)).
|
| 37 |
+
* **Copy corrected text:** a working clipboard button with a toast message ("تم نسخ النص بنجاح"), with a fallback for iframes.
|
| 38 |
+
* **Fast, informative loading:** the Quran index starts downloading before the scripts run, the Hadith index loads in the
|
| 39 |
+
background, files are cached by the browser (Cache API), verification results and demo examples are cached as JSON in
|
| 40 |
+
`localStorage`, and animated skeleton cards plus a three-step progress bar show what is happening.
|
| 41 |
+
|
| 42 |
+
## CAMeLBERT-MSA support
|
| 43 |
+
|
| 44 |
+
`web/camelbert.js` exposes one function, `detectSpans`, that returns BIO-tagged tokens (`B-Ayah`, `I-Ayah`, `B-Hadith`,
|
| 45 |
+
`I-Hadith`, `O`) and spans:
|
| 46 |
+
|
| 47 |
+
* **Live mode:** set `HF_MODEL_ID` (and `HF_TOKEN` for a private model) in `web/config.js`. The page calls the Hugging Face
|
| 48 |
+
Inference API for a fine-tuned token-classification model. The spans are then verified and corrected by the local pipeline.
|
| 49 |
+
* **Simulation mode (default):** with no model ID set, **no model weights are loaded**. The spans come from the standard
|
| 50 |
+
detector and are shown as BIO tags with deterministic pseudo-confidence values. The interface labels this clearly as
|
| 51 |
+
*Simulation*. Use it to demonstrate the flow, not to report model accuracy.
|
| 52 |
+
|
| 53 |
+
## Workflow
|
| 54 |
+
|
| 55 |
+
```
|
| 56 |
+
text -> detection (quotation marks + corpus match, then corpus scan for unannounced quotes) -> BM25 retrieval
|
| 57 |
+
-> span-local alignment (phonetic skeleton, sliding window, LCS, dynamic gap)
|
| 58 |
+
-> exact: verified | altered Quran passage: source correction | uncertain: human review | nothing similar: no source
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
| Decision | Colour | Meaning |
|
| 62 |
+
|---|---|---|
|
| 63 |
+
| موثّق (verified) | green | Equals its source after ignoring spelling and diacritic conventions. |
|
| 64 |
+
| غير مطابق (mismatch) | red | Differs from the source (Quran: the exact source text is offered) or no source resembles it. |
|
| 65 |
+
| يحتاج مراجعة بشرية (human review) | amber | Evidence is insufficient or ambiguous. No correction is produced. |
|
| 66 |
+
|
| 67 |
+
## Benchmark alignment
|
| 68 |
+
|
| 69 |
+
`benchmark.py` converts a pipeline result to the structure of the IslamicEval 2025 subtasks, mirroring the bundled
|
| 70 |
+
development subset (`data/islamiceval_dev_subset.jsonl`):
|
| 71 |
+
|
| 72 |
+
| Subtask | Field | Values |
|
| 73 |
+
|---|---|---|
|
| 74 |
+
| 1A detection | `Span_Start`, `Span_End`, `Span_Type` | character offsets (end exclusive, quotation marks excluded); `Ayah` / `Hadith` |
|
| 75 |
+
| 1B verification | `Label` | `Correct` / `Incorrect` (anything not verified is reported as `Incorrect`) |
|
| 76 |
+
| 1C correction | `Correction` | exact source text for incorrect spans, or `خطأ` when no authentic source exists |
|
| 77 |
+
|
| 78 |
+
TSV columns: `Response_ID, Span_Start, Span_End, Span_Type, Label, Correction`. **Check the column names against the
|
| 79 |
+
organisers' current submission page before an official submission**; the official sites could not be fetched automatically
|
| 80 |
+
while preparing this version, so the exact official column names are not verified here.
|
| 81 |
+
|
| 82 |
+
## Run and deploy
|
| 83 |
+
|
| 84 |
+
```bash
|
| 85 |
+
python index_builder.py # only after changing data/
|
| 86 |
+
python build_static_space.py # writes ./static_space/
|
| 87 |
+
```
|
| 88 |
+
|
| 89 |
+
Upload the **contents** of `static_space/` to a Hugging Face Space with SDK = *Static* (free). Local Gradio version:
|
| 90 |
+
`pip install -r requirements.txt && OPENAI_API_KEY=... python app.py`. Tests: `python -m unittest discover -s tests`.
|
| 91 |
+
|
| 92 |
+
## API key (important)
|
| 93 |
+
|
| 94 |
+
A Static Space has no server, so a key placed in `web/config.js` is **readable by every visitor**. This build ships the
|
| 95 |
+
ChatGPT key in that file for the live demo. Keep a small monthly spend limit on the key in the OpenAI dashboard, revoke it
|
| 96 |
+
after the evaluation, and never commit it to a public repository (GitHub scans for and revokes exposed OpenAI keys).
|
| 97 |
+
A safer production design is a small proxy (for example a Cloudflare Worker) that holds the key as a secret.
|
| 98 |
+
|
| 99 |
+
## Project structure
|
| 100 |
+
|
| 101 |
+
| Path | Role |
|
| 102 |
+
|---|---|
|
| 103 |
+
| `web/` | Front end: `index.html`, `styles.css` (glassmorphism, RTL), `app.js` (UI), `worker.js` (Pyodide worker), `cache.js`, `camelbert.js`, `config.js` |
|
| 104 |
+
| `normalization.py`, `idgham.py` | Phonetic-aware Arabic normalisation; mushaf idgham rendering for corrections |
|
| 105 |
+
| `index_builder.py`, `retrieval.py` | Pre-built BM25 / n-gram indexes and the lazy-loading retriever |
|
| 106 |
+
| `alignment.py`, `similarity.py` | Word alignment and similarity indicators |
|
| 107 |
+
| `detector.py`, `scanner.py` | Quotation detection (corpus-first; trigger phrases optional) and corpus scanning |
|
| 108 |
+
| `verifier.py` | Verification, source-backed correction, decisions |
|
| 109 |
+
| `benchmark.py` | IslamicEval-style JSON / TSV export |
|
| 110 |
+
| `llm_client.py`, `app.py`, `ui.py` | LLM client, Gradio app and Python entry points used by the page, HTML rendering |
|
| 111 |
+
| `evaluate.py`, `tests/`, `research/` | Prototype evaluation, tests, original notebook and optional training code |
|
| 112 |
+
|
| 113 |
+
## Evaluation (development subset)
|
| 114 |
+
|
| 115 |
+
Computed with `python evaluate.py` on the bundled IslamicEval 2025 development subset after decoupling detection from
|
| 116 |
+
trigger phrases. These numbers are **not** the earlier research baseline (1A macro F1 0.9091 with CAMeLBERT-MSA; 1B 93.12%; 1C 70.39%).
|
| 117 |
+
|
| 118 |
+
| Component | Result |
|
| 119 |
+
|---|---|
|
| 120 |
+
| 1A detection, character-level macro F1, hybrid detector | 0.8247 |
|
| 121 |
+
| 1B verdict accuracy on gold spans (247 spans; always-"Correct" baseline 59.5%) | 93.12% |
|
| 122 |
+
| 1C correction accuracy (179 spans; always-"no source" baseline 62.0%) | 74.30% |
|
| 123 |
+
|
| 124 |
+
## Limitations
|
| 125 |
+
|
| 126 |
+
* Unannounced-quotation detection is corpus matching, not understanding; text far from the bundled corpora is missed.
|
| 127 |
+
* The tool aids textual checking; it is not a fatwa and does not replace specialist review.
|
| 128 |
+
* The first visit downloads the Pyodide runtime and about 16 MB of data; later visits use the browser cache.
|
| 129 |
+
|
| 130 |
+
## Competition requirements
|
| 131 |
+
|
| 132 |
+
See [REFERENCES.md](REFERENCES.md) for the sources used. Fill in the rules and deadlines from the organisers' documents
|
| 133 |
+
(<https://islamicaich.org/>) in your submission; they could not be read automatically while preparing this version.
|
| 134 |
+
|
| 135 |
+
License: MIT.
|
README_AR.md
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# التحقق من هلوسة القرآن والحديث وتصحيحها
|
| 2 |
+
|
| 3 |
+
النسخة الإنجليزية: [README.md](README.md)
|
| 4 |
+
|
| 5 |
+
يكتشف الاقتباسات القرآنية والحديثية في أي نص (مثل إجابات النماذج اللغوية)، ويتحقق من كل اقتباس بمقارنته بالمصادر، ويعرض
|
| 6 |
+
الدليل، ويقترح تصحيحًا **مأخوذًا حرفيًا من نص المصدر**. وإذا لم تكفِ الأدلة فإنه يصرّح بذلك ويحيل الحالة إلى المراجعة البشرية.
|
| 7 |
+
واجهة المنتج عربية بالكامل.
|
| 8 |
+
|
| 9 |
+
## العرض المباشر
|
| 10 |
+
|
| 11 |
+
**[LIVE_DEPLOYMENT_URL]** <!-- استبدل هذا الرابط برابط Hugging Face Space بعد النشر -->
|
| 12 |
+
|
| 13 |
+
## المزايا
|
| 14 |
+
|
| 15 |
+
* **تحقّق مباشر:** الصق أي فقرة، فيكتشف النظام كل آية وحديث، ويقارنها كلمةً بكلمة، ويقترح نص المصدر تصحيحًا.
|
| 16 |
+
* **اسأل ثم تحقّق:** اكتب سؤالًا فيجيب عميل ChatGPT (OpenAI) المهيّأ مسبقًا، ثم يتحقق المحرك من الإجابة ويعيد نسخة مصحّحة.
|
| 17 |
+
لا توجد قائمة لاختيار النموذج في هذه الصفحة.
|
| 18 |
+
* **دون الحاجة إلى عبارات تمهيدية:** يتعرّف النظام على الاقتباس بمطابقته مع مراجع القرآن والحديث، سواء وُضع بين علامات
|
| 19 |
+
تنصيص أو لم يوضع، وسواء سبقته عبارة مثل «قال الله تعالى» أو «قال رسول الله ﷺ» أو لم تسبقه. هذه العبارات احتياط ضعيف فقط
|
| 20 |
+
للاقتباسات المحرَّفة التي لا يستطيع المرجع تأكيدها.
|
| 21 |
+
* **قائمة نموذج الاستخراج (بالإنجليزية):** *Standard · Rules + Corpus* أو *CAMeLBERT-MSA · Fine-tuned* (التفاصيل أدناه).
|
| 22 |
+
* **تصدير بصيغة المسابقة:** JSON وTSV بهيكل المهام 1A / 1B / 1C (انظر «مواءمة المعايير»).
|
| 23 |
+
* **نسخ النص المصحّح:** زر يعمل فعليًا مع رسالة «تم نسخ النص بنجاح»، وبديل احتياطي داخل الإطارات (iframe).
|
| 24 |
+
* **تحميل سريع ومُعلِم:** يبدأ تنزيل فهرس القرآن قبل تشغيل السكربتات، ويُحمَّل فهرس الحديث في الخلفية، وتُخزَّن الملفات في
|
| 25 |
+
ذاكرة المتصفح (Cache API)، وتُخزَّن النتائج وأمثلة العرض بصيغة JSON في `localStorage`، وتعرض بطاقات هيكلية متحركة وشريط
|
| 26 |
+
تقدّم من ثلاث خطوات ما يجري.
|
| 27 |
+
|
| 28 |
+
## دعم CAMeLBERT-MSA
|
| 29 |
+
|
| 30 |
+
يوفّر `web/camelbert.js` دالة `detectSpans` تعيد وسوم BIO (`B-Ayah` و`I-Ayah` و`B-Hadith` و`I-Hadith` و`O`) والمقاطع:
|
| 31 |
+
|
| 32 |
+
* **الوضع الحي:** اضبط `HF_MODEL_ID` (و`HF_TOKEN` للنموذج الخاص) في `web/config.js`، فتستدعي الصفحة واجهة Hugging Face
|
| 33 |
+
Inference لنموذج تصنيف رموز مضبوط. ثم يتحقق خط المعالجة المحلي من المقاطع ويصحّحها.
|
| 34 |
+
* **وضع المحاكاة (الافتراضي):** عند عدم ضبط معرّف النموذج **لا تُحمَّل أي أوزان نموذج**. تأتي المقاطع من المستخرج القياسي
|
| 35 |
+
وتُعرض بوسوم BIO وقيم ثقة شبه عشوائية ثابتة، وتُظهر الواجهة بوضوح أنها *Simulation*. استخدمه لعرض سير العمل لا للإبلاغ عن دقة النموذج.
|
| 36 |
+
|
| 37 |
+
## سير العمل
|
| 38 |
+
|
| 39 |
+
```
|
| 40 |
+
نص -> استخراج (علامات التنصيص + مطابقة المرجع، ثم مسح المرجع للاقتباسات غير المعلَنة) -> استرجاع BM25
|
| 41 |
+
-> محاذاة موضعية (الهيكل الصوتي، نافذة منزلقة، LCS، فجوة ديناميكية)
|
| 42 |
+
-> مطابق: موثّق | آية محرّفة: تصحيح من المصدر | غير مؤكد: مراجعة بشرية | لا شبيه: لا يوجد مصدر
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
## مواءمة المعايير
|
| 46 |
+
|
| 47 |
+
يحوّل `benchmark.py` نتيجة المعالجة إلى هيكل مهام IslamicEval 2025 مطابقًا لعيّنة التطوير المضمّنة
|
| 48 |
+
(`data/islamiceval_dev_subset.jsonl`):
|
| 49 |
+
|
| 50 |
+
| المهمة | الحقل | القيم |
|
| 51 |
+
|---|---|---|
|
| 52 |
+
| 1A الاستخراج | `Span_Start` و`Span_End` و`Span_Type` | مواضع الأحرف (النهاية حصرية، دون علامات التنصيص)؛ `Ayah` / `Hadith` |
|
| 53 |
+
| 1B التحقق | `Label` | `Correct` / `Incorrect` (كل ما لم يُوثَّق يُبلَّغ عنه `Incorrect`) |
|
| 54 |
+
| 1C التصحيح | `Correction` | نص المصدر الحرفي للمقاطع الخاطئة، أو `خطأ` إن لم يوجد مصدر موثوق |
|
| 55 |
+
|
| 56 |
+
أعمدة TSV: `Response_ID, Span_Start, Span_End, Span_Type, Label, Correction`. **راجع أسماء الأعمدة في صفحة التسليم الرسمية
|
| 57 |
+
للمنظّمين قبل أي تسليم رسمي**؛ فلم يمكن جلب المواقع الرسمية آليًا أثناء إعداد هذه النسخة، لذا لم تُتحقق الأسماء الرسمية الدقيقة هنا.
|
| 58 |
+
|
| 59 |
+
## التشغيل والنشر
|
| 60 |
+
|
| 61 |
+
```bash
|
| 62 |
+
python index_builder.py # فقط بعد تعديل data/
|
| 63 |
+
python build_static_space.py # ينشئ ./static_space/
|
| 64 |
+
```
|
| 65 |
+
|
| 66 |
+
ارفع **محتويات** `static_space/` إلى Hugging Face Space من نوع *Static* (مجاني). النسخة المحلية بواجهة Gradio:
|
| 67 |
+
`pip install -r requirements.txt && OPENAI_API_KEY=... python app.py`. الاختبارات: `python -m unittest discover -s tests`.
|
| 68 |
+
|
| 69 |
+
## مفتاح الواجهة البرمجية (مهم)
|
| 70 |
+
|
| 71 |
+
لا يملك Static Space خادمًا، لذا فإن أي مفتاح في `web/config.js` **يستطيع كل زائر قراءته**. تضع هذه النسخة مفتاح ChatGPT في
|
| 72 |
+
هذا الملف لغرض العرض المباشر. حدّد سقف إنفاق شهريًا صغيرًا للمفتاح من لوحة OpenAI، وألغِه بعد التقييم، ولا ترفعه إلى مستودع
|
| 73 |
+
عام (يفحص GitHub المفاتيح المكشوفة ويلغي مفاتيح OpenAI). التصميم الأكثر أمانًا للإنتاج وسيط صغير (مثل Cloudflare Worker) يحتفظ بالمفتاح سرًّا.
|
| 74 |
+
|
| 75 |
+
## هيكل المشروع
|
| 76 |
+
|
| 77 |
+
| المسار | الدور |
|
| 78 |
+
|---|---|
|
| 79 |
+
| `web/` | الواجهة: `index.html` و`styles.css` (زجاجية، RTL) و`app.js` و`worker.js` و`cache.js` و`camelbert.js` و`config.js` |
|
| 80 |
+
| `normalization.py` و`idgham.py` | تطبيع صوتي للعربية؛ إظهار الإدغام في التصحيحات |
|
| 81 |
+
| `index_builder.py` و`retrieval.py` | فهارس BM25 / n-gram مبنية مسبقًا والمسترجِع الكسول |
|
| 82 |
+
| `alignment.py` و`similarity.py` | محاذاة الكلمات ومؤشرات التشابه |
|
| 83 |
+
| `detector.py` و`scanner.py` | الاستخراج (المرجع أولًا؛ العبارات التمهيدية اختيارية) ومسح المرجع |
|
| 84 |
+
| `verifier.py` | التحقق والتصحيح المستند إلى المصدر والقرارات |
|
| 85 |
+
| `benchmark.py` | تصدير JSON / TSV بهيكل IslamicEval |
|
| 86 |
+
| `llm_client.py` و`app.py` و`ui.py` | عميل النموذج وتطبيق Gradio ونقاط الدخول لبايثون وعرض HTML |
|
| 87 |
+
| `evaluate.py` و`tests/` و`research/` | التقييم الأولي والاختبارات والدفتر الأصلي |
|
| 88 |
+
|
| 89 |
+
## التقييم (عيّنة التطوير)
|
| 90 |
+
|
| 91 |
+
حُسب بـ `python evaluate.py` على عيّنة تطوير IslamicEval 2025 المضمّنة بعد فصل الاستخراج عن العبارات التمهيدية. هذه الأرقام
|
| 92 |
+
**ليست** خط الأساس البحثي السابق (1A: 0.9091 بـ CAMeLBERT-MSA؛ 1B: 93.12%؛ 1C: 70.39%).
|
| 93 |
+
|
| 94 |
+
| المكوّن | النتيجة |
|
| 95 |
+
|---|---|
|
| 96 |
+
| 1A الاستخراج، F1 على مستوى الأحرف، المستخرج الهجين | 0.8247 |
|
| 97 |
+
| 1B دقة الحكم على المقاطع المرجعية (247 مقطعًا؛ خط الأساس «صحيح دائمًا» 59.5%) | 93.12% |
|
| 98 |
+
| 1C دقة التصحيح (179 مقطعًا؛ خط الأساس «لا مصدر دائمًا» 62.0%) | 74.30% |
|
| 99 |
+
|
| 100 |
+
## القيود
|
| 101 |
+
|
| 102 |
+
* اكتشاف الاقتباسات غير المعلَنة مطابقة للمرجع لا فهم؛ يفوته النص البعيد عن المراجع المضمّنة.
|
| 103 |
+
* الأداة مساعدة للتدقيق النصي وليست فتوى ولا بديلًا عن المراجعة المتخصصة.
|
| 104 |
+
* تنزّل الزيارة الأولى بيئة Pyodide وحوالي 16 ميغابايت من البيانات؛ وتستفيد الزيارات التالية من ذاكرة المتصفح.
|
| 105 |
+
|
| 106 |
+
## متطلبات المسابقة
|
| 107 |
+
|
| 108 |
+
انظر [REFERENCES_AR.md](REFERENCES_AR.md) للمصادر المستخدمة. أدرِج شروط المسابقة ومواعيدها من وثائق المنظّمين
|
| 109 |
+
(<https://islamicaich.org/>) في تسليمك؛ فلم يمكن قراءتها آليًا أثناء إعداد هذه النسخة.
|
| 110 |
+
|
| 111 |
+
الرخصة: MIT.
|
REFERENCES.md
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# References & Data Provenance
|
| 2 |
+
|
| 3 |
+
Arabic version: [REFERENCES_AR.md](REFERENCES_AR.md)
|
| 4 |
+
|
| 5 |
+
This file documents what the project actually uses. Where the supplied materials do not record a fact (for example the
|
| 6 |
+
upstream origin or licence of a data file), that is stated instead of guessed.
|
| 7 |
+
|
| 8 |
+
## Project Specification
|
| 9 |
+
|
| 10 |
+
`IslamicAIChallengePresentation.pdf`: the project proposal for the challenge *تحدي الذكاء الاصطناعي في خدمة المحتوى
|
| 11 |
+
الإسلامي 2026* (12 slides, Arabic). It defines the workflow (detect, retrieve, verify with evidence, correct or refer to
|
| 12 |
+
human review), the safety principles (source first, traceable evidence, abstain when unsure) and the expected deliverables.
|
| 13 |
+
It also reports the previous IslamicEval 2025 research baselines (1A macro F1 0.9091 with CAMeLBERT-MSA; 1B accuracy
|
| 14 |
+
93.12%; 1C accuracy 70.39%). Those are historical figures, not results of this application.
|
| 15 |
+
|
| 16 |
+
## IslamicEval 2025
|
| 17 |
+
|
| 18 |
+
* **Official source:** Mubarak, H. et al. (2025). *IslamicEval 2025: The First Shared Task of Capturing LLMs Hallucination in
|
| 19 |
+
Islamic Content.* Proceedings of The Third Arabic Natural Language Processing Conference: Shared Tasks, pp. 480–493,
|
| 20 |
+
Suzhou, China. ACL Anthology ID `2025.arabicnlp-sharedtasks.67`, DOI `10.18653/v1/2025.arabicnlp-sharedtasks.67`.
|
| 21 |
+
The task has two subtasks; this project uses only Subtask 1 (hallucination detection and correction of quoted
|
| 22 |
+
ayahs and Hadith). Subtask 2 (question answering) is not used.
|
| 23 |
+
* **Subtasks represented:** 1A (quotation span detection), 1B (verification: Correct / Incorrect), 1C (correction of
|
| 24 |
+
incorrect quotations; `خطأ` marks "no source").
|
| 25 |
+
* **Files actually used:** `data/islamiceval_dev_subset.jsonl`, 150 real development responses (50 each for 1A, 1B, 1C),
|
| 26 |
+
with 194 / 247 / 179 annotated spans. It was taken from the project's unified dataset file
|
| 27 |
+
(`islamic_unified_dataset.jsonl`), which the research notebook builds from the development files
|
| 28 |
+
`dev_SubtaskA/B/C` (`.xml` + `.tsv`). Only the records with `source` = `real_A`, `real_B`, `real_C` were kept. The
|
| 29 |
+
1,500 synthetic records of the unified file are not in this evaluation subset: they are training data generated by the
|
| 30 |
+
notebook from the Quran / Hadith corpora, not part of IslamicEval.
|
| 31 |
+
* **How it is used:** `evaluate.py` reads the subset for the prototype evaluation; the optional training script reads the unified file. The running application reads neither.
|
| 32 |
+
* **Not included:** the rest of the IslamicEval 2025 data (including training and test data).
|
| 33 |
+
|
| 34 |
+
## Dataset Provenance
|
| 35 |
+
|
| 36 |
+
| File | Content | What is verifiable |
|
| 37 |
+
|---|---|---|
|
| 38 |
+
| `data/quran.json` | 6,236 ayahs, 114 surahs; fields `surah_id`, `surah_name`, `ayah_id`, `ayah_text` (with diacritics) | Supplied by the project team as `quranic_verses.json`. Used unchanged. |
|
| 39 |
+
| `data/hadith.json` | 34,994 records; fields `hadithID`, `BookID` (1–6), `title`, `hadithTxt`, `Matn` (present in 31,811 records) | Supplied by the project team as `six_hadith_books.json`; only whitespace outside strings was removed (JSON re-serialised compactly), content unchanged. |
|
| 40 |
+
| `data/islamiceval_dev_subset.jsonl` | See above | Derived from the project's unified dataset file. |
|
| 41 |
+
| `data/islamic_unified_dataset.jsonl` | 1,650 records: 150 real IslamicEval development responses + 1,500 synthetic ones generated by the research notebook | Supplied by the project team; used only by the optional `research/train_detector.py`. |
|
| 42 |
+
| `index/quran.idx.gz`, `index/hadith.idx.gz` | Pre-tokenised search indexes derived from `data/quran.json` and `data/hadith.json` by `index_builder.py` | Generated; rebuild with `python index_builder.py`. |
|
| 43 |
+
|
| 44 |
+
The research notebook uses `quranic_verses.json` and `six_hadith_books.json` as the reference database of its Subtask 1C
|
| 45 |
+
scoring routine (copied there from the organisers' `scoring.py`). The supplied materials do **not** record where these
|
| 46 |
+
two files were originally obtained or under which licence. The team should record the upstream source and licence here
|
| 47 |
+
before redistributing the repository publicly.
|
| 48 |
+
|
| 49 |
+
## Research References
|
| 50 |
+
|
| 51 |
+
* Mubarak et al. (2025), cited above. This is the only paper the project relies on directly (task definition and
|
| 52 |
+
subtask structure).
|
| 53 |
+
|
| 54 |
+
## Models
|
| 55 |
+
|
| 56 |
+
* **The application uses no machine-learning model and loads no model weights.** Detection is rule-based plus a corpus scan; retrieval
|
| 57 |
+
is BM25 over pre-built indexes; verification is word / character alignment.
|
| 58 |
+
* `research/train_detector.py` is optional research code (fine-tuning `aubmindlab/bert-base-arabertv2` or CAMeLBERT-MSA for Subtask 1A).
|
| 59 |
+
It is not used by the application and no trained weights exist in this repository.
|
| 60 |
+
* **Mode B** calls a third-party language model chosen by the user (Google Gemini `generateContent`, OpenAI chat completions, or the
|
| 61 |
+
Hugging Face router) with the user's own key. Default model names (`gemini-2.5-flash`, `gpt-4o-mini`, `Qwen/Qwen2.5-72B-Instruct`)
|
| 62 |
+
are editable placeholders; only the Gemini name was checked against the provider's documentation. Provider request formats were
|
| 63 |
+
tested against a local mock server, not against the live services.
|
| 64 |
+
* The 1A baseline of 0.9091 in the proposal comes from earlier work with CAMeLBERT-MSA, not from this code.
|
| 65 |
+
|
| 66 |
+
## Libraries
|
| 67 |
+
|
| 68 |
+
| Library | Purpose | Source | Version | Licence |
|
| 69 |
+
|---|---|---|---|---|
|
| 70 |
+
| Python standard library (`difflib`, `re`, `unicodedata`, `json`, …) | Matching, normalisation, data loading | https://www.python.org | Developed and tested with 3.12.3 | Python Software Foundation License |
|
| 71 |
+
| Gradio | Web interface (local / server runs only) | https://github.com/gradio-app/gradio | `>=5.0,<6`; not installed in the authoring environment (see below) | Apache-2.0 (as published by the project) |
|
| 72 |
+
| RapidFuzz | Optional faster Levenshtein similarity (pure-Python fallback exists) | https://github.com/rapidfuzz/RapidFuzz | `>=3.0`; not installed in the authoring environment | MIT (as published by the project) |
|
| 73 |
+
| NumPy | Character-level F1 in `evaluate.py` | https://numpy.org | 2.4.4 (tested) | BSD-3-Clause (as published by the project) |
|
| 74 |
+
| Pyodide | Runs the Python code inside a browser Web Worker for the Static Space demo (loaded from the jsDelivr CDN) | https://pyodide.org | 0.26.4 | MPL-2.0 (as published by the project) |
|
| 75 |
+
|
| 76 |
+
Licences above are those declared by the respective projects; confirm them at the source before relying on them.
|
| 77 |
+
|
| 78 |
+
## Deployment
|
| 79 |
+
|
| 80 |
+
* GitHub for the source code.
|
| 81 |
+
* Hugging Face **Static** Space for the public demo: `index.html` starts a Web Worker that loads Pyodide 0.26.4 from the jsDelivr CDN
|
| 82 |
+
and runs the project's Python modules in the visitor's browser (`build_static_space.py` assembles the folder). Gradio is used only
|
| 83 |
+
for local runs, because Gradio and Docker Spaces require a paid Hugging Face plan.
|
| 84 |
+
* No deployment has been performed from this repository yet, so no public URL exists in the documentation.
|
| 85 |
+
|
| 86 |
+
## Licensing & Attribution
|
| 87 |
+
|
| 88 |
+
* Source code: MIT (see `LICENSE`), applied to the code only.
|
| 89 |
+
* Data files in `data/` are third-party resources and are not covered by that licence. Their licences are not recorded in
|
| 90 |
+
the supplied materials (see Dataset Provenance).
|
| 91 |
+
* IslamicEval 2025 data belongs to the shared-task organisers; follow their terms and cite the overview paper.
|
| 92 |
+
|
| 93 |
+
## Reproducibility
|
| 94 |
+
|
| 95 |
+
* **Python:** 3.10 or newer; developed and tested with 3.12.3.
|
| 96 |
+
* **Requirements:** `requirements.txt`. The core logic was tested with the Python standard library and NumPy only. The
|
| 97 |
+
Gradio interface, RapidFuzz and Pyodide could not be installed or downloaded in the authoring environment (no network
|
| 98 |
+
access). The rendering functions were tested natively, and the static page was exercised in a browser with a stubbed
|
| 99 |
+
Pyodide; the real Gradio server and the real Pyodide runtime were not run there.
|
| 100 |
+
* **Data:** the three files in `data/`, as described above.
|
| 101 |
+
* **Preprocessing:** Arabic normalisation in `retrieval.py` (diacritic and tatweel removal, alef unification, punctuation
|
| 102 |
+
stripping, and for matching ta marbuta / alef maqsura folding); the index is built in memory at start-up.
|
| 103 |
+
* **Execution:** `python index_builder.py` (only after changing `data/`); `python app.py`; `python evaluate.py`; `python -m unittest discover -s tests`; `python build_static_space.py` for the browser demo.
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
## Competition and benchmark links (not verified automatically)
|
| 107 |
+
|
| 108 |
+
The following were provided with the task and are cited as references. Automated access to the GitHub repository was blocked
|
| 109 |
+
(robots rules) and the organiser pages and PDF were not read while preparing this version, so no rule, deadline or file format
|
| 110 |
+
is quoted from them here:
|
| 111 |
+
|
| 112 |
+
* IslamicEval 2025 task site: <https://sites.google.com/view/islamiceval-2025/home>
|
| 113 |
+
* Subtask 1 code repository: <https://github.com/qcri/IslamicEval-2025-Subtask-1>
|
| 114 |
+
* Challenge site and timeline: <https://islamicaich.org/>
|
| 115 |
+
* Hackathon rules (PDF): <https://islamicaich.org/files/HackathonFile/da2OrbRMEsNrIQaoq8z6jEv26opKXdQmq1i0XfXo.pdf>
|
| 116 |
+
|
| 117 |
+
## Model references
|
| 118 |
+
|
| 119 |
+
* CAMeLBERT-MSA (CAMeL Lab, `CAMeL-Lab/bert-base-arabic-camelbert-msa`) is the base model for the optional fine-tuned
|
| 120 |
+
token classifier described in the README. The browser page runs it as a simulation unless `HF_MODEL_ID` is configured.
|
| 121 |
+
* "Ask then Verify" uses the OpenAI Chat Completions API (`gpt-4o-mini` by default, set in `web/config.js`).
|
REFERENCES_AR.md
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# المراجع ومصادر البيانات
|
| 2 |
+
|
| 3 |
+
النسخة الإنجليزية الكاملة: [REFERENCES.md](REFERENCES.md)
|
| 4 |
+
|
| 5 |
+
يوثّق هذا الملف ما يستخدمه المشروع فعلًا. وحيث لا تسجّل المواد المقدَّمة معلومة ما (كالأصل أو الرخصة لملف بيانات) يُذكر ذلك صراحة بدل التخمين.
|
| 6 |
+
|
| 7 |
+
## مواصفات المشروع
|
| 8 |
+
|
| 9 |
+
`IslamicAIChallengePresentation.pdf`: عرض المشروع لتحدي *الذكاء الاصطناعي في خدمة المحتوى الإسلامي 2026*. يحدد سير العمل (استخراج،
|
| 10 |
+
استرجاع، تحقق بالدليل، تصحيح أو إحالة للمراجعة البشرية) ومبادئ السلامة (المصدر أولًا، دليل قابل للتتبّع، الامتناع عند عدم اليقين).
|
| 11 |
+
ويذكر أيضًا خطوط الأساس البحثية السابقة لـ IslamicEval 2025 (1A: 0.9091؛ 1B: 93.12%؛ 1C: 70.39%) وهي أرقام تاريخية لا نتائج هذا التطبيق.
|
| 12 |
+
|
| 13 |
+
## IslamicEval 2025
|
| 14 |
+
|
| 15 |
+
* **المصدر الرسمي:** Mubarak وآخرون (2025). *IslamicEval 2025: The First Shared Task of Capturing LLMs Hallucination in Islamic Content.*
|
| 16 |
+
أعمال المؤتمر الثالث لمعالجة اللغة العربية: المهام المشتركة، ص 480–493، سوتشو، الصين. المعرّف `2025.arabicnlp-sharedtasks.67`،
|
| 17 |
+
DOI: `10.18653/v1/2025.arabicnlp-sharedtasks.67`. يستخدم المشروع المهمة الأولى فقط (اكتشاف الهلوسة وتصحيحها في الآيات والأحاديث المقتبسة).
|
| 18 |
+
* **المهام الفرعية:** 1A (استخراج مواضع الاقتباس)، 1B (التحقق: Correct / Incorrect)، 1C (تصحيح الاقتباسات الخاطئة؛ `خطأ` تعني «لا مصدر»).
|
| 19 |
+
* **الملف المستخدم فعلًا:** `data/islamiceval_dev_subset.jsonl` (150 إجابة تطوير حقيقية، 50 لكل مهمة)، مأخوذ من الملف الموحّد للمشروع.
|
| 20 |
+
ولا يستخدمه التطبيق أثناء التشغيل، بل `evaluate.py` فقط.
|
| 21 |
+
* **غير مضمَّن:** باقي بيانات IslamicEval 2025 (التدريب والاختبار).
|
| 22 |
+
|
| 23 |
+
## روابط المسابقة والمعايير (لم يُتحقق منها آليًا)
|
| 24 |
+
|
| 25 |
+
وردت هذه الروابط مع المهمة وتُذكر مراجعَ. مُنع الوصول الآلي إلى مستودع GitHub، ولم تُقرأ صفحات المنظّمين وملف PDF أثناء إعداد هذه
|
| 26 |
+
النسخة، لذلك لا يُنقل هنا أي شرط أو موعد أو صيغة ملف منها:
|
| 27 |
+
|
| 28 |
+
* موقع المهمة: <https://sites.google.com/view/islamiceval-2025/home>
|
| 29 |
+
* مستودع شيفرة المهمة الأولى: <https://github.com/qcri/IslamicEval-2025-Subtask-1>
|
| 30 |
+
* موقع التحدي والجدول الزمني: <https://islamicaich.org/>
|
| 31 |
+
* لائحة الهاكاثون (PDF): <https://islamicaich.org/files/HackathonFile/da2OrbRMEsNrIQaoq8z6jEv26opKXdQmq1i0XfXo.pdf>
|
| 32 |
+
|
| 33 |
+
## مراجع النماذج
|
| 34 |
+
|
| 35 |
+
* CAMeLBERT-MSA (مختبر CAMeL، `CAMeL-Lab/bert-base-arabic-camelbert-msa`) هو النموذج الأساس لمصنِّف الرموز المضبوط الاختياري المذكور
|
| 36 |
+
في README. وتشغّله الصفحة محاكاةً ما لم يُضبط `HF_MODEL_ID`.
|
| 37 |
+
* يستخدم «اسأل ثم تحقّق» واجهة OpenAI Chat Completions (النموذج `gpt-4o-mini` افتراضيًا، ويُضبط في `web/config.js`).
|
alignment.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Word-level sequence alignment between a quotation and a source text.
|
| 2 |
+
|
| 3 |
+
* Phonetic skeletons (``normalization.aligned_words``) make Uthmani / modern spelling and diacritics irrelevant for
|
| 4 |
+
matching, while every substituted, missing, extra or re-ordered word still shows up as an edit operation.
|
| 5 |
+
* A dynamic sliding window locates the best-matching region inside long sources (Hadith with chains of narrators).
|
| 6 |
+
* The gap threshold scales with quotation length: a short quotation tolerates no stray words, a long one a few.
|
| 7 |
+
* Diacritic conflicts are reported separately as notes, because spelling of harakat differs legitimately between prints.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from difflib import SequenceMatcher
|
| 12 |
+
from math import ceil
|
| 13 |
+
from typing import Container, Dict, List, Optional, Sequence, Tuple
|
| 14 |
+
|
| 15 |
+
from idgham import apply_idgham
|
| 16 |
+
from normalization import aligned_words, vowel_signature
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def lcs_length(a: Sequence[str], b: Sequence[str]) -> int:
|
| 20 |
+
m, n = len(a), len(b)
|
| 21 |
+
if m == 0 or n == 0:
|
| 22 |
+
return 0
|
| 23 |
+
if m < n:
|
| 24 |
+
a, b, m, n = b, a, n, m
|
| 25 |
+
prev = [0] * (n + 1)
|
| 26 |
+
for i in range(m):
|
| 27 |
+
curr = [0] * (n + 1)
|
| 28 |
+
for j in range(n):
|
| 29 |
+
curr[j + 1] = prev[j] + 1 if a[i] == b[j] else max(curr[j], prev[j + 1])
|
| 30 |
+
prev = curr
|
| 31 |
+
return prev[n]
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def allowed_gap(n_tokens: int) -> int:
|
| 35 |
+
"""Dynamic gap threshold: how many differing words still count as 'the same passage, altered'."""
|
| 36 |
+
return 0 if n_tokens <= 3 else max(1, ceil(0.2 * n_tokens))
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def find_window(quote: List[str], source: List[str], gap: int) -> Tuple[int, int]:
|
| 40 |
+
"""Best source region for the quotation: windows of ``len(quote)+gap`` ranked by shared words, refined by LCS."""
|
| 41 |
+
n, window = len(quote), len(quote) + gap
|
| 42 |
+
if len(source) <= window + 2 * gap + 8:
|
| 43 |
+
return 0, len(source)
|
| 44 |
+
quote_set = set(quote)
|
| 45 |
+
prefix = [0]
|
| 46 |
+
for word in source:
|
| 47 |
+
prefix.append(prefix[-1] + (word in quote_set))
|
| 48 |
+
starts = sorted(range(len(source) - window + 1), key=lambda i: prefix[i + window] - prefix[i], reverse=True)[:8]
|
| 49 |
+
best_start = max(starts, key=lambda i: lcs_length(quote, source[i : i + window]))
|
| 50 |
+
return max(0, best_start - gap), min(len(source), best_start + window + gap)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def contains_sequence(source: Sequence[str], quote: Sequence[str]) -> bool:
|
| 54 |
+
"""True if ``quote`` appears as a contiguous run of whole words in ``source``."""
|
| 55 |
+
n = len(quote)
|
| 56 |
+
return n > 0 and any(source[i : i + n] == list(quote) for i in range(len(source) - n + 1))
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def locate_region(q_norm: List[str], s_norm: List[str], gap: int, slack: Optional[int] = None):
|
| 60 |
+
"""Region of the source that corresponds to the quotation (matched part plus slack for edge differences).
|
| 61 |
+
|
| 62 |
+
Returns ``(lo, hi, matcher)`` where the matcher compares the quotation with ``s_norm[lo:hi]``."""
|
| 63 |
+
slack = gap if slack is None else slack
|
| 64 |
+
lo, hi = find_window(q_norm, s_norm, gap)
|
| 65 |
+
matcher = SequenceMatcher(None, q_norm, s_norm[lo:hi], autojunk=False)
|
| 66 |
+
blocks = [b for b in matcher.get_matching_blocks() if b.size]
|
| 67 |
+
if blocks:
|
| 68 |
+
first, last = blocks[0], blocks[-1]
|
| 69 |
+
base = lo
|
| 70 |
+
lo = base + max(0, first.b - first.a - slack)
|
| 71 |
+
hi = min(hi, base + last.b + last.size + (len(q_norm) - last.a - last.size) + slack)
|
| 72 |
+
matcher = SequenceMatcher(None, q_norm, s_norm[lo:hi], autojunk=False)
|
| 73 |
+
return lo, hi, matcher
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def best_region(quote: str, source: str) -> str:
|
| 77 |
+
"""The part of ``source`` that a (possibly partial) quotation refers to, as original words.
|
| 78 |
+
|
| 79 |
+
Scoring a snippet against the whole verse / Hadith would punish it for being shorter than its source; comparing it
|
| 80 |
+
with this region keeps the score about what was actually quoted."""
|
| 81 |
+
q_pairs, s_pairs = aligned_words(quote), aligned_words(source)
|
| 82 |
+
if not q_pairs or len(s_pairs) <= len(q_pairs) + allowed_gap(len(q_pairs)) + 2:
|
| 83 |
+
return source
|
| 84 |
+
q_norm, s_norm = [p[1] for p in q_pairs], [p[1] for p in s_pairs]
|
| 85 |
+
lo, hi, _ = locate_region(q_norm, s_norm, allowed_gap(len(q_norm)), slack=0)
|
| 86 |
+
return " ".join(p[0] for p in s_pairs[lo:hi]) if hi > lo else source
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _diacritic_notes(quote_words: List[str], source_words: List[str], idgham_words: List[str]) -> List[dict]:
|
| 90 |
+
"""Words whose written harakat contradict the source (missing harakat are fine; idgham spelling is accepted)."""
|
| 91 |
+
notes = []
|
| 92 |
+
for q_word, s_word, g_word in zip(quote_words, source_words, idgham_words):
|
| 93 |
+
q_sig = vowel_signature(q_word)
|
| 94 |
+
if not any(marks for _, marks in q_sig):
|
| 95 |
+
continue
|
| 96 |
+
conflict = True
|
| 97 |
+
for variant in (s_word, g_word):
|
| 98 |
+
v_sig = vowel_signature(variant)
|
| 99 |
+
if len(v_sig) == len(q_sig) and all(set(qm) <= set(vm) for (_, qm), (_, vm) in zip(q_sig, v_sig)):
|
| 100 |
+
conflict = False
|
| 101 |
+
break
|
| 102 |
+
if conflict:
|
| 103 |
+
notes.append({"word": q_word, "source_word": s_word})
|
| 104 |
+
return notes
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def _orthographic_variant(q_tokens: Sequence[str], s_tokens: Sequence[str], vocabulary: Optional[Container[str]]) -> bool:
|
| 108 |
+
"""Same letters written differently: spacing (ياأيها / يا أيها) or a medial alef (إسحق / إسحاق).
|
| 109 |
+
|
| 110 |
+
The alef variant is tolerated only when the quoted spelling is not itself a word of the corpus (so قتل / قاتل stay a
|
| 111 |
+
real difference)."""
|
| 112 |
+
if "".join(q_tokens) == "".join(s_tokens):
|
| 113 |
+
return True
|
| 114 |
+
if vocabulary is not None and len(q_tokens) == 1 and len(s_tokens) == 1:
|
| 115 |
+
q, s = q_tokens[0], s_tokens[0]
|
| 116 |
+
return q.replace("ا", "") == s.replace("ا", "") and q not in vocabulary
|
| 117 |
+
return False
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def align(quote: str, source: str, vocabulary: Optional[Container[str]] = None) -> Dict[str, object]:
|
| 121 |
+
"""Align ``quote`` to the best matching region of ``source`` and describe every difference.
|
| 122 |
+
|
| 123 |
+
``vocabulary``: phonetic-skeleton words of the corpus; enables the medial-alef spelling tolerance."""
|
| 124 |
+
q_pairs, s_pairs = aligned_words(quote), aligned_words(source)
|
| 125 |
+
idgham_pairs = aligned_words(apply_idgham(source)) if len(source) < 20000 else s_pairs
|
| 126 |
+
if len(idgham_pairs) != len(s_pairs):
|
| 127 |
+
idgham_pairs = s_pairs
|
| 128 |
+
q_orig, q_norm = [p[0] for p in q_pairs], [p[1] for p in q_pairs]
|
| 129 |
+
s_orig, s_norm = [p[0] for p in s_pairs], [p[1] for p in s_pairs]
|
| 130 |
+
g_orig = [p[0] for p in idgham_pairs]
|
| 131 |
+
|
| 132 |
+
gap = allowed_gap(len(q_norm))
|
| 133 |
+
lo, hi, matcher = locate_region(q_norm, s_norm, gap)
|
| 134 |
+
|
| 135 |
+
raw_ops = [(tag, i1, i2, j1 + lo, j2 + lo) for tag, i1, i2, j1, j2 in matcher.get_opcodes()]
|
| 136 |
+
# Source words before / after a partial quotation are not errors unless the quotation itself contains them elsewhere
|
| 137 |
+
# (a moved word); drop such edge insertions.
|
| 138 |
+
deleted = {w for tag, i1, i2, _, _ in raw_ops if tag in ("delete", "replace") for w in q_norm[i1:i2]}
|
| 139 |
+
for position in (0, -1):
|
| 140 |
+
if len(raw_ops) > 1 and raw_ops[position][0] == "insert" and not (set(s_norm[raw_ops[position][3]:raw_ops[position][4]]) & deleted):
|
| 141 |
+
raw_ops.pop(position)
|
| 142 |
+
|
| 143 |
+
ops, quote_side, source_side, missing, extra, notes = [], [], [], [], [], []
|
| 144 |
+
mismatches = 0
|
| 145 |
+
orthographic = 0
|
| 146 |
+
for position, (tag, i1, i2, j1, j2) in enumerate(raw_ops):
|
| 147 |
+
source_idx = list(range(j1, j2))
|
| 148 |
+
if tag == "replace" and _orthographic_variant(q_norm[i1:i2], s_norm[j1:j2], vocabulary):
|
| 149 |
+
tag, orthographic = "equal", orthographic + 1 # spelling-only difference, not a different word
|
| 150 |
+
if tag == "replace" and position in (0, len(raw_ops) - 1) and len(raw_ops) > 1 and (j2 - j1) > (i2 - i1):
|
| 151 |
+
# at the edges the slack may have pulled in unrelated source words: keep only words the quotation also uses
|
| 152 |
+
keep = [j for j in source_idx if s_norm[j] in set(q_norm[i1:i2]) | deleted]
|
| 153 |
+
source_idx = keep
|
| 154 |
+
tag = "replace" if keep else "delete"
|
| 155 |
+
ops.append({"op": tag, "span": " ".join(q_orig[i1:i2]), "source": " ".join(s_orig[j] for j in source_idx)})
|
| 156 |
+
if tag == "equal":
|
| 157 |
+
notes += _diacritic_notes(q_orig[i1:i2], s_orig[j1:j2], g_orig[j1:j2])
|
| 158 |
+
else:
|
| 159 |
+
mismatches += max(i2 - i1, len(source_idx))
|
| 160 |
+
quote_side += q_norm[i1:i2]
|
| 161 |
+
source_side += [s_norm[j] for j in source_idx]
|
| 162 |
+
extra += q_orig[i1:i2] if tag in ("delete", "replace") else []
|
| 163 |
+
missing += [s_orig[j] for j in source_idx] if tag in ("insert", "replace") else []
|
| 164 |
+
|
| 165 |
+
matched = sum(i2 - i1 for tag, i1, i2, _, _ in raw_ops if tag == "equal")
|
| 166 |
+
exact = bool(q_norm) and mismatches == 0
|
| 167 |
+
return {
|
| 168 |
+
"word_similarity": round(2 * matched / max(len(q_norm) + (raw_ops[-1][4] - raw_ops[0][3] if raw_ops else 0), 1), 3),
|
| 169 |
+
"word_diff": ops,
|
| 170 |
+
"missing_from_span": missing,
|
| 171 |
+
"extra_in_span": extra,
|
| 172 |
+
"source_excerpt": " ".join(op["source"] for op in ops if op["source"]),
|
| 173 |
+
"exact": exact,
|
| 174 |
+
"mismatches": mismatches,
|
| 175 |
+
"allowed_gap": gap,
|
| 176 |
+
"near": (not exact) and matched > 0 and mismatches <= gap,
|
| 177 |
+
"reordered": bool(quote_side) and sorted(quote_side) == sorted(source_side),
|
| 178 |
+
"diacritic_notes": notes,
|
| 179 |
+
"orthographic_variants": orthographic,
|
| 180 |
+
}
|
app.js
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/* Main-thread UI. Heavy work (Python/Pyodide) lives in worker.js. */
|
| 2 |
+
(function () {
|
| 3 |
+
const cfg = window.ICV_CONFIG, cache = window.icvCache, $ = (id) => document.getElementById(id);
|
| 4 |
+
const PY_FILES = ["app.py", "verifier.py", "retrieval.py", "index_builder.py", "normalization.py", "alignment.py", "similarity.py",
|
| 5 |
+
"detector.py", "scanner.py", "idgham.py", "ui.py", "llm_client.py", "benchmark.py", "index/quran.idx.gz", "demo/examples.json"];
|
| 6 |
+
const SYSTEM_PROMPT = "أنت مساعد معرفي في العلوم الإسلامية. أجب بالعربية بإيجاز ودقة. عند الاستشهاد بآية قرآنية أو حديث نبوي اكتب نصه كاملًا واذكر السورة ورقم الآية أو مصدر الحديث.";
|
| 7 |
+
const HTTP_MESSAGES = { 400: "رفض الخادم الطلب.", 401: "مفتاح الواجهة البرمجية غير صالح.", 403: "المفتاح غير مصرّح له بهذا الطلب.", 429: "تم تجاوز حد الاستخدام أو الرصيد؛ حاول لاحقًا." };
|
| 8 |
+
const esc = (s) => String(s).replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c]));
|
| 9 |
+
|
| 10 |
+
let worker = null, ready = false, nextId = 0, examples = [], exampleIndex = 0, lastText = "";
|
| 11 |
+
const pending = new Map();
|
| 12 |
+
|
| 13 |
+
/* ---------- toast + clipboard ---------- */
|
| 14 |
+
let toastTimer = 0;
|
| 15 |
+
function toast(msg, error) {
|
| 16 |
+
const t = $("toast"); t.textContent = msg; t.classList.toggle("err", !!error); t.classList.add("show");
|
| 17 |
+
clearTimeout(toastTimer); toastTimer = setTimeout(() => t.classList.remove("show"), 2200);
|
| 18 |
+
}
|
| 19 |
+
async function copyText(text) {
|
| 20 |
+
try { if (navigator.clipboard && window.isSecureContext) { await navigator.clipboard.writeText(text); return true; } } catch (e) { /* fall back */ }
|
| 21 |
+
const area = document.createElement("textarea"); // works inside iframes / non-secure contexts
|
| 22 |
+
area.value = text; area.setAttribute("readonly", ""); area.style.cssText = "position:fixed;top:-1000px;opacity:0";
|
| 23 |
+
document.body.appendChild(area); area.select(); area.setSelectionRange(0, text.length);
|
| 24 |
+
let ok = false; try { ok = document.execCommand("copy"); } catch (e) { ok = false; }
|
| 25 |
+
area.remove(); return ok;
|
| 26 |
+
}
|
| 27 |
+
window.icvCopy = async (button) => {
|
| 28 |
+
const ok = await copyText(button.dataset.text || "");
|
| 29 |
+
toast(ok ? "تم نسخ النص بنجاح" : "تعذّر نسخ النص، حدّده يدويًا", !ok);
|
| 30 |
+
};
|
| 31 |
+
|
| 32 |
+
/* ---------- status, boot progress, skeletons ---------- */
|
| 33 |
+
function setStatus(text, busy = true) { $("status").innerHTML = (busy && text ? '<span class="spinner"></span>' : "") + esc(text); }
|
| 34 |
+
function setStep(step, fraction) {
|
| 35 |
+
document.querySelectorAll(".boot .step").forEach((el) => {
|
| 36 |
+
const n = Number(el.dataset.step); el.classList.toggle("done", n < step); el.classList.toggle("on", n === step);
|
| 37 |
+
});
|
| 38 |
+
if (fraction != null) { const bar = $("bootbar"); bar.classList.remove("indeterminate"); bar.firstElementChild.style.width = Math.round(fraction * 100) + "%"; }
|
| 39 |
+
}
|
| 40 |
+
function skeleton(n = 3) {
|
| 41 |
+
const card = '<div class="panel sk-card"><div class="skeleton sk-title"></div><div class="skeleton sk-line w1"></div><div class="skeleton sk-line w2"></div><div class="skeleton sk-line w3"></div></div>';
|
| 42 |
+
$("results").innerHTML = '<div class="icv" aria-busy="true">' + card.repeat(n) + "</div>";
|
| 43 |
+
}
|
| 44 |
+
function notice(text, kind) { $("results").innerHTML = '<div class="icv"><div class="notice ' + kind + '">' + esc(text) + "</div></div>"; }
|
| 45 |
+
function busy(flag) { for (const id of ["verify", "example", "ask"]) $(id).disabled = flag || !ready; }
|
| 46 |
+
|
| 47 |
+
/* ---------- worker RPC ---------- */
|
| 48 |
+
function call(type, payload) {
|
| 49 |
+
return new Promise((resolve) => { const id = ++nextId; pending.set(id, resolve); worker.postMessage({ type, id, ...payload }); });
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
/* ---------- model card (Subtask 1A, CAMeLBERT-MSA) ---------- */
|
| 53 |
+
function modelCard(det) {
|
| 54 |
+
const chips = det.tags.map((t) => {
|
| 55 |
+
const kind = t.tag === "O" ? "O" : t.tag.slice(2);
|
| 56 |
+
return '<span class="chip ' + kind + '">' + esc(t.token) + "<small>" + esc(t.tag) + "</small></span>";
|
| 57 |
+
}).join("");
|
| 58 |
+
const badge = det.simulated ? '<span class="badge-sim">Simulation · no weights loaded</span>' : '<span class="badge-live">Live · Hugging Face</span>';
|
| 59 |
+
return '<div class="icv"><div class="panel model-card"><div class="head"><span class="title">CAMeLBERT-MSA · Token Classification (Subtask 1A)</span>' + badge +
|
| 60 |
+
'</div><div class="chips">' + chips + "</div></div></div>";
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
/* ---------- direct verification ---------- */
|
| 64 |
+
async function runDirect() {
|
| 65 |
+
const text = $("text").value;
|
| 66 |
+
if (!text.trim()) return notice("الرجاء إدخال نص للتحقق منه.", "warn");
|
| 67 |
+
const engine = $("engine").value, key = "r:" + cache.hash(engine + "|" + text);
|
| 68 |
+
busy(true); lastText = text; $("export").hidden = true;
|
| 69 |
+
const hit = cache.get(key);
|
| 70 |
+
if (hit) { $("results").innerHTML = hit; setStatus("تم عرض النتيجة من الذاكرة المحلية.", false); $("export").hidden = false; busy(false); return; }
|
| 71 |
+
skeleton(); setStatus("جارٍ التحقق… (قد ينتظر إتمام تجهيز كتب الحديث)");
|
| 72 |
+
let html;
|
| 73 |
+
if (engine === "camelbert") {
|
| 74 |
+
setStatus("CAMeLBERT-MSA: جارٍ استخراج المقاطع…");
|
| 75 |
+
const det = await window.icvDetectors.detectSpans(text, async () => JSON.parse((await call("detect", { text })) || "[]"));
|
| 76 |
+
setStatus("جارٍ التحقق من المقاطع المستخرجة…");
|
| 77 |
+
const report = await call("verify_spans", { text, spans: JSON.stringify(det.spans) });
|
| 78 |
+
html = report === null ? null : modelCard(det) + report;
|
| 79 |
+
} else {
|
| 80 |
+
html = await call("verify", { text, generated: false });
|
| 81 |
+
}
|
| 82 |
+
if (html === null) notice("حدث خطأ غير متوقع أثناء التحقق.", "bad");
|
| 83 |
+
else { $("results").innerHTML = html; cache.set(key, html); $("export").hidden = false; }
|
| 84 |
+
setStatus("", false); busy(false);
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
/* ---------- Ask then Verify: pre-configured ChatGPT client ---------- */
|
| 88 |
+
async function askChatGPT(prompt) {
|
| 89 |
+
if (!prompt.trim()) throw new Error("اكتب سؤالًا أولًا.");
|
| 90 |
+
if (!cfg.OPENAI_API_KEY) throw new Error("لم يتم ضبط مفتاح ChatGPT في ملف config.js.");
|
| 91 |
+
let response;
|
| 92 |
+
try {
|
| 93 |
+
response = await fetch(cfg.OPENAI_ENDPOINT, { method: "POST",
|
| 94 |
+
headers: { "Content-Type": "application/json", Authorization: "Bearer " + cfg.OPENAI_API_KEY },
|
| 95 |
+
body: JSON.stringify({ model: cfg.OPENAI_MODEL, messages: [{ role: "system", content: SYSTEM_PROMPT }, { role: "user", content: prompt.trim() }] }) });
|
| 96 |
+
} catch (e) { throw new Error("تعذّر الاتصال بالخدمة. تحقق من الاتصال بالإنترنت."); }
|
| 97 |
+
if (!response.ok) throw new Error(HTTP_MESSAGES[response.status] || "فشل الطلب (الرمز " + response.status + ").");
|
| 98 |
+
const data = await response.json(), answer = data.choices && data.choices[0] && data.choices[0].message.content;
|
| 99 |
+
if (!answer || !answer.trim()) throw new Error("لم يُرجع النموذج أي نص.");
|
| 100 |
+
return answer.trim();
|
| 101 |
+
}
|
| 102 |
+
async function runAsk() {
|
| 103 |
+
busy(true); $("export").hidden = true;
|
| 104 |
+
try {
|
| 105 |
+
skeleton(2); setStatus("جارٍ سؤال ChatGPT…");
|
| 106 |
+
const answer = await askChatGPT($("prompt").value);
|
| 107 |
+
setStatus("جارٍ التحقق من إجابة النموذج…");
|
| 108 |
+
const html = await call("verify", { text: answer, generated: true });
|
| 109 |
+
if (html === null) notice("تعذّر التحقق من إجابة النموذج.", "bad");
|
| 110 |
+
else { $("results").innerHTML = html; lastText = answer; $("export").hidden = false; }
|
| 111 |
+
} catch (e) { notice(e.message, "warn"); }
|
| 112 |
+
setStatus("", false); busy(false);
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
/* ---------- benchmark export (IslamicEval-style 1A/1B/1C) ---------- */
|
| 116 |
+
async function exportBenchmark(kind) {
|
| 117 |
+
if (!lastText) return;
|
| 118 |
+
toast("Preparing export…");
|
| 119 |
+
const raw = await call("benchmark", { text: lastText });
|
| 120 |
+
if (!raw) return toast("تعذّر التصدير", true);
|
| 121 |
+
const data = JSON.parse(raw), body = kind === "tsv" ? data.tsv : JSON.stringify(data, null, 2);
|
| 122 |
+
const blob = new Blob([body], { type: kind === "tsv" ? "text/tab-separated-values" : "application/json" });
|
| 123 |
+
const a = document.createElement("a"); a.href = URL.createObjectURL(blob); a.download = "islamiceval_" + data.Response_ID + "." + kind;
|
| 124 |
+
document.body.appendChild(a); a.click(); a.remove(); setTimeout(() => URL.revokeObjectURL(a.href), 4000);
|
| 125 |
+
if (!(await copyText(body))) return; toast("Exported · copied to clipboard");
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
/* ---------- boot ---------- */
|
| 129 |
+
function start() {
|
| 130 |
+
try { worker = new Worker("worker.js"); } catch (e) { return setStatus("تعذّر تشغيل النظام: " + e.message, false); }
|
| 131 |
+
worker.onerror = (ev) => setStatus("تعذّر تشغيل النظام: " + (ev.message || ""), false);
|
| 132 |
+
worker.onmessage = (event) => {
|
| 133 |
+
const m = event.data;
|
| 134 |
+
if (m.type === "status") { setStatus(m.text); setStep(m.step, m.step === 1 ? 0.1 : m.step === 2 ? 0.3 : null); }
|
| 135 |
+
else if (m.type === "progress") setStep(3, 0.35 + 0.65 * (m.total ? m.got / m.total : 0.5));
|
| 136 |
+
else if (m.type === "error") setStatus("تعذّر تشغيل النظام: " + m.text + ". جرّب متصفح كمبيوتر حديثًا ثم أعد تحميل الصفحة.", false);
|
| 137 |
+
else if (m.type === "result") { const r = pending.get(m.id); pending.delete(m.id); r(m.value); }
|
| 138 |
+
else if (m.type === "ready") { ready = true; busy(false); setStep(3, 0.35); setStatus("النظام جاهز. جارٍ تجهيز كتب الحديث في الخلفية…"); }
|
| 139 |
+
else if (m.type === "hadith") { setStep(4, 1); setStatus("جاهز بالكامل. يعمل الكود داخل متصفحك ولا يُرسل نصك إلى أي خادم.", false); setTimeout(() => $("boot").hidden = true, 1800); }
|
| 140 |
+
};
|
| 141 |
+
worker.postMessage({ type: "boot", base: location.href, pyodideVersion: cfg.PYODIDE_VERSION, files: PY_FILES, lazy: ["index/hadith.idx.gz"] });
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
document.querySelectorAll(".tab").forEach((tab) => tab.addEventListener("click", () => {
|
| 145 |
+
document.querySelectorAll(".tab").forEach((t) => t.classList.toggle("active", t === tab));
|
| 146 |
+
for (const name of ["direct", "ask"]) $("tab-" + name).hidden = tab.dataset.tab !== name;
|
| 147 |
+
}));
|
| 148 |
+
$("engine").addEventListener("change", () => { $("engine-hint").textContent = $("engine").selectedOptions[0].dataset.hint; });
|
| 149 |
+
$("engine").dispatchEvent(new Event("change"));
|
| 150 |
+
$("verify").addEventListener("click", runDirect);
|
| 151 |
+
$("ask").addEventListener("click", runAsk);
|
| 152 |
+
$("example").addEventListener("click", () => { if (!examples.length) return; $("text").value = examples[exampleIndex++ % examples.length].text; runDirect(); });
|
| 153 |
+
$("export-json").addEventListener("click", () => exportBenchmark("json"));
|
| 154 |
+
$("export-tsv").addEventListener("click", () => exportBenchmark("tsv"));
|
| 155 |
+
document.addEventListener("click", (e) => { const b = e.target.closest && e.target.closest(".copy-btn"); if (b && !b.getAttribute("onclick")) window.icvCopy(b); });
|
| 156 |
+
cache.json("demo/examples.json").then((d) => { examples = d; }).catch(() => cache.json("examples.json").then((d) => { examples = d; }).catch(() => {}));
|
| 157 |
+
busy(true); setStep(1, 0.05); start();
|
| 158 |
+
})();
|
app.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""التحقق من هلوسة القرآن والحديث وتصحيحها: Gradio interface.
|
| 2 |
+
|
| 3 |
+
python app.py # http://127.0.0.1:7860
|
| 4 |
+
|
| 5 |
+
Mode A verifies pasted text. Mode B asks a language model first and verifies its answer. API keys are taken from the
|
| 6 |
+
form or, preferably, from environment variables (GEMINI_API_KEY, OPENAI_API_KEY, HF_TOKEN) so that a deployment can
|
| 7 |
+
keep them as secrets.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import json
|
| 12 |
+
import logging
|
| 13 |
+
import os
|
| 14 |
+
import threading
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from typing import List, Optional
|
| 17 |
+
|
| 18 |
+
import ui
|
| 19 |
+
from llm_client import PROVIDERS, LLMError, LLMSettings, generate
|
| 20 |
+
from verifier import MAX_INPUT_CHARS, IslamicContentVerifier
|
| 21 |
+
|
| 22 |
+
logger = logging.getLogger(__name__)
|
| 23 |
+
|
| 24 |
+
EXAMPLES_PATH = Path(__file__).resolve().parent / "demo" / "examples.json"
|
| 25 |
+
PROVIDER_LABELS = {"gemini": "جوجل جيميناي", "openai": "أوبن إيه آي", "huggingface": "هاغينغ فيس"}
|
| 26 |
+
KEY_ENV = {"gemini": "GEMINI_API_KEY", "openai": "OPENAI_API_KEY", "huggingface": "HF_TOKEN"}
|
| 27 |
+
|
| 28 |
+
_pipeline: Optional[IslamicContentVerifier] = None
|
| 29 |
+
_lock = threading.Lock()
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def get_pipeline() -> IslamicContentVerifier:
|
| 33 |
+
"""Created once. The Quran index loads immediately; the Hadith index loads lazily (see ``warm_in_background``)."""
|
| 34 |
+
global _pipeline
|
| 35 |
+
with _lock:
|
| 36 |
+
if _pipeline is None:
|
| 37 |
+
_pipeline = IslamicContentVerifier()
|
| 38 |
+
return _pipeline
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def warm_in_background() -> None:
|
| 42 |
+
threading.Thread(target=lambda: get_pipeline().retriever.warm(), daemon=True).start()
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def load_examples(path: Path = EXAMPLES_PATH) -> List[dict]:
|
| 46 |
+
try:
|
| 47 |
+
with open(path, encoding="utf-8") as handle:
|
| 48 |
+
return json.load(handle)
|
| 49 |
+
except (OSError, json.JSONDecodeError):
|
| 50 |
+
logger.exception("Could not load demo examples from %s", path)
|
| 51 |
+
return []
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def verify_text(text: str) -> str:
|
| 55 |
+
"""Mode A. Never raises: problems become Arabic notices."""
|
| 56 |
+
if not text or not text.strip():
|
| 57 |
+
return ui.render_message("الرجاء إدخال نص للتحقق منه.", "warn")
|
| 58 |
+
try:
|
| 59 |
+
return ui.render_results(get_pipeline().analyze(text))
|
| 60 |
+
except ValueError:
|
| 61 |
+
return ui.render_message(f"النص طويل جدًا (الحد الأقصى {MAX_INPUT_CHARS} حرف).", "warn")
|
| 62 |
+
except Exception:
|
| 63 |
+
logger.exception("Verification failed")
|
| 64 |
+
return ui.render_message("حدث خطأ غير متوقع أثناء التحقق.", "bad")
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def verify_generated_answer(answer: str) -> str:
|
| 68 |
+
"""Verify a model answer and show it above the report (also used by the in-browser page)."""
|
| 69 |
+
try:
|
| 70 |
+
return ui.render_results(get_pipeline().analyze(answer), generated_answer=answer)
|
| 71 |
+
except Exception:
|
| 72 |
+
logger.exception("Verification of the generated answer failed")
|
| 73 |
+
return ui.render_message("تعذّر التحقق من إجابة النموذج.", "bad")
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def analyze_benchmark(text: str, response_id: str = "R001") -> str:
|
| 77 |
+
"""IslamicEval-style JSON (1A/1B/1C rows + TSV) for a text. Used by the browser page's export button."""
|
| 78 |
+
from benchmark import benchmark_json
|
| 79 |
+
return benchmark_json(get_pipeline().analyze(text), response_id)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def detect_spans(text: str) -> str:
|
| 83 |
+
"""Detection only (Subtask 1A), as JSON ``[{label, start, end, text}]`` for the browser's model-selection step."""
|
| 84 |
+
spans = get_pipeline().detect(text)
|
| 85 |
+
return json.dumps([{"label": s.label, "start": s.start, "end": s.end, "text": s.text} for s in spans], ensure_ascii=False)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def verify_given_spans(text: str, spans_json: str) -> str:
|
| 89 |
+
"""Verify and correct spans supplied by an external detector (e.g. fine-tuned CAMeLBERT-MSA); returns the HTML report."""
|
| 90 |
+
try:
|
| 91 |
+
spans = [s for s in json.loads(spans_json) if s["label"] in ("Ayah", "Hadith") and 0 <= s["start"] < s["end"] <= len(text)]
|
| 92 |
+
return ui.render_results(get_pipeline().analyze_spans(text, spans))
|
| 93 |
+
except Exception:
|
| 94 |
+
logger.exception("Verification of given spans failed")
|
| 95 |
+
return ui.render_message("تعذّر التحقق من المقاطع المحدَّدة.", "bad")
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def ask_then_verify(prompt: str, provider: str = "openai", model: str = "", api_key: str = "") -> str:
|
| 99 |
+
"""Mode B: answer with the pre-configured ChatGPT client (key from OPENAI_API_KEY), then verify every quotation."""
|
| 100 |
+
key = (api_key or "").strip() or os.environ.get(KEY_ENV.get(provider, ""), "")
|
| 101 |
+
try:
|
| 102 |
+
answer = generate(LLMSettings(provider=provider, api_key=key, model=model or ""), prompt)
|
| 103 |
+
except LLMError as exc:
|
| 104 |
+
return ui.render_message(str(exc), "warn")
|
| 105 |
+
return verify_generated_answer(answer)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def build_interface():
|
| 109 |
+
import gradio as gr
|
| 110 |
+
|
| 111 |
+
examples = load_examples()
|
| 112 |
+
|
| 113 |
+
def next_example(index: int):
|
| 114 |
+
if not examples:
|
| 115 |
+
return "", 0
|
| 116 |
+
return examples[index % len(examples)]["text"], (index + 1) % len(examples)
|
| 117 |
+
|
| 118 |
+
def show_default_model(provider: str):
|
| 119 |
+
return PROVIDERS[provider]["model"]
|
| 120 |
+
|
| 121 |
+
with gr.Blocks(title="التحقق من هلوسة القرآن والحديث وتصحيحها", css=ui.CSS, theme=gr.themes.Base(primary_hue="emerald", neutral_hue="stone")) as demo:
|
| 122 |
+
gr.HTML(ui.HERO)
|
| 123 |
+
with gr.Tabs():
|
| 124 |
+
with gr.Tab("تحقق مباشر"):
|
| 125 |
+
example_index = gr.State(0)
|
| 126 |
+
text_input = gr.Textbox(label="النص المراد التحقق منه", lines=9, max_lines=24, placeholder=ui.PLACEHOLDER,
|
| 127 |
+
rtl=True, elem_classes="input-area")
|
| 128 |
+
with gr.Row():
|
| 129 |
+
verify_button = gr.Button("تحقّق من النص", variant="primary", scale=3)
|
| 130 |
+
example_button = gr.Button("جرّب مثالًا", variant="secondary", scale=2)
|
| 131 |
+
results = gr.HTML(elem_classes="results")
|
| 132 |
+
verify_button.click(verify_text, inputs=text_input, outputs=results)
|
| 133 |
+
example_button.click(next_example, inputs=example_index, outputs=[text_input, example_index]).then(
|
| 134 |
+
verify_text, inputs=text_input, outputs=results)
|
| 135 |
+
with gr.Tab("اسأل ثم تحقّق"):
|
| 136 |
+
gr.HTML('<div class="icv"><div class="notice">اكتب سؤالًا، وسيجيب ChatGPT، ثم يفحص النظام كل آية '
|
| 137 |
+
'وحديث ورد في إجابته ويعرض الأخطاء والتصحيحات.</div></div>')
|
| 138 |
+
prompt = gr.Textbox(label="سؤالك", lines=3, placeholder=ui.PROMPT_PLACEHOLDER, rtl=True, elem_classes="input-area")
|
| 139 |
+
ask_button = gr.Button("اسأل ثم تحقّق", variant="primary")
|
| 140 |
+
answer_results = gr.HTML(elem_classes="results")
|
| 141 |
+
ask_button.click(ask_then_verify, inputs=prompt, outputs=answer_results)
|
| 142 |
+
gr.HTML(ui.DISCLAIMER)
|
| 143 |
+
return demo
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def main() -> None:
|
| 147 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
| 148 |
+
get_pipeline()
|
| 149 |
+
warm_in_background()
|
| 150 |
+
build_interface().queue().launch(share=os.environ.get("ICV_SHARE") == "1")
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
if __name__ == "__main__":
|
| 154 |
+
main()
|
benchmark.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""IslamicEval-style export of a pipeline result.
|
| 2 |
+
|
| 3 |
+
The schema mirrors the project's bundled development subset (``data/islamiceval_dev_subset.jsonl``) and the three
|
| 4 |
+
subtasks of IslamicEval 2025:
|
| 5 |
+
|
| 6 |
+
* 1A detection ``Span_Start`` / ``Span_End`` / ``Span_Type`` (``Ayah`` | ``Hadith``; character offsets, end exclusive,
|
| 7 |
+
quotation marks excluded)
|
| 8 |
+
* 1B verification ``Label`` (``Correct`` | ``Incorrect``)
|
| 9 |
+
* 1C correction ``Correction`` the exact source text, or ``NO_SOURCE`` (``خطأ``) when no authentic source exists
|
| 10 |
+
|
| 11 |
+
Only incorrect spans carry a correction. Nothing is generated: a correction is always the text of a retrieved source.
|
| 12 |
+
Check the column names against the organisers' current submission page before an official submission.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import csv
|
| 17 |
+
import io
|
| 18 |
+
import json
|
| 19 |
+
from typing import Dict, List
|
| 20 |
+
|
| 21 |
+
NO_SOURCE = "خطأ"
|
| 22 |
+
TSV_COLUMNS = ["Response_ID", "Span_Start", "Span_End", "Span_Type", "Label", "Correction"]
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def to_benchmark(result: dict, response_id: str = "R001") -> Dict:
|
| 26 |
+
rows: List[dict] = []
|
| 27 |
+
for span in result["spans"]:
|
| 28 |
+
incorrect = span["status"] != "VERIFIED"
|
| 29 |
+
correction = None
|
| 30 |
+
if incorrect:
|
| 31 |
+
proposal = span.get("correction")
|
| 32 |
+
correction = proposal["text"] if span["status"] == "CORRECTED" and proposal else NO_SOURCE
|
| 33 |
+
rows.append({
|
| 34 |
+
"Response_ID": response_id,
|
| 35 |
+
"Span_Start": span["start"],
|
| 36 |
+
"Span_End": span["end"],
|
| 37 |
+
"Span_Type": span["type"],
|
| 38 |
+
"Label": "Incorrect" if incorrect else "Correct",
|
| 39 |
+
"Correction": correction,
|
| 40 |
+
"Status": span["status"], # extra, human-readable: VERIFIED / CORRECTED / UNSUPPORTED / HUMAN_REVIEW
|
| 41 |
+
"Text": span["text"],
|
| 42 |
+
})
|
| 43 |
+
return {"Response_ID": response_id, "Subtask_1A": [r for r in rows], "rows": rows, "summary": result["summary"]}
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def to_tsv(benchmark: Dict) -> str:
|
| 47 |
+
buffer = io.StringIO()
|
| 48 |
+
writer = csv.writer(buffer, delimiter="\t", lineterminator="\n")
|
| 49 |
+
writer.writerow(TSV_COLUMNS)
|
| 50 |
+
for row in benchmark["rows"]:
|
| 51 |
+
writer.writerow([row[c] if row[c] is not None else "" for c in TSV_COLUMNS])
|
| 52 |
+
return buffer.getvalue()
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def benchmark_json(result: dict, response_id: str = "R001") -> str:
|
| 56 |
+
data = to_benchmark(result, response_id)
|
| 57 |
+
data["tsv"] = to_tsv(data)
|
| 58 |
+
data.pop("Subtask_1A")
|
| 59 |
+
return json.dumps(data, ensure_ascii=False)
|
cache.js
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/* Small JSON cache on top of localStorage (all calls are failure-safe: the cache is only an optimisation). */
|
| 2 |
+
(function () {
|
| 3 |
+
const cfg = window.ICV_CONFIG, P = "icv:" + cfg.CACHE_VERSION + ":";
|
| 4 |
+
const hash = (s) => { let h = 2166136261; for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 16777619); } return (h >>> 0).toString(36) + s.length.toString(36); };
|
| 5 |
+
const get = (key) => { try { const v = JSON.parse(localStorage.getItem(P + key)); return v ? v.d : null; } catch (e) { return null; } };
|
| 6 |
+
function set(key, data) {
|
| 7 |
+
try {
|
| 8 |
+
localStorage.setItem(P + key, JSON.stringify({ t: Date.now(), d: data }));
|
| 9 |
+
const keys = Object.keys(localStorage).filter((k) => k.startsWith(P + "r:"));
|
| 10 |
+
if (keys.length > cfg.CACHE_MAX_ENTRIES) { // drop the oldest results
|
| 11 |
+
keys.map((k) => [k, (JSON.parse(localStorage.getItem(k)) || {}).t || 0]).sort((a, b) => a[1] - b[1])
|
| 12 |
+
.slice(0, keys.length - cfg.CACHE_MAX_ENTRIES).forEach(([k]) => localStorage.removeItem(k));
|
| 13 |
+
}
|
| 14 |
+
} catch (e) { /* quota or private mode */ }
|
| 15 |
+
}
|
| 16 |
+
function purgeOld() { try { Object.keys(localStorage).filter((k) => k.startsWith("icv:") && !k.startsWith(P)).forEach((k) => localStorage.removeItem(k)); } catch (e) {} }
|
| 17 |
+
async function json(url) { // stale-while-revalidate for small JSON files such as the demo examples
|
| 18 |
+
const cached = get("j:" + url);
|
| 19 |
+
const fresh = fetch(url).then((r) => r.json()).then((d) => { set("j:" + url, d); return d; });
|
| 20 |
+
if (cached) { fresh.catch(() => {}); return cached; }
|
| 21 |
+
return fresh;
|
| 22 |
+
}
|
| 23 |
+
purgeOld();
|
| 24 |
+
window.icvCache = { hash, get, set, json };
|
| 25 |
+
})();
|
camelbert.js
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/* Detection back-ends for Subtask 1A.
|
| 2 |
+
- "standard": rules + corpus scanner (runs in the Python worker).
|
| 3 |
+
- "camelbert": fine-tuned CAMeLBERT-MSA token classifier (BIO tags: B/I-Ayah, B/I-Hadith, O).
|
| 4 |
+
* With ICV_CONFIG.HF_MODEL_ID set, the Hugging Face Inference API is called.
|
| 5 |
+
* Without it, a clearly labelled SIMULATION runs: no model weights are loaded; the spans come from the
|
| 6 |
+
standard detector and are re-expressed as BIO tags with deterministic pseudo-confidences. */
|
| 7 |
+
(function () {
|
| 8 |
+
const cfg = window.ICV_CONFIG;
|
| 9 |
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
| 10 |
+
|
| 11 |
+
function bioTags(text, spans) {
|
| 12 |
+
const tokens = []; const re = /\S+/g; let m;
|
| 13 |
+
while ((m = re.exec(text))) {
|
| 14 |
+
const s = spans.find((x) => m.index < x.end && m.index + m[0].length > x.start);
|
| 15 |
+
let tag = "O";
|
| 16 |
+
if (s) tag = (m.index <= s.start || !tokens.length || !String(tokens[tokens.length - 1].tag).endsWith(s.label) ? "B-" : "I-") + s.label;
|
| 17 |
+
tokens.push({ token: m[0], tag });
|
| 18 |
+
}
|
| 19 |
+
return tokens;
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
async function viaHuggingFace(text) {
|
| 23 |
+
const headers = { "Content-Type": "application/json" };
|
| 24 |
+
if (cfg.HF_TOKEN) headers.Authorization = "Bearer " + cfg.HF_TOKEN;
|
| 25 |
+
const r = await fetch("https://api-inference.huggingface.co/models/" + cfg.HF_MODEL_ID, {
|
| 26 |
+
method: "POST", headers, body: JSON.stringify({ inputs: text, parameters: { aggregation_strategy: "simple" }, options: { wait_for_model: true } })
|
| 27 |
+
});
|
| 28 |
+
if (!r.ok) throw new Error("HF " + r.status);
|
| 29 |
+
const groups = await r.json();
|
| 30 |
+
const spans = groups.map((g) => ({ label: /hadith/i.test(g.entity_group) ? "Hadith" : "Ayah", start: g.start, end: g.end, score: g.score }))
|
| 31 |
+
.filter((s) => s.end > s.start);
|
| 32 |
+
return { spans, simulated: false };
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
/* detectSpans: returns { spans, tags, simulated, note }. `standard` is a function returning the standard detector's spans. */
|
| 36 |
+
async function detectSpans(text, standard) {
|
| 37 |
+
if (cfg.HF_MODEL_ID) {
|
| 38 |
+
try { const out = await viaHuggingFace(text); out.tags = bioTags(text, out.spans); return out; }
|
| 39 |
+
catch (e) { /* fall through to the simulation so a demo never breaks */ }
|
| 40 |
+
}
|
| 41 |
+
await sleep(500 + Math.min(700, text.length / 8)); // model "inference" latency, for a realistic demo
|
| 42 |
+
const spans = await standard();
|
| 43 |
+
const conf = (s) => 0.9 + ((parseInt(window.icvCache.hash(text + s.start), 36) % 90) / 1000);
|
| 44 |
+
const scored = spans.map((s) => ({ ...s, score: Math.round(conf(s) * 1000) / 1000 }));
|
| 45 |
+
return { spans: scored, tags: bioTags(text, scored), simulated: true };
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
window.icvDetectors = { detectSpans, bioTags };
|
| 49 |
+
})();
|
config.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/* Runtime configuration. Loaded before app.js. */
|
| 2 |
+
window.ICV_CONFIG = {
|
| 3 |
+
/* "Ask then Verify" uses this pre-configured OpenAI (ChatGPT) client.
|
| 4 |
+
WARNING: this file is public in a Static Space. Keep a small monthly spend limit on the key. */
|
| 5 |
+
OPENAI_API_KEY: "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",
|
| 6 |
+
OPENAI_MODEL: "gpt-4o-mini",
|
| 7 |
+
OPENAI_ENDPOINT: "https://api.openai.com/v1/chat/completions",
|
| 8 |
+
|
| 9 |
+
/* Fine-tuned CAMeLBERT-MSA token classifier hosted on Hugging Face.
|
| 10 |
+
Leave HF_MODEL_ID empty to run the in-browser simulation. */
|
| 11 |
+
HF_MODEL_ID: "", // e.g. "your-username/camelbert-msa-islamiceval-1a"
|
| 12 |
+
HF_TOKEN: "", // optional, only for private models
|
| 13 |
+
|
| 14 |
+
CACHE_VERSION: "v2",
|
| 15 |
+
CACHE_MAX_ENTRIES: 40,
|
| 16 |
+
PYODIDE_VERSION: "0.26.4"
|
| 17 |
+
};
|
detector.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Quotation detection (Subtask 1A): finds Quran / Hadith quotations in a text and their boundaries.
|
| 2 |
+
|
| 3 |
+
The detector is rule-based: quotation marks and brackets mark candidate segments and a corpus lookup decides the type
|
| 4 |
+
(Ayah / Hadith). Introductory phrases such as "قال الله تعالى" are NOT required; they are only a soft fallback for
|
| 5 |
+
altered quotations the corpus cannot confirm. No model weights are loaded."""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import re
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
from typing import List, Optional
|
| 11 |
+
|
| 12 |
+
from normalization import normalize_for_matching, normalize_lenient, normalize_strict, tokenize
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
# --------------------------------------------------------------------------------------------------------------
|
| 16 |
+
@dataclass
|
| 17 |
+
class DetectedSpan:
|
| 18 |
+
start: int
|
| 19 |
+
end: int # exclusive
|
| 20 |
+
label: str # 'Ayah' | 'Hadith'
|
| 21 |
+
confidence: Optional[float] # None for the rule backend
|
| 22 |
+
source: str # 'rules' | 'given'
|
| 23 |
+
text: str = ""
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
QUOTE_CHARS = " \t\r\n\"“”«»﴿﴾{}()[]"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def trim_span(text: str, start: int, end: int):
|
| 30 |
+
"""Drop whitespace and quotation marks at both edges (gold spans exclude the quote marks)."""
|
| 31 |
+
while start < end and text[start] in QUOTE_CHARS:
|
| 32 |
+
start += 1
|
| 33 |
+
while end > start and text[end - 1] in QUOTE_CHARS + ".،,؛:":
|
| 34 |
+
end -= 1
|
| 35 |
+
return start, end
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
AYAH_TRIGGERS = [
|
| 39 |
+
"قال الله", "قوله تعالى", "قال تعالى", "يقول الله", "يقول تعالى", "قال سبحانه", "قوله سبحانه", "في كتابه",
|
| 40 |
+
"سورة", "الآية", "الاية", "الآيات", "القرآن", "القران", "كتاب الله", "عز وجل", "جل جلاله", "فقال تعالى",
|
| 41 |
+
"ذكر الله", "﴿",
|
| 42 |
+
]
|
| 43 |
+
HADITH_TRIGGERS = [
|
| 44 |
+
"رسول الله", "النبي", "صلى الله عليه وسلم", "ﷺ", "عليه الصلاة والسلام", "حديث", "رواه", "روى", "متفق عليه",
|
| 45 |
+
"الحديث", "فقال", "قال ص", "صلى الله عليه", "وسلم",
|
| 46 |
+
]
|
| 47 |
+
_FORMULA_WORDS = {
|
| 48 |
+
normalize_for_matching(w)
|
| 49 |
+
for w in "قال قالت رسول الله صلى عليه وسلم النبي تعالى سبحانه عز وجل فقال يقول الكريم الشريف الحديث الآية روى رواه عن أن أنه البخاري ومسلم".split()
|
| 50 |
+
}
|
| 51 |
+
_BRACKET_PAIRS = [("“", "”"), ("«", "»"), ("﴿", "﴾"), ("{", "}"), ("(", ")"), ("[", "]")]
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class RuleDetector:
|
| 55 |
+
"""Quotation-mark and trigger-phrase detector with an optional corpus lookup (no training, no GPU)."""
|
| 56 |
+
|
| 57 |
+
def __init__(self, retriever=None, min_words: int = 3, context_chars: int = 110,
|
| 58 |
+
min_corpus_cov: float = 0.6) -> None:
|
| 59 |
+
self.kb, self.min_words, self.context_chars, self.min_corpus_cov = retriever, min_words, context_chars, min_corpus_cov
|
| 60 |
+
|
| 61 |
+
@staticmethod
|
| 62 |
+
def _segments(text: str):
|
| 63 |
+
segments = []
|
| 64 |
+
quote_positions = [m.start() for m in re.finditer('"', text)]
|
| 65 |
+
if len(quote_positions) % 2 == 0:
|
| 66 |
+
pairs = zip(quote_positions[0::2], quote_positions[1::2]) # opening/closing pairs
|
| 67 |
+
else: # a stray quote: fall back to every consecutive pair
|
| 68 |
+
pairs = zip(quote_positions, quote_positions[1:])
|
| 69 |
+
for a, b in pairs:
|
| 70 |
+
segments.append((a + 1, b))
|
| 71 |
+
for opener, closer in _BRACKET_PAIRS:
|
| 72 |
+
for m in re.finditer(re.escape(opener) + r"(.*?)" + re.escape(closer), text, re.S):
|
| 73 |
+
segments.append((m.start(1), m.end(1)))
|
| 74 |
+
return segments
|
| 75 |
+
|
| 76 |
+
@staticmethod
|
| 77 |
+
def _trigger_type(context: str) -> Optional[str]:
|
| 78 |
+
best_end, best_label = -1, None
|
| 79 |
+
for label, triggers in (("Ayah", AYAH_TRIGGERS), ("Hadith", HADITH_TRIGGERS)):
|
| 80 |
+
for trigger in triggers:
|
| 81 |
+
pos = context.rfind(trigger)
|
| 82 |
+
if pos >= 0 and pos + len(trigger) > best_end:
|
| 83 |
+
best_end, best_label = pos + len(trigger), label
|
| 84 |
+
return best_label
|
| 85 |
+
|
| 86 |
+
def _corpus_coverage(self, span: str):
|
| 87 |
+
"""Highest word coverage of the span by any top Quran ayah / Hadith candidate."""
|
| 88 |
+
if self.kb is None:
|
| 89 |
+
return 0.0, 0.0
|
| 90 |
+
quran_words = set(tokenize(normalize_strict(span)))
|
| 91 |
+
if not quran_words:
|
| 92 |
+
return 0.0, 0.0
|
| 93 |
+
quran_cov = max(
|
| 94 |
+
(len(quran_words & set(tokenize(normalize_strict(c["text"])))) / len(quran_words)
|
| 95 |
+
for c in self.kb.search_quran_ayahs(span, top_k=5)),
|
| 96 |
+
default=0.0,
|
| 97 |
+
)
|
| 98 |
+
hadith_words = set(tokenize(normalize_lenient(span)))
|
| 99 |
+
hadith_cov = max(
|
| 100 |
+
(len(hadith_words & set(tokenize(normalize_lenient(c["text"])))) / len(hadith_words)
|
| 101 |
+
for c in self.kb.search_hadith(span, top_k=5)),
|
| 102 |
+
default=0.0,
|
| 103 |
+
) if hadith_words else 0.0
|
| 104 |
+
return quran_cov, hadith_cov
|
| 105 |
+
|
| 106 |
+
def detect(self, text: str) -> List[DetectedSpan]:
|
| 107 |
+
candidates = []
|
| 108 |
+
for start, end in self._segments(text):
|
| 109 |
+
start, end = trim_span(text, start, end)
|
| 110 |
+
if end <= start:
|
| 111 |
+
continue
|
| 112 |
+
inner = text[start:end]
|
| 113 |
+
words = [w for w in normalize_for_matching(inner).split() if w]
|
| 114 |
+
if len(words) < self.min_words or len(inner) > 3000:
|
| 115 |
+
continue
|
| 116 |
+
if sum(w in _FORMULA_WORDS for w in words) / len(words) >= 0.6:
|
| 117 |
+
continue
|
| 118 |
+
trigger = self._trigger_type(text[max(0, start - self.context_chars):start])
|
| 119 |
+
quran_cov, hadith_cov = self._corpus_coverage(inner)
|
| 120 |
+
label = None
|
| 121 |
+
# 1) The corpus decides: a quoted segment that matches the Quran / Hadith needs no introductory phrase.
|
| 122 |
+
if max(quran_cov, hadith_cov) >= self.min_corpus_cov and len(words) >= 4:
|
| 123 |
+
label = "Ayah" if quran_cov >= hadith_cov else "Hadith"
|
| 124 |
+
# 2) Introductory phrases are only a soft fallback for altered quotations that the corpus cannot confirm.
|
| 125 |
+
elif trigger:
|
| 126 |
+
label = trigger
|
| 127 |
+
other, mine = (hadith_cov, quran_cov) if trigger == "Ayah" else (quran_cov, hadith_cov)
|
| 128 |
+
if other >= 0.8 and mine < 0.5:
|
| 129 |
+
label = "Hadith" if trigger == "Ayah" else "Ayah"
|
| 130 |
+
if label is None:
|
| 131 |
+
continue
|
| 132 |
+
candidates.append((bool(trigger), max(quran_cov, hadith_cov), end - start, start, end, label))
|
| 133 |
+
|
| 134 |
+
candidates.sort(key=lambda c: (c[0], c[1], c[2]), reverse=True) # trigger first, then corpus match, then length
|
| 135 |
+
taken = []
|
| 136 |
+
for _, _, _, start, end, label in candidates:
|
| 137 |
+
if all(end <= t_start or start >= t_end for t_start, t_end, _ in taken):
|
| 138 |
+
taken.append((start, end, label))
|
| 139 |
+
taken.sort()
|
| 140 |
+
return [DetectedSpan(s, e, label, None, "rules", text[s:e]) for s, e, label in taken]
|
examples.json
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"id": "correct_ayah",
|
| 4 |
+
"title": "آية صحيحة",
|
| 5 |
+
"scenario": "correct",
|
| 6 |
+
"text": "قال الله تعالى: \"يَا أَيُّهَا الَّذِينَ آمَنُوا اسْتَعِينُوا بِالصَّبْرِ وَالصَّلَاةِ ۚ إِنَّ اللَّهَ مَعَ الصَّابِرِينَ\". وفي الآية بشارة لمن صبر.",
|
| 7 |
+
"built_from": [
|
| 8 |
+
"سورة البقرة 153"
|
| 9 |
+
],
|
| 10 |
+
"observed_statuses": [
|
| 11 |
+
"VERIFIED"
|
| 12 |
+
]
|
| 13 |
+
},
|
| 14 |
+
{
|
| 15 |
+
"id": "correct_hadith",
|
| 16 |
+
"title": "حديث صحيح",
|
| 17 |
+
"scenario": "correct",
|
| 18 |
+
"text": "وقال رسول الله ﷺ: \"إِنَّمَا الْأَعْمَالُ بِالنِّيَّاتِ ، وَإِنَّمَا لِكُلِّ امْرِئٍ مَا نَوَى ، فَمَنْ كَانَتْ هِجْرَتُهُ إِلَى دُنْيَا يُصِيبُهَا ، أَوْ إِلَى امْرَأَةٍ يَنْكِحُهَا ، فَهِجْرَتُهُ إِلَى مَا هَاجَرَ إِلَيْهِ\". ويستفاد منه أهمية النية.",
|
| 19 |
+
"built_from": [
|
| 20 |
+
"حديث رقم 5"
|
| 21 |
+
],
|
| 22 |
+
"observed_statuses": [
|
| 23 |
+
"VERIFIED"
|
| 24 |
+
]
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"id": "substituted_word",
|
| 28 |
+
"title": "كلمة مستبدلة: «ثم استقم» بدل «فاستقم»",
|
| 29 |
+
"scenario": "altered",
|
| 30 |
+
"text": "قال الله تعالى: \"ثُمَّ اسْتَقِمْ كَمَا أُمِرْتَ وَمَنْ تَابَ مَعَكَ\".",
|
| 31 |
+
"built_from": [
|
| 32 |
+
"سورة هود 112 (فَاسْتَقِمْ ← ثُمَّ اسْتَقِمْ)"
|
| 33 |
+
],
|
| 34 |
+
"observed_statuses": [
|
| 35 |
+
"CORRECTED"
|
| 36 |
+
]
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
"id": "swapped_order",
|
| 40 |
+
"title": "تبديل ترتيب الكلمات",
|
| 41 |
+
"scenario": "altered",
|
| 42 |
+
"text": "قال الله تعالى: \"كَمَا أُمِرْتَ فَاسْتَقِمْ وَمَنْ تَابَ مَعَكَ\".",
|
| 43 |
+
"built_from": [
|
| 44 |
+
"سورة هود 112 (تقديم وتأخير)"
|
| 45 |
+
],
|
| 46 |
+
"observed_statuses": [
|
| 47 |
+
"CORRECTED"
|
| 48 |
+
]
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"id": "wrong_diacritic",
|
| 52 |
+
"title": "تشكيل خاطئ لكلمة",
|
| 53 |
+
"scenario": "altered",
|
| 54 |
+
"text": "قال الله تعالى: \"فَاسْتَقَمْ كَمَا أُمِرْتَ وَمَنْ تَابَ مَعَكَ\".",
|
| 55 |
+
"built_from": [
|
| 56 |
+
"سورة هود 112 (فَاسْتَقِمْ ← فَاسْتَقَمْ)"
|
| 57 |
+
],
|
| 58 |
+
"observed_statuses": [
|
| 59 |
+
"VERIFIED"
|
| 60 |
+
]
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"id": "missing_diacritics",
|
| 64 |
+
"title": "بدون تشكيل (مقبول)",
|
| 65 |
+
"scenario": "correct",
|
| 66 |
+
"text": "قال الله تعالى: \"فاستقم كما أمرت ومن تاب معك\".",
|
| 67 |
+
"built_from": [
|
| 68 |
+
"سورة هود 112 بلا تشكيل"
|
| 69 |
+
],
|
| 70 |
+
"observed_statuses": [
|
| 71 |
+
"VERIFIED"
|
| 72 |
+
]
|
| 73 |
+
},
|
| 74 |
+
{
|
| 75 |
+
"id": "partial_citation",
|
| 76 |
+
"title": "اقتباس جزئي من آية",
|
| 77 |
+
"scenario": "partial",
|
| 78 |
+
"text": "قال الله تعالى: \"۞ وَقَضَىٰ رَبُّكَ أَلَّا تَعْبُدُوا إِلَّا\".",
|
| 79 |
+
"built_from": [
|
| 80 |
+
"سورة الإسراء 23 (أول ست كلمات)"
|
| 81 |
+
],
|
| 82 |
+
"observed_statuses": [
|
| 83 |
+
"VERIFIED"
|
| 84 |
+
]
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"id": "missing_word",
|
| 88 |
+
"title": "كلمة ناقصة",
|
| 89 |
+
"scenario": "altered",
|
| 90 |
+
"text": "قال الله تعالى: \"يَا أَيُّهَا آمَنُوا اسْتَعِينُوا بِالصَّبْرِ وَالصَّلَاةِ ۚ إِنَّ اللَّهَ مَعَ الصَّابِرِينَ\".",
|
| 91 |
+
"built_from": [
|
| 92 |
+
"سورة البقرة 153 (حُذفت الكلمة الثالثة)"
|
| 93 |
+
],
|
| 94 |
+
"observed_statuses": [
|
| 95 |
+
"CORRECTED"
|
| 96 |
+
]
|
| 97 |
+
},
|
| 98 |
+
{
|
| 99 |
+
"id": "unannounced_quote",
|
| 100 |
+
"title": "اقتباس غير معلَن بلا علامات تنصيص",
|
| 101 |
+
"scenario": "unannounced",
|
| 102 |
+
"text": "وهنا نتذكر أن العبد عليه أن يثبت وألا يتراجع، فاستقم كما أمرت ومن تاب معك ولا تطغوا، وهذا من أعظم ما يعين على الطاعة.",
|
| 103 |
+
"built_from": [
|
| 104 |
+
"سورة هود 112 داخل الجملة"
|
| 105 |
+
],
|
| 106 |
+
"observed_statuses": [
|
| 107 |
+
"VERIFIED"
|
| 108 |
+
]
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"id": "uncertain_hadith",
|
| 112 |
+
"title": "حديث مركّب من حديثين",
|
| 113 |
+
"scenario": "uncertain",
|
| 114 |
+
"text": "روى البخاري أن النبي ﷺ قال: \"إِنَّمَا الْأَعْمَالُ بِالنِّيَّاتِ ، وَإِنَّمَا لِكُلِّ امْرِئٍ مَا نَوَى ، فَمَنْ كَانَتْ هِجْرَتُهُ أَهْلَ الْأَرْضِ يَرْحَمْكُمْ مَنْ فِي السَّمَاءِ\".",
|
| 115 |
+
"built_from": [
|
| 116 |
+
"حديث رقم 5 (النصف الأول)",
|
| 117 |
+
"حديث رقم 95766 (النصف الثاني)"
|
| 118 |
+
],
|
| 119 |
+
"observed_statuses": [
|
| 120 |
+
"HUMAN_REVIEW"
|
| 121 |
+
]
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"id": "no_quotation",
|
| 125 |
+
"title": "نص بلا اقتباسات",
|
| 126 |
+
"scenario": "none",
|
| 127 |
+
"text": "الصبر من أهم الصفات التي ينبغي أن يتحلى بها الإنسان في حياته، ويساعده على مواجهة المصاعب بثبات وهدوء.",
|
| 128 |
+
"built_from": [],
|
| 129 |
+
"observed_statuses": []
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"id": "mixed_response",
|
| 133 |
+
"title": "رد مختلط",
|
| 134 |
+
"scenario": "mixed",
|
| 135 |
+
"text": "بخصوص بر الوالدين، قال الله تعالى: \"۞ وَقَضَىٰ رَبُّكَ أَلَّا تَعْبُدُوا إِلَّا إِيَّاهُ وَبِالْوَالِدَيْنِ إِحْسَانًا ۚ إِمَّا يَبْلُغَنَّ عِنْدَكَ الْكِبَرَ أَحَدُهُمَا أَوْ كِلَاهُمَا فَلَا تَقُلْ لَهُمَا أُفٍّ وَلَا تَنْهَرْهُمَا وَقُلْ لَهُمَا قَوْلًا كَرِيمًا\". وقال رسول الله ﷺ: \"الرَّاحِمُونَ يَرْحَمُهُمُ الرَّحْمَنُ ، ارْحَمُوا أَهْلَ الْأَرْضِ يَرْحَمْكُمْ مَنْ فِي السَّمَاءِ\". وقال سبحانه: \"يَا أَيُّهَا آمَنُوا اسْتَعِينُوا بِالصَّبْرِ وَالصَّلَاةِ ۚ إِنَّ اللَّهَ مَعَ الصَّابِرِينَ\". وهذا من مكارم الأخلاق.",
|
| 136 |
+
"built_from": [
|
| 137 |
+
"سورة الإسراء 23",
|
| 138 |
+
"حديث رقم 95766",
|
| 139 |
+
"سورة البقرة 153 (كلمة ناقصة)"
|
| 140 |
+
],
|
| 141 |
+
"observed_statuses": [
|
| 142 |
+
"VERIFIED",
|
| 143 |
+
"VERIFIED",
|
| 144 |
+
"CORRECTED"
|
| 145 |
+
]
|
| 146 |
+
}
|
| 147 |
+
]
|
hadith.idx.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6f5792317d7577220e0bd4b1aab84be1ed665c17f974ab7ba1cd8920f34c9955
|
| 3 |
+
size 13597012
|
hadith.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f1f514d7a2de2d4179d9588a861cedf07e81630886444f37d0c08c2cc777075a
|
| 3 |
+
size 48498788
|
idgham.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Idgham rendering of the Quran text in the published mushaf convention (used by the Subtask 1C gold corrections).
|
| 2 |
+
|
| 3 |
+
The flat Quran text writes assimilated letters without a shadda; the mushaf marks the assimilation. Covers noon
|
| 4 |
+
sakinah / tanween before ن م ر ل, meem sakinah before م and lam sakinah before ل ر, also across ayah-number markers
|
| 5 |
+
such as ``(20)``. The و / ي cases are excluded on purpose because the mushaf convention is inconsistent there."""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import re
|
| 9 |
+
|
| 10 |
+
_MARKER = re.compile(r"^\(\d+\)$")
|
| 11 |
+
|
| 12 |
+
_SUKUN, _SHADDA, _FATHATAN = "\u0652", "\u0651", "\u064B"
|
| 13 |
+
_TANWEEN = set("\u064B\u064C\u064D")
|
| 14 |
+
_IDGHAM_AFTER_NOON = set("نمرل")
|
| 15 |
+
_IDGHAM_AFTER_LAM = set("لر")
|
| 16 |
+
_DIACRITIC_CHARS = set(
|
| 17 |
+
"\u0610\u0611\u0612\u0613\u0614\u0615\u0616\u0617\u0618\u0619\u061A"
|
| 18 |
+
"\u064B\u064C\u064D\u064E\u064F\u0650\u0651\u0652\u0670"
|
| 19 |
+
"\u06D6\u06D7\u06D8\u06D9\u06DA\u06DB\u06DC\u06DF\u06E0\u06E1\u06E2\u06E3\u06E4"
|
| 20 |
+
"\u06E7\u06E8\u06EA\u06EB\u06EC\u06ED"
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _base_letters(word: str) -> str:
|
| 25 |
+
return "".join(c for c in word if c not in _DIACRITIC_CHARS)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _insert_shadda(word: str) -> str:
|
| 29 |
+
return word if not word else word[0] + _SHADDA + word[1:]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _ends_with_tanween(word: str) -> bool:
|
| 33 |
+
if not word:
|
| 34 |
+
return False
|
| 35 |
+
if word[-1] in _TANWEEN:
|
| 36 |
+
return True
|
| 37 |
+
return len(word) >= 2 and word[-1] in "اى" and word[-2] == _FATHATAN
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def apply_idgham(text: str, extended: bool = True) -> str:
|
| 41 |
+
"""Convert the flat Quran text into the mushaf rendering that marks assimilation with a shadda.
|
| 42 |
+
|
| 43 |
+
Covers noon sakinah / tanween before ن م ر ل, meem sakinah before م and lam sakinah before ل ر, also across
|
| 44 |
+
ayah-number markers such as ``(20)``. The و / ي cases are excluded on purpose because the mushaf convention
|
| 45 |
+
is inconsistent there."""
|
| 46 |
+
words = text.split(" ")
|
| 47 |
+
noon_set = _IDGHAM_AFTER_NOON if extended else set("نم")
|
| 48 |
+
i = 0
|
| 49 |
+
while i < len(words):
|
| 50 |
+
word = words[i]
|
| 51 |
+
if _MARKER.match(word) or not word:
|
| 52 |
+
i += 1
|
| 53 |
+
continue
|
| 54 |
+
j = i + 1
|
| 55 |
+
while j < len(words) and _MARKER.match(words[j]):
|
| 56 |
+
j += 1
|
| 57 |
+
if j < len(words):
|
| 58 |
+
base = _base_letters(words[j])
|
| 59 |
+
first = base[0] if base else ""
|
| 60 |
+
if word.endswith("\u0646" + _SUKUN) and first in noon_set:
|
| 61 |
+
words[i], words[j] = word[:-1], _insert_shadda(words[j])
|
| 62 |
+
elif _ends_with_tanween(word) and first in noon_set:
|
| 63 |
+
words[j] = _insert_shadda(words[j])
|
| 64 |
+
elif word.endswith("\u0645" + _SUKUN) and first == "\u0645":
|
| 65 |
+
words[i], words[j] = word[:-1], _insert_shadda(words[j])
|
| 66 |
+
elif extended and word.endswith("\u0644" + _SUKUN) and first in _IDGHAM_AFTER_LAM:
|
| 67 |
+
words[i], words[j] = word[:-1], _insert_shadda(words[j])
|
| 68 |
+
i += 1
|
| 69 |
+
return " ".join(words)
|
| 70 |
+
|
| 71 |
+
|
index.html
CHANGED
|
@@ -1,19 +1,64 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html>
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
</html>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="ar" dir="rtl">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
|
| 6 |
+
<title>التحقق من هلوسة القرآن والحديث وتصحيحها</title>
|
| 7 |
+
<meta name="description" content="التحقق من هلوسة القرآن والحديث وتصحيحها، يعمل بالكامل داخل المتصفح.">
|
| 8 |
+
<link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
| 9 |
+
<link rel="preconnect" href="https://cdn.jsdelivr.net" crossorigin>
|
| 10 |
+
<link rel="preload" as="script" href="worker.js">
|
| 11 |
+
<link rel="preload" as="fetch" href="index/quran.idx.gz" crossorigin>
|
| 12 |
+
<link rel="prefetch" as="fetch" href="index/hadith.idx.gz" crossorigin>
|
| 13 |
+
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Inter:wght@500;600&display=swap">
|
| 14 |
+
<link rel="stylesheet" href="styles.css">
|
| 15 |
+
</head>
|
| 16 |
+
<body class="icv-page">
|
| 17 |
+
<div class="page">
|
| 18 |
+
|
| 19 |
+
<div class="icv"><div class="hero">
|
| 20 |
+
<svg class="mark" viewBox="0 0 64 64" fill="none" stroke="#E2C06E" stroke-width="1.8" aria-hidden="true"><rect x="14" y="14" width="36" height="36"/><rect x="14" y="14" width="36" height="36" transform="rotate(45 32 32)"/><circle cx="32" cy="32" r="7" fill="#E2C06E" stroke="none"/></svg>
|
| 21 |
+
<h1>التحقق من هلوسة القرآن والحديث وتصحيحها</h1>
|
| 22 |
+
<p class="tagline">تحقّق من آيات القرآن والأحاديث النبوية بالدليل</p>
|
| 23 |
+
<p class="sub">يكتشف النظام الاقتباسات داخل أي نص، ويسترجع نصوصها من المصادر، ويقارنها كلمةً بكلمة، ثم يعرض الدليل.
|
| 24 |
+
وإذا لم تكفِ الأدلة فلن يختلق تصحيحًا، بل يحيل الحالة إلى المراجعة البشرية.</p>
|
| 25 |
+
<div class="flow"><span>كشف</span><i>‹</i><span>استرجاع</span><i>‹</i><span>محاذاة</span><i>‹</i><span>دليل</span><i>‹</i><span>قرار</span></div>
|
| 26 |
+
</div></div>
|
| 27 |
+
|
| 28 |
+
<div id="boot" class="panel boot" role="status">
|
| 29 |
+
<div class="steps"><span class="step on" data-step="1">1 · Python runtime</span><span class="step" data-step="2">2 · Quran index</span><span class="step" data-step="3">3 · Hadith books</span></div>
|
| 30 |
+
<div id="bootbar" class="progress indeterminate"><i></i></div>
|
| 31 |
+
</div>
|
| 32 |
+
<div class="tabs"><button class="tab active" data-tab="direct">تحقّق مباشر</button><button class="tab" data-tab="ask">اسأل ثم تحقّق</button></div>
|
| 33 |
+
<section id="tab-direct" class="form glass">
|
| 34 |
+
<label for="text">النص المراد التحقق منه</label>
|
| 35 |
+
<textarea id="text" placeholder="الصق هنا أي نص يتضمن آيات أو أحاديث، سواء كانت بين علامات تنصيص أو بدونها، ومع أي صياغة تمهيدية أو بدونها…"></textarea>
|
| 36 |
+
<div class="engine-row">
|
| 37 |
+
<label for="engine" class="en" style="margin:0">Detection model</label>
|
| 38 |
+
<select id="engine" class="en">
|
| 39 |
+
<option value="standard" data-hint="Rule-based detector with corpus matching. Runs fully in your browser.">Standard · Rules + Corpus</option>
|
| 40 |
+
<option value="camelbert" data-hint="Fine-tuned CAMeLBERT-MSA token classifier (Hugging Face). Runs as a simulation until a model ID is set in config.js.">CAMeLBERT-MSA · Fine-tuned</option>
|
| 41 |
+
</select>
|
| 42 |
+
<span id="engine-hint" class="hint en"></span>
|
| 43 |
+
</div>
|
| 44 |
+
<div class="btns"><button id="verify" class="btn-main" disabled>تحقّق من النص</button><button id="example" class="btn-alt" disabled>جرّب مثالًا</button></div>
|
| 45 |
+
</section>
|
| 46 |
+
<section id="tab-ask" class="form glass" hidden>
|
| 47 |
+
<div class="icv"><div class="notice">اكتب سؤالًا، وسيجيب ChatGPT، ثم يفحص النظام كل آية وحديث ورد في إجابته ويعرض الأخطاء والتصحيحات.</div></div>
|
| 48 |
+
<label for="prompt">سؤالك</label>
|
| 49 |
+
<textarea id="prompt" class="short" placeholder="اكتب سؤالك، مثل: اشرح لي فضل الصبر في القرآن والسنة مع ذكر الأدلة."></textarea>
|
| 50 |
+
<div class="btns"><button id="ask" class="btn-main" disabled>اسأل ثم تحقّق</button></div>
|
| 51 |
+
</section>
|
| 52 |
+
<div id="status" role="status" aria-live="polite"></div>
|
| 53 |
+
<div id="results" class="results"></div>
|
| 54 |
+
<div id="export" class="export" hidden><button id="export-json" type="button">Export JSON (1A · 1B · 1C)</button><button id="export-tsv" type="button">Export TSV</button></div>
|
| 55 |
+
<div class="icv"><div class="disclaimer">أداة مساعدة للتدقيق النصي وليست فتوى ولا بديلًا عن المراجعة المتخصصة.
|
| 56 |
+
النتائج مبنية على مراجع القرآن الكريم والكتب الستة المضمّنة فقط.</div></div>
|
| 57 |
+
</div>
|
| 58 |
+
<div id="toast" role="status" aria-live="polite"></div>
|
| 59 |
+
<script src="config.js"></script>
|
| 60 |
+
<script src="cache.js"></script>
|
| 61 |
+
<script src="camelbert.js"></script>
|
| 62 |
+
<script src="app.js"></script>
|
| 63 |
+
</body>
|
| 64 |
</html>
|
index_builder.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Build the pre-tokenised search indexes shipped in ``index/``.
|
| 2 |
+
|
| 3 |
+
python index_builder.py # writes index/quran.idx.gz and index/hadith.idx.gz
|
| 4 |
+
|
| 5 |
+
An index is a gzip-compressed pickle holding the raw records, their phonetic skeletons and BM25 postings, so the
|
| 6 |
+
application never re-tokenises the corpora at start-up. The pickles are produced locally by this script and read
|
| 7 |
+
back only by this project; never load an index file from an untrusted source.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import gzip
|
| 12 |
+
import json
|
| 13 |
+
import math
|
| 14 |
+
import pickle
|
| 15 |
+
import sys
|
| 16 |
+
import time
|
| 17 |
+
import zlib
|
| 18 |
+
from array import array
|
| 19 |
+
from collections import Counter, defaultdict
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
from typing import Dict, List, Sequence, Tuple
|
| 22 |
+
|
| 23 |
+
from normalization import content_words, normalize_for_matching, normalize_strict, phonetic_key, tokenize
|
| 24 |
+
|
| 25 |
+
INDEX_VERSION = 3
|
| 26 |
+
BM25_K1, BM25_B = 1.2, 0.75
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class BM25Index:
|
| 30 |
+
"""Okapi BM25 over a postings table ``term -> (doc ids, term frequencies)``."""
|
| 31 |
+
|
| 32 |
+
def __init__(self, postings: Dict[str, Tuple[array, array]], doc_len: array) -> None:
|
| 33 |
+
self.postings, self.doc_len = postings, doc_len
|
| 34 |
+
self.n_docs = len(doc_len)
|
| 35 |
+
self.avg_len = (sum(doc_len) / self.n_docs) if self.n_docs else 1.0
|
| 36 |
+
self.idf = {t: math.log(1 + (self.n_docs - len(d) + 0.5) / (len(d) + 0.5)) for t, (d, _) in postings.items()}
|
| 37 |
+
|
| 38 |
+
def search(self, terms: Sequence[str], top_k: int) -> List[Tuple[int, float]]:
|
| 39 |
+
scores: Dict[int, float] = defaultdict(float)
|
| 40 |
+
for term in set(terms):
|
| 41 |
+
entry = self.postings.get(term)
|
| 42 |
+
if entry is None:
|
| 43 |
+
continue
|
| 44 |
+
idf = self.idf[term]
|
| 45 |
+
for doc, tf in zip(*entry):
|
| 46 |
+
norm = 1 - BM25_B + BM25_B * self.doc_len[doc] / self.avg_len
|
| 47 |
+
scores[doc] += idf * tf * (BM25_K1 + 1) / (tf + BM25_K1 * norm)
|
| 48 |
+
return sorted(scores.items(), key=lambda item: item[1], reverse=True)[:top_k]
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _postings(documents: List[List[str]]) -> Tuple[Dict[str, Tuple[array, array]], array]:
|
| 52 |
+
docs: Dict[str, array] = defaultdict(lambda: array("I"))
|
| 53 |
+
tfs: Dict[str, array] = defaultdict(lambda: array("H"))
|
| 54 |
+
doc_len = array("I")
|
| 55 |
+
for doc_id, tokens in enumerate(documents):
|
| 56 |
+
doc_len.append(len(tokens))
|
| 57 |
+
for term, tf in Counter(tokens).items():
|
| 58 |
+
docs[term].append(doc_id)
|
| 59 |
+
tfs[term].append(min(tf, 65535))
|
| 60 |
+
return {term: (docs[term], tfs[term]) for term in docs}, doc_len
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _read_json(path: Path) -> list:
|
| 64 |
+
opener = gzip.open if path.suffix == ".gz" else open
|
| 65 |
+
with opener(path, "rt", encoding="utf-8") as handle:
|
| 66 |
+
data = json.load(handle)
|
| 67 |
+
if not isinstance(data, list):
|
| 68 |
+
raise ValueError(f"Unexpected corpus format in {path}: expected a JSON list")
|
| 69 |
+
return data
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def anchor_keys(tokens: Sequence[str]) -> List[Tuple[int, int]]:
|
| 74 |
+
"""Seed keys for unannounced-quotation search: ``(crc32, position)`` for every phonetic trigram plus two gapped
|
| 75 |
+
trigrams that survive a single substituted or inserted word."""
|
| 76 |
+
keys = [phonetic_key(t) for t in tokens]
|
| 77 |
+
out = []
|
| 78 |
+
for i in range(len(keys) - 2):
|
| 79 |
+
out.append((zlib.crc32(f"{keys[i]} {keys[i + 1]} {keys[i + 2]}".encode()), i))
|
| 80 |
+
if i + 3 < len(keys):
|
| 81 |
+
out.append((zlib.crc32(f"{keys[i]} {keys[i + 1]} _ {keys[i + 3]}".encode()), i))
|
| 82 |
+
out.append((zlib.crc32(f"{keys[i]} _ {keys[i + 2]} {keys[i + 3]}".encode()), i))
|
| 83 |
+
return out
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def build_quran_index(path: Path) -> dict:
|
| 87 |
+
records, norm, word_count = [], [], []
|
| 88 |
+
all_index: Dict[str, List[int]] = defaultdict(list)
|
| 89 |
+
by_surah: Dict[int, Dict[int, int]] = defaultdict(dict)
|
| 90 |
+
documents: List[List[str]] = []
|
| 91 |
+
anchors: Dict[int, List[Tuple[int, int]]] = defaultdict(list)
|
| 92 |
+
for entry in _read_json(Path(path)):
|
| 93 |
+
text = (entry.get("ayah_text") or "").strip()
|
| 94 |
+
if not text:
|
| 95 |
+
continue
|
| 96 |
+
idx = len(records)
|
| 97 |
+
records.append({"surah_id": entry.get("surah_id"), "surah_name": entry.get("surah_name", ""),
|
| 98 |
+
"ayah_id": entry.get("ayah_id"), "text": text})
|
| 99 |
+
by_surah[entry.get("surah_id")][entry.get("ayah_id")] = idx
|
| 100 |
+
norm.append(normalize_for_matching(text))
|
| 101 |
+
strict_tokens = tokenize(normalize_strict(text))
|
| 102 |
+
word_count.append(len(strict_tokens))
|
| 103 |
+
for word in set(strict_tokens):
|
| 104 |
+
all_index[word].append(idx)
|
| 105 |
+
documents.append(content_words(norm[-1].split()))
|
| 106 |
+
for key, pos in anchor_keys(norm[-1].split()):
|
| 107 |
+
anchors[key].append((idx, pos))
|
| 108 |
+
postings, doc_len = _postings(documents)
|
| 109 |
+
return {"version": INDEX_VERSION, "records": records, "norm": norm, "word_count": word_count,
|
| 110 |
+
"all_index": dict(all_index), "by_surah": dict(by_surah), "postings": postings, "doc_len": doc_len,
|
| 111 |
+
"anchors": dict(anchors)}
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def build_hadith_index(path: Path) -> dict:
|
| 115 |
+
records, documents = [], []
|
| 116 |
+
for entry in _read_json(Path(path)):
|
| 117 |
+
if not entry:
|
| 118 |
+
continue
|
| 119 |
+
matn = (entry.get("Matn") or "").strip() or None
|
| 120 |
+
full = (entry.get("hadithTxt") or "").strip() or None
|
| 121 |
+
if not (matn or full):
|
| 122 |
+
continue
|
| 123 |
+
records.append({"hadithID": entry.get("hadithID"), "book": entry.get("BookID"), "title": entry.get("title"),
|
| 124 |
+
"matn": matn, "full": full})
|
| 125 |
+
words = content_words(normalize_for_matching(full or matn).split())
|
| 126 |
+
if matn and full: # words of the matn that the full text may lack
|
| 127 |
+
extra = set(content_words(normalize_for_matching(matn).split())) - set(words)
|
| 128 |
+
words += sorted(extra)
|
| 129 |
+
documents.append(words)
|
| 130 |
+
postings, doc_len = _postings(documents)
|
| 131 |
+
return {"version": INDEX_VERSION, "records": records, "postings": postings, "doc_len": doc_len}
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def save_index(index: dict, path: Path) -> None:
|
| 135 |
+
path = Path(path)
|
| 136 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 137 |
+
with gzip.open(path, "wb", compresslevel=6) as handle:
|
| 138 |
+
pickle.dump(index, handle, protocol=pickle.HIGHEST_PROTOCOL)
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def load_index(path: Path) -> dict:
|
| 142 |
+
with gzip.open(path, "rb") as handle:
|
| 143 |
+
index = pickle.load(handle) # produced by save_index() from this project's own data
|
| 144 |
+
if index.get("version") != INDEX_VERSION:
|
| 145 |
+
raise ValueError("Index version mismatch; rebuild with index_builder.py")
|
| 146 |
+
return index
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def main() -> None:
|
| 150 |
+
base = Path(__file__).resolve().parent
|
| 151 |
+
out = base / "index"
|
| 152 |
+
for name, builder, source in (("quran", build_quran_index, base / "data" / "quran.json"),
|
| 153 |
+
("hadith", build_hadith_index, base / "data" / "hadith.json")):
|
| 154 |
+
started = time.time()
|
| 155 |
+
source = source if source.is_file() else source.with_name(source.name + ".gz")
|
| 156 |
+
index = builder(source)
|
| 157 |
+
save_index(index, out / f"{name}.idx.gz")
|
| 158 |
+
size = (out / f"{name}.idx.gz").stat().st_size / 1e6
|
| 159 |
+
print(f"{name}: {len(index['records'])} records -> index/{name}.idx.gz ({size:.1f} MB, {time.time() - started:.1f}s)")
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
if __name__ == "__main__":
|
| 163 |
+
sys.exit(main())
|
islamic_unified_dataset.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
islamiceval_dev_subset.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
llm_client.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Mode B: ask a language model a question, then verify its answer.
|
| 2 |
+
|
| 3 |
+
Supported providers (the user supplies their own API key; nothing is stored by this module):
|
| 4 |
+
|
| 5 |
+
* ``gemini`` Google Gemini ``generateContent`` REST endpoint
|
| 6 |
+
* ``openai`` OpenAI chat completions
|
| 7 |
+
* ``huggingface`` Hugging Face Inference Providers (OpenAI-compatible router)
|
| 8 |
+
|
| 9 |
+
Request building and response parsing are pure functions so they can be tested without network access. The same
|
| 10 |
+
request shapes are implemented in JavaScript in the static page (``build_static_space.py``).
|
| 11 |
+
"""
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import json
|
| 15 |
+
import urllib.error
|
| 16 |
+
import urllib.request
|
| 17 |
+
from dataclasses import dataclass
|
| 18 |
+
from typing import Dict, Optional, Tuple
|
| 19 |
+
|
| 20 |
+
SYSTEM_PROMPT = (
|
| 21 |
+
"أنت مساعد معرفي في العلوم الإسلامية. أجب بالعربية بإيجاز ودقة. عند الاستشهاد بآية قرآنية أو حديث نبوي اكتب نصه كاملًا "
|
| 22 |
+
"بين علامتي تنصيص مزدوجتين \"...\" بعد عبارة تمهيدية مثل: قال الله تعالى: أو قال رسول الله ﷺ:. "
|
| 23 |
+
"لا تضع بين علامات التنصيص إلا نص الآية أو الحديث، واذكر السورة ورقم الآية أو مصدر الحديث بعد الاقتباس."
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
PROVIDERS: Dict[str, Dict[str, str]] = {
|
| 27 |
+
"gemini": {"label": "Google Gemini", "model": "gemini-2.5-flash",
|
| 28 |
+
"base_url": "https://generativelanguage.googleapis.com/v1beta"},
|
| 29 |
+
"openai": {"label": "OpenAI", "model": "gpt-4o-mini", "base_url": "https://api.openai.com/v1"},
|
| 30 |
+
"huggingface": {"label": "Hugging Face", "model": "Qwen/Qwen2.5-72B-Instruct", "base_url": "https://router.huggingface.co/v1"},
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class LLMError(RuntimeError):
|
| 35 |
+
"""Raised with a user-presentable Arabic message."""
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@dataclass
|
| 39 |
+
class LLMSettings:
|
| 40 |
+
provider: str
|
| 41 |
+
api_key: str
|
| 42 |
+
model: str = ""
|
| 43 |
+
base_url: str = "" # override for tests or compatible gateways
|
| 44 |
+
timeout: float = 60.0
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def build_request(settings: LLMSettings, prompt: str) -> Tuple[str, Dict[str, str], dict]:
|
| 48 |
+
"""``(url, headers, json_body)`` for the chosen provider."""
|
| 49 |
+
if settings.provider not in PROVIDERS:
|
| 50 |
+
raise LLMError("مزوّد النموذج غير مدعوم.")
|
| 51 |
+
info = PROVIDERS[settings.provider]
|
| 52 |
+
base = (settings.base_url or info["base_url"]).rstrip("/")
|
| 53 |
+
model = settings.model.strip() or info["model"]
|
| 54 |
+
if settings.provider == "gemini":
|
| 55 |
+
url = f"{base}/models/{model}:generateContent"
|
| 56 |
+
headers = {"Content-Type": "application/json", "x-goog-api-key": settings.api_key}
|
| 57 |
+
body = {"systemInstruction": {"parts": [{"text": SYSTEM_PROMPT}]}, "contents": [{"role": "user", "parts": [{"text": prompt}]}]}
|
| 58 |
+
return url, headers, body
|
| 59 |
+
url = f"{base}/chat/completions"
|
| 60 |
+
headers = {"Content-Type": "application/json", "Authorization": f"Bearer {settings.api_key}"}
|
| 61 |
+
body = {"model": model, "messages": [{"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": prompt}]}
|
| 62 |
+
return url, headers, body
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def parse_response(provider: str, payload: dict) -> str:
|
| 66 |
+
try:
|
| 67 |
+
if provider == "gemini":
|
| 68 |
+
text = "".join(part.get("text", "") for part in payload["candidates"][0]["content"]["parts"])
|
| 69 |
+
else:
|
| 70 |
+
text = payload["choices"][0]["message"]["content"]
|
| 71 |
+
except (KeyError, IndexError, TypeError) as exc:
|
| 72 |
+
raise LLMError("وصلت استجابة غير متوقعة من النموذج.") from exc
|
| 73 |
+
if not text or not text.strip():
|
| 74 |
+
raise LLMError("لم يُرجع النموذج أي نص.")
|
| 75 |
+
return text.strip()
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
_HTTP_MESSAGES = {
|
| 79 |
+
400: "رفض المزوّد الطلب؛ تحقق من اسم النموذج.",
|
| 80 |
+
401: "مفتاح الواجهة البرمجية غير صالح.",
|
| 81 |
+
403: "المفتاح غير مصرّح له باستخدام هذا النموذج.",
|
| 82 |
+
404: "النموذج غير موجود عند المزوّد؛ تحقق من اسمه.",
|
| 83 |
+
429: "تجاوزت حد الاستخدام المسموح؛ حاول لاحقًا.",
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def generate(settings: LLMSettings, prompt: str) -> str:
|
| 88 |
+
"""Return the model's answer. Raises ``LLMError`` with an Arabic message on any failure."""
|
| 89 |
+
if not prompt or not prompt.strip():
|
| 90 |
+
raise LLMError("اكتب سؤالًا أولًا.")
|
| 91 |
+
if not settings.api_key or not settings.api_key.strip():
|
| 92 |
+
raise LLMError("أدخل مفتاح الواجهة البرمجية للمزوّد.")
|
| 93 |
+
url, headers, body = build_request(settings, prompt.strip())
|
| 94 |
+
request = urllib.request.Request(url, data=json.dumps(body).encode("utf-8"), headers=headers, method="POST")
|
| 95 |
+
try:
|
| 96 |
+
with urllib.request.urlopen(request, timeout=settings.timeout) as response:
|
| 97 |
+
payload = json.loads(response.read().decode("utf-8"))
|
| 98 |
+
except urllib.error.HTTPError as exc:
|
| 99 |
+
raise LLMError(_HTTP_MESSAGES.get(exc.code, f"فشل الطلب (الرمز {exc.code}).")) from exc
|
| 100 |
+
except (urllib.error.URLError, TimeoutError) as exc:
|
| 101 |
+
raise LLMError("تعذّر الاتصال بالمزوّد؛ تحقق من الإنترنت.") from exc
|
| 102 |
+
except json.JSONDecodeError as exc:
|
| 103 |
+
raise LLMError("وصلت استجابة غير مقروءة من المزوّد.") from exc
|
| 104 |
+
return parse_response(settings.provider, payload)
|
normalization.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Phonetic-aware Arabic normalisation shared by indexing, retrieval, alignment and verification.
|
| 2 |
+
|
| 3 |
+
Three levels, from gentle to aggressive:
|
| 4 |
+
|
| 5 |
+
* ``normalize_strict`` diacritics / tatweel / Quranic marks removed, alef forms unified (hamza on waw / ya kept).
|
| 6 |
+
* ``normalize_lenient`` strict + ta marbuta -> ha and alef maqsura -> ya (used for Hadith, whose spelling varies).
|
| 7 |
+
* ``normalize_for_matching`` the phonetic skeleton used for retrieval and word alignment: additionally folds hamza
|
| 8 |
+
carriers and keeps Arabic letters only.
|
| 9 |
+
|
| 10 |
+
All levels also reconcile the Uthmani mushaf script with Modern Standard Arabic: alef wasla (ٱ) becomes alef, the
|
| 11 |
+
dagger alef / "alif khanjariyah" (ـٰ) and tatweel are removed, Persian ya / kaf and the ligature ﷲ are mapped to Arabic
|
| 12 |
+
letters, and zero-width / bidi control characters are dropped.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import re
|
| 17 |
+
import unicodedata
|
| 18 |
+
from typing import List, Tuple
|
| 19 |
+
|
| 20 |
+
_ZERO_WIDTH = re.compile("[\u200b-\u200f\u202a-\u202e\u2066-\u2069\ufeff]")
|
| 21 |
+
_MARKS = re.compile(r"[\u0610-\u061A\u064B-\u065F\u0670\u06D6-\u06DC\u06DF-\u06E4\u06E7\u06E8\u06EA-\u06ED\u0640]")
|
| 22 |
+
_MATCH_MARKS = re.compile(r"[\u0610-\u061A\u064B-\u065F\u0670\u06D6-\u06DC\u06DF-\u06E8\u06EA-\u06ED\u0640]")
|
| 23 |
+
_PUNCTUATION = re.compile(r"[،؛؟!،.,:;'\"()\[\]{}<>«»\-_/\\|@#$%^&*+=~`﴿﴾]")
|
| 24 |
+
_SPACES = re.compile(r"\s+")
|
| 25 |
+
_ALEF_FORMS = re.compile(r"[أإآٱ]")
|
| 26 |
+
_PERSIAN = str.maketrans({"ی": "ي", "ې": "ي", "ک": "ك", "ە": "ه", "ہ": "ه", "ۀ": "ه"})
|
| 27 |
+
|
| 28 |
+
STOPWORDS = frozenset(
|
| 29 |
+
"""من في على ان أن إن الى إلى عن مع ما لا لم لن قد و ثم أو او هو هي هم انت أنتم كان كانت يكون تكون قال قالت
|
| 30 |
+
هذا هذه ذلك تلك الذي التي الذين اللاتي اللائي كل بعض غير عند بين حتى إذا اذا لو لكن بل يا أيها ايها""".split()
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _prefold(text: str) -> str:
|
| 35 |
+
"""Script-level clean-up common to every normalisation level."""
|
| 36 |
+
text = unicodedata.normalize("NFC", text).replace("ﷲ", "الله")
|
| 37 |
+
return _ZERO_WIDTH.sub("", text).translate(_PERSIAN)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def normalize_strict(text) -> str:
|
| 41 |
+
if not text:
|
| 42 |
+
return ""
|
| 43 |
+
text = _MARKS.sub("", _prefold(text))
|
| 44 |
+
text = _ALEF_FORMS.sub("ا", text)
|
| 45 |
+
return _SPACES.sub(" ", _PUNCTUATION.sub(" ", text)).strip()
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def normalize_lenient(text) -> str:
|
| 49 |
+
return normalize_strict(text).replace("ة", "ه").replace("ى", "ي")
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def normalize_for_matching(text) -> str:
|
| 53 |
+
"""Phonetic skeleton: Arabic letters only, hamza carriers / ta marbuta / alef maqsura folded."""
|
| 54 |
+
if not text:
|
| 55 |
+
return ""
|
| 56 |
+
text = _MATCH_MARKS.sub("", _prefold(text))
|
| 57 |
+
for source, target in (("أ", "ا"), ("إ", "ا"), ("آ", "ا"), ("ٱ", "ا"), ("ؤ", "و"), ("ئ", "ي"), ("ة", "ه"), ("ى", "ي")):
|
| 58 |
+
text = text.replace(source, target)
|
| 59 |
+
text = re.sub(r"[^\u0621-\u064A\s]", " ", text) # Arabic letters only: drops Arabic punctuation, digits, Latin
|
| 60 |
+
return _SPACES.sub(" ", text).strip()
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def tokenize(text: str) -> List[str]:
|
| 64 |
+
return [token for token in text.split() if token]
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def content_words(tokens) -> List[str]:
|
| 68 |
+
"""Drop stop-words and single-letter tokens."""
|
| 69 |
+
return [t for t in tokens if t not in STOPWORDS and len(t) > 1]
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def char_ngrams(text: str, n: int = 4) -> set:
|
| 73 |
+
return {text[i : i + n] for i in range(len(text) - n + 1)}
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
# ---- word-level helpers used by the aligner -----------------------------------------------------------------------
|
| 77 |
+
_ARABIC_LETTER = re.compile(r"[\u0621-\u064A]")
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def aligned_words(text: str) -> List[Tuple[str, str]]:
|
| 81 |
+
"""``[(original_word, phonetic_skeleton)]``; ayah markers like ``(12)`` and mark-only tokens are dropped."""
|
| 82 |
+
pairs = []
|
| 83 |
+
for word in text.split():
|
| 84 |
+
skeleton = normalize_for_matching(word)
|
| 85 |
+
if skeleton and _ARABIC_LETTER.search(skeleton):
|
| 86 |
+
pairs.append((word, skeleton.replace(" ", "")))
|
| 87 |
+
return pairs
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
_VOWELS = {"\u064E": "a", "\u064F": "u", "\u0650": "i", "\u064B": "A", "\u064C": "U", "\u064D": "I", "\u0651": "~"}
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def vowel_signature(word: str) -> List[Tuple[str, str]]:
|
| 94 |
+
"""Per base letter, the set of short vowels / tanween / shadda written on it (sukun and Quranic marks ignored)."""
|
| 95 |
+
word = _prefold(word).replace("ٱ", "ا")
|
| 96 |
+
signature: List[Tuple[str, str]] = []
|
| 97 |
+
for char in word:
|
| 98 |
+
if char in _VOWELS:
|
| 99 |
+
if signature:
|
| 100 |
+
letter, marks = signature[-1]
|
| 101 |
+
signature[-1] = (letter, "".join(sorted(set(marks + _VOWELS[char]))))
|
| 102 |
+
elif "\u0621" <= char <= "\u064A":
|
| 103 |
+
signature.append((char, ""))
|
| 104 |
+
return signature
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
# ---- phonetic sequence matching --------------------------------------------------------------------------------
|
| 108 |
+
# Letters that are commonly confused in writing or dictation are folded into one class, so an altered or mis-spelled
|
| 109 |
+
# quotation can still be *found*; the verifier then reports the exact differences.
|
| 110 |
+
_PHONETIC_CLASSES = {"ص": "س", "ث": "س", "ذ": "ز", "ظ": "ز", "ض": "ز", "ط": "ت", "ك": "ق", "ح": "ه", "غ": "ع"}
|
| 111 |
+
_PHONETIC_TABLE = str.maketrans(_PHONETIC_CLASSES)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def phonetic_key(skeleton_word: str) -> str:
|
| 115 |
+
"""Sound-alike key of a phonetic-skeleton word (confusable consonants folded, doubled letters collapsed)."""
|
| 116 |
+
word = skeleton_word.translate(_PHONETIC_TABLE)
|
| 117 |
+
return "".join(ch for i, ch in enumerate(word) if i == 0 or ch != word[i - 1])
|
quran.idx.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:87eb1db1dffa15d95d757b157a1569cad93e6265c15736c9f4ee62c17b2a2c98
|
| 3 |
+
size 2181000
|
quran.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
retrieval.py
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Source retrieval over the Quran and the six Hadith books: BM25 candidate search with pre-built, lazily loaded indexes.
|
| 2 |
+
|
| 3 |
+
Start-up cost is kept small by shipping the corpora as pre-tokenised, gzip-compressed pickle indexes
|
| 4 |
+
(``index/quran.idx.gz``, ``index/hadith.idx.gz``, produced by ``index_builder.py``):
|
| 5 |
+
|
| 6 |
+
* the small Quran index loads with the retriever;
|
| 7 |
+
* the large Hadith index loads on first use (or via ``warm()``);
|
| 8 |
+
* if an index file is missing it is rebuilt from ``data/*.json`` (slower, a few seconds) so the code never breaks.
|
| 9 |
+
|
| 10 |
+
Retrieval pipeline
|
| 11 |
+
Quran : word-vote F1 (coverage x precision, exact-quote friendly) + BM25 candidates
|
| 12 |
+
Hadith: BM25 recall, then character 4-gram cosine re-rank
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import logging
|
| 17 |
+
import threading
|
| 18 |
+
from collections import OrderedDict
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
from typing import Dict, Iterable, List, Optional, Sequence, Tuple
|
| 21 |
+
|
| 22 |
+
from index_builder import BM25Index, build_hadith_index, build_quran_index, load_index, save_index
|
| 23 |
+
from normalization import char_ngrams, content_words, normalize_for_matching, normalize_lenient, normalize_strict, tokenize
|
| 24 |
+
|
| 25 |
+
logger = logging.getLogger(__name__)
|
| 26 |
+
|
| 27 |
+
BASE_DIR = Path(__file__).resolve().parent
|
| 28 |
+
DATA_DIR = BASE_DIR / "data"
|
| 29 |
+
INDEX_DIR = BASE_DIR / "index"
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class CorpusError(RuntimeError):
|
| 33 |
+
"""Raised when a corpus or index file is missing or malformed."""
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class SourceRetriever:
|
| 37 |
+
"""Quran + Hadith retriever. ``quran`` is available immediately; ``hadith`` is loaded lazily."""
|
| 38 |
+
|
| 39 |
+
def __init__(self, index_dir: Path = INDEX_DIR, data_dir: Path = DATA_DIR) -> None:
|
| 40 |
+
self.index_dir, self.data_dir = Path(index_dir), Path(data_dir)
|
| 41 |
+
self._hadith_lock = threading.Lock()
|
| 42 |
+
self._hadith: Optional[dict] = None
|
| 43 |
+
self._norm_cache: "OrderedDict[Tuple[int, str], str]" = OrderedDict()
|
| 44 |
+
|
| 45 |
+
quran = self._load("quran", build_quran_index)
|
| 46 |
+
self.quran: List[dict] = quran["records"]
|
| 47 |
+
self.q_norm_match: List[str] = quran["norm"]
|
| 48 |
+
self.q_word_count: List[int] = quran["word_count"]
|
| 49 |
+
self.q_all_index: Dict[str, List[int]] = quran["all_index"]
|
| 50 |
+
self.quran_by_surah: Dict[int, Dict[int, int]] = quran["by_surah"]
|
| 51 |
+
self.quran_bm25 = BM25Index(quran["postings"], quran["doc_len"])
|
| 52 |
+
self.quran_anchors: Dict[int, List[Tuple[int, int]]] = quran["anchors"]
|
| 53 |
+
logger.info("Quran index ready: %d ayahs", len(self.quran))
|
| 54 |
+
|
| 55 |
+
# ---- loading ----------------------------------------------------------------------------------------------
|
| 56 |
+
def _load(self, name: str, builder) -> dict:
|
| 57 |
+
path = self.index_dir / f"{name}.idx.gz"
|
| 58 |
+
if path.is_file():
|
| 59 |
+
try:
|
| 60 |
+
return load_index(path)
|
| 61 |
+
except Exception:
|
| 62 |
+
logger.warning("Index %s is unreadable; rebuilding from data/", path)
|
| 63 |
+
source = self.data_dir / ("quran.json" if name == "quran" else "hadith.json")
|
| 64 |
+
gz = source.with_name(source.name + ".gz")
|
| 65 |
+
source = source if source.is_file() else gz
|
| 66 |
+
if not source.is_file():
|
| 67 |
+
raise CorpusError(f"Neither {path} nor the source corpus {source} was found")
|
| 68 |
+
index = builder(source)
|
| 69 |
+
try:
|
| 70 |
+
save_index(index, path)
|
| 71 |
+
except OSError:
|
| 72 |
+
logger.info("Could not cache %s (read-only file system); continuing in memory", path)
|
| 73 |
+
return index
|
| 74 |
+
|
| 75 |
+
def warm(self) -> None:
|
| 76 |
+
"""Load the Hadith index now (otherwise it loads on the first Hadith query)."""
|
| 77 |
+
_ = self.hadith_data
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def hadith_loaded(self) -> bool:
|
| 81 |
+
return self._hadith is not None
|
| 82 |
+
|
| 83 |
+
@property
|
| 84 |
+
def hadith_data(self) -> dict:
|
| 85 |
+
if self._hadith is None:
|
| 86 |
+
with self._hadith_lock:
|
| 87 |
+
if self._hadith is None:
|
| 88 |
+
data = self._load("hadith", build_hadith_index)
|
| 89 |
+
data["bm25"] = BM25Index(data["postings"], data["doc_len"])
|
| 90 |
+
self._hadith = data
|
| 91 |
+
logger.info("Hadith index ready: %d records", len(data["records"]))
|
| 92 |
+
return self._hadith
|
| 93 |
+
|
| 94 |
+
@property
|
| 95 |
+
def hadith(self) -> List[dict]:
|
| 96 |
+
return self.hadith_data["records"]
|
| 97 |
+
|
| 98 |
+
# ---- Quran ------------------------------------------------------------------------------------------------
|
| 99 |
+
@property
|
| 100 |
+
def quran_vocabulary(self) -> frozenset:
|
| 101 |
+
"""Phonetic-skeleton words that occur in the Quran (lets the aligner tell spelling variants from real words)."""
|
| 102 |
+
if getattr(self, "_vocab", None) is None:
|
| 103 |
+
self._vocab = frozenset(word for text in self.q_norm_match for word in text.split())
|
| 104 |
+
return self._vocab
|
| 105 |
+
|
| 106 |
+
def search_quran_ayahs(self, query: str, top_k: int = 25, extra_bm25: int = 10) -> List[dict]:
|
| 107 |
+
"""Single-ayah candidates: word-vote F1 first, then the best BM25 matches (rare-word hits for partial quotes)."""
|
| 108 |
+
query_words = tokenize(normalize_strict(query))
|
| 109 |
+
if not query_words:
|
| 110 |
+
return []
|
| 111 |
+
votes: Dict[int, int] = {}
|
| 112 |
+
for word in query_words:
|
| 113 |
+
for idx in self.q_all_index.get(word, ()):
|
| 114 |
+
votes[idx] = votes.get(idx, 0) + 1
|
| 115 |
+
scored: List[Tuple[int, float]] = []
|
| 116 |
+
for idx, vote in votes.items():
|
| 117 |
+
coverage = vote / len(query_words)
|
| 118 |
+
precision = vote / self.q_word_count[idx] if self.q_word_count[idx] else 0.0
|
| 119 |
+
f1 = 2 * coverage * precision / (coverage + precision) if coverage + precision > 0 else 0.0
|
| 120 |
+
scored.append((idx, f1))
|
| 121 |
+
scored.sort(key=lambda item: item[1], reverse=True)
|
| 122 |
+
scores = dict(scored)
|
| 123 |
+
ranked = [idx for idx, _ in scored[:top_k]]
|
| 124 |
+
seen = set(ranked)
|
| 125 |
+
bm25_hits = self.quran_bm25.search(content_words(normalize_for_matching(query).split()), extra_bm25)
|
| 126 |
+
ranked += [idx for idx, _ in bm25_hits if idx not in seen]
|
| 127 |
+
results = []
|
| 128 |
+
for idx in ranked:
|
| 129 |
+
candidate = dict(self.quran[idx])
|
| 130 |
+
candidate.update(type="Quran", retrieval_score=scores.get(idx, 0.0))
|
| 131 |
+
results.append(candidate)
|
| 132 |
+
return results
|
| 133 |
+
|
| 134 |
+
def quran_seed_ayahs(self, query_words: Sequence[str], top_k: int = 25) -> List[int]:
|
| 135 |
+
"""BM25-ranked ayah indices used to seed the multi-ayah window search."""
|
| 136 |
+
return [idx for idx, _ in self.quran_bm25.search(query_words, top_k)]
|
| 137 |
+
|
| 138 |
+
# ---- Hadith -----------------------------------------------------------------------------------------------
|
| 139 |
+
def hadith_candidates(self, query_words: Sequence[str], top_k: int) -> List[int]:
|
| 140 |
+
return [idx for idx, _ in self.hadith_data["bm25"].search(query_words, top_k)]
|
| 141 |
+
|
| 142 |
+
def hadith_norm(self, idx: int, field: str) -> Optional[str]:
|
| 143 |
+
"""Phonetic skeleton of a Hadith field (``matn`` or ``full``), computed on demand and cached."""
|
| 144 |
+
record = self.hadith[idx]
|
| 145 |
+
raw = record.get(field)
|
| 146 |
+
if not raw:
|
| 147 |
+
return None
|
| 148 |
+
key = (idx, field)
|
| 149 |
+
if key in self._norm_cache:
|
| 150 |
+
self._norm_cache.move_to_end(key)
|
| 151 |
+
return self._norm_cache[key]
|
| 152 |
+
value = normalize_for_matching(raw)
|
| 153 |
+
self._norm_cache[key] = value
|
| 154 |
+
if len(self._norm_cache) > 4096:
|
| 155 |
+
self._norm_cache.popitem(last=False)
|
| 156 |
+
return value
|
| 157 |
+
|
| 158 |
+
def search_hadith(self, query: str, top_k: int = 15, pool: int = 60) -> List[dict]:
|
| 159 |
+
"""BM25 recall, then character 4-gram cosine re-rank."""
|
| 160 |
+
words = content_words(normalize_for_matching(query).split())
|
| 161 |
+
if not words:
|
| 162 |
+
return []
|
| 163 |
+
query_grams = char_ngrams(normalize_lenient(query))
|
| 164 |
+
results = []
|
| 165 |
+
for idx in self.hadith_candidates(words, pool):
|
| 166 |
+
record = self.hadith[idx]
|
| 167 |
+
text = record["matn"] or record["full"]
|
| 168 |
+
doc_grams = char_ngrams(normalize_lenient(text))
|
| 169 |
+
cosine = (
|
| 170 |
+
len(query_grams & doc_grams) / ((len(query_grams) * len(doc_grams)) ** 0.5)
|
| 171 |
+
if query_grams and doc_grams
|
| 172 |
+
else 0.0
|
| 173 |
+
)
|
| 174 |
+
results.append(
|
| 175 |
+
{
|
| 176 |
+
"type": "Hadith",
|
| 177 |
+
"idx": idx,
|
| 178 |
+
"hadithID": record["hadithID"],
|
| 179 |
+
"book": record["book"],
|
| 180 |
+
"title": record["title"],
|
| 181 |
+
"text": text,
|
| 182 |
+
"has_matn": bool(record["matn"]),
|
| 183 |
+
"retrieval_score": cosine,
|
| 184 |
+
}
|
| 185 |
+
)
|
| 186 |
+
results.sort(key=lambda c: c["retrieval_score"], reverse=True)
|
| 187 |
+
return results[:top_k]
|
scanner.py
ADDED
|
@@ -0,0 +1,313 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Detection of unannounced and embedded quotations (no quotation marks, no introductory phrase needed).
|
| 2 |
+
|
| 3 |
+
The scanner reads the text as a stream of phonetic-skeleton words and looks for stretches that follow the Quran or a
|
| 4 |
+
Hadith closely:
|
| 5 |
+
|
| 6 |
+
1. *Seeds* a sliding window of 3 words (plus two gapped variants that survive one changed word) is looked up in the
|
| 7 |
+
pre-built phonetic n-gram index of the Quran.
|
| 8 |
+
2. *Chaining* seeds of the same ayah on a consistent diagonal are chained into a candidate region; regions of adjacent
|
| 9 |
+
ayahs that touch in the text are merged.
|
| 10 |
+
3. *Boundaries* the region grows word by word (tolerating one substituted word) while the text keeps following the
|
| 11 |
+
source, and never crosses a sentence boundary on its own; this is the contextual boundary step.
|
| 12 |
+
4. *Evidence* a region is kept only if enough of its words match and the matched words are rare enough (summed IDF), so
|
| 13 |
+
everyday phrases that merely occur in the Quran are not reported.
|
| 14 |
+
5. *Hadith* clause-sized windows are sent to BM25, and the best record is aligned word by word with the same rules.
|
| 15 |
+
|
| 16 |
+
Phonetic keys make the search tolerant to sound-alike spelling, but the verifier still reports every real difference.
|
| 17 |
+
"""
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import re
|
| 21 |
+
from collections import defaultdict
|
| 22 |
+
from dataclasses import dataclass
|
| 23 |
+
from difflib import SequenceMatcher
|
| 24 |
+
from typing import Dict, List, Optional, Sequence, Tuple
|
| 25 |
+
|
| 26 |
+
from alignment import aligned_words
|
| 27 |
+
from detector import DetectedSpan, RuleDetector, trim_span
|
| 28 |
+
from index_builder import anchor_keys
|
| 29 |
+
from normalization import content_words, normalize_for_matching, phonetic_key
|
| 30 |
+
from retrieval import SourceRetriever
|
| 31 |
+
|
| 32 |
+
_ARABIC = re.compile(r"[\u0621-\u064A]")
|
| 33 |
+
_STRONG_BOUNDARY = re.compile(r"[.؟?!؛;:\n…]")
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
@dataclass
|
| 37 |
+
class Token:
|
| 38 |
+
skeleton: str
|
| 39 |
+
key: str
|
| 40 |
+
start: int
|
| 41 |
+
end: int
|
| 42 |
+
boundary_after: bool # a sentence-level punctuation mark or line break follows this word
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def tokenize_with_offsets(text: str) -> List[Token]:
|
| 46 |
+
matches = list(re.finditer(r"\S+", text))
|
| 47 |
+
tokens: List[Token] = []
|
| 48 |
+
for i, match in enumerate(matches):
|
| 49 |
+
skeleton = normalize_for_matching(match.group()).replace(" ", "")
|
| 50 |
+
if not skeleton or not _ARABIC.search(skeleton):
|
| 51 |
+
if tokens and _STRONG_BOUNDARY.search(match.group()):
|
| 52 |
+
tokens[-1].boundary_after = True
|
| 53 |
+
continue
|
| 54 |
+
gap_end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
|
| 55 |
+
trailing = text[match.end():gap_end] + match.group()[-2:]
|
| 56 |
+
tokens.append(Token(skeleton, phonetic_key(skeleton), match.start(), match.end(),
|
| 57 |
+
bool(_STRONG_BOUNDARY.search(trailing))))
|
| 58 |
+
return tokens
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
@dataclass
|
| 62 |
+
class _Region:
|
| 63 |
+
start: int # token index in the run (inclusive)
|
| 64 |
+
end: int # exclusive
|
| 65 |
+
label: str
|
| 66 |
+
matched: int
|
| 67 |
+
ratio: float
|
| 68 |
+
idf: float
|
| 69 |
+
surah: Optional[int] = None
|
| 70 |
+
first_ayah: Optional[int] = None
|
| 71 |
+
last_ayah: Optional[int] = None
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
class CorpusScanner:
|
| 75 |
+
"""Finds Quran / Hadith stretches in free text. Thresholds are deliberately conservative."""
|
| 76 |
+
|
| 77 |
+
def __init__(self, retriever: SourceRetriever, min_tokens: int = 4, min_ratio: float = 0.7, min_idf: float = 8.0,
|
| 78 |
+
hadith_min_tokens: int = 6, hadith_min_idf: float = 14.0, scan_hadith: bool = True) -> None:
|
| 79 |
+
self.kb = retriever
|
| 80 |
+
self.min_tokens, self.min_ratio, self.min_idf = min_tokens, min_ratio, min_idf
|
| 81 |
+
self.hadith_min_tokens, self.hadith_min_idf = hadith_min_tokens, hadith_min_idf
|
| 82 |
+
self.scan_hadith = scan_hadith
|
| 83 |
+
|
| 84 |
+
# ---- public -----------------------------------------------------------------------------------------------
|
| 85 |
+
def scan(self, text: str, exclude: Sequence[Tuple[int, int]] = ()) -> List[DetectedSpan]:
|
| 86 |
+
tokens = tokenize_with_offsets(text)
|
| 87 |
+
runs = self._runs(tokens, exclude)
|
| 88 |
+
spans: List[DetectedSpan] = []
|
| 89 |
+
for run in runs:
|
| 90 |
+
for region in self._merge(self._scan_quran(run)):
|
| 91 |
+
spans.append(self._to_span(text, run, region))
|
| 92 |
+
if self.scan_hadith:
|
| 93 |
+
for run in runs:
|
| 94 |
+
taken = [(s.start, s.end) for s in spans]
|
| 95 |
+
for region in self._scan_hadith(run, taken):
|
| 96 |
+
spans.append(self._to_span(text, run, region))
|
| 97 |
+
return sorted(spans, key=lambda s: s.start)
|
| 98 |
+
|
| 99 |
+
# ---- helpers ----------------------------------------------------------------------------------------------
|
| 100 |
+
@staticmethod
|
| 101 |
+
def _runs(tokens: List[Token], exclude: Sequence[Tuple[int, int]]) -> List[List[Token]]:
|
| 102 |
+
runs, current = [], []
|
| 103 |
+
for token in tokens:
|
| 104 |
+
if any(token.start < e and token.end > s for s, e in exclude):
|
| 105 |
+
if current:
|
| 106 |
+
runs.append(current)
|
| 107 |
+
current = []
|
| 108 |
+
else:
|
| 109 |
+
current.append(token)
|
| 110 |
+
if current:
|
| 111 |
+
runs.append(current)
|
| 112 |
+
return runs
|
| 113 |
+
|
| 114 |
+
@staticmethod
|
| 115 |
+
def _to_span(text: str, run: List[Token], region: _Region) -> DetectedSpan:
|
| 116 |
+
start, end = trim_span(text, run[region.start].start, run[region.end - 1].end)
|
| 117 |
+
return DetectedSpan(start, end, region.label, round(region.ratio, 3), "scan", text[start:end])
|
| 118 |
+
|
| 119 |
+
# ---- Quran ------------------------------------------------------------------------------------------------
|
| 120 |
+
def _scan_quran(self, run: List[Token]) -> List[_Region]:
|
| 121 |
+
kb = self.kb
|
| 122 |
+
if len(run) < self.min_tokens:
|
| 123 |
+
return []
|
| 124 |
+
hits: Dict[int, List[Tuple[int, int]]] = defaultdict(list)
|
| 125 |
+
for key_hash, pos in anchor_keys([t.skeleton for t in run]):
|
| 126 |
+
postings = kb.quran_anchors.get(key_hash)
|
| 127 |
+
if not postings or len(postings) > 60: # skip phrases that occur everywhere
|
| 128 |
+
continue
|
| 129 |
+
for ayah, source_pos in postings:
|
| 130 |
+
hits[ayah].append((pos, source_pos))
|
| 131 |
+
|
| 132 |
+
regions: List[_Region] = []
|
| 133 |
+
for ayah, found in hits.items():
|
| 134 |
+
found.sort()
|
| 135 |
+
chains: List[dict] = []
|
| 136 |
+
for pos, source_pos in found:
|
| 137 |
+
diagonal = source_pos - pos
|
| 138 |
+
for chain in chains:
|
| 139 |
+
if abs(diagonal - chain["d"]) <= 2 and pos - chain["last"] <= 5:
|
| 140 |
+
chain["hits"].append((pos, source_pos))
|
| 141 |
+
chain["last"], chain["d"] = pos, diagonal
|
| 142 |
+
break
|
| 143 |
+
else:
|
| 144 |
+
chains.append({"d": diagonal, "last": pos, "hits": [(pos, source_pos)]})
|
| 145 |
+
for chain in chains:
|
| 146 |
+
region = self._grow(run, ayah, chain)
|
| 147 |
+
if region is not None:
|
| 148 |
+
regions.append(region)
|
| 149 |
+
return self._non_overlapping(regions)
|
| 150 |
+
|
| 151 |
+
def _grow(self, run: List[Token], ayah: int, chain: dict) -> Optional[_Region]:
|
| 152 |
+
kb = self.kb
|
| 153 |
+
source = kb.q_norm_match[ayah].split()
|
| 154 |
+
source_keys = [phonetic_key(w) for w in source]
|
| 155 |
+
diagonal = sorted(source_pos - pos for pos, source_pos in chain["hits"])[len(chain["hits"]) // 2]
|
| 156 |
+
start = min(pos for pos, _ in chain["hits"])
|
| 157 |
+
end = min(len(run), max(pos for pos, _ in chain["hits"]) + 3)
|
| 158 |
+
|
| 159 |
+
def key_at(i: int) -> Optional[str]:
|
| 160 |
+
return source_keys[i + diagonal] if 0 <= i + diagonal < len(source_keys) else None
|
| 161 |
+
|
| 162 |
+
while start > 0 and not run[start - 1].boundary_after: # grow left while the text keeps following the source
|
| 163 |
+
if run[start - 1].key == key_at(start - 1):
|
| 164 |
+
start -= 1
|
| 165 |
+
elif start >= 2 and not run[start - 2].boundary_after and run[start - 2].key == key_at(start - 2):
|
| 166 |
+
start -= 2 # one substituted word
|
| 167 |
+
else:
|
| 168 |
+
break
|
| 169 |
+
while end < len(run) and not run[end - 1].boundary_after:
|
| 170 |
+
if run[end].key == key_at(end):
|
| 171 |
+
end += 1
|
| 172 |
+
elif end + 1 < len(run) and run[end + 1].key == key_at(end + 1):
|
| 173 |
+
end += 2
|
| 174 |
+
else:
|
| 175 |
+
break
|
| 176 |
+
|
| 177 |
+
start = self._soft_left(run, start, diagonal, source_keys)
|
| 178 |
+
end = self._soft_right(run, end, diagonal, source_keys)
|
| 179 |
+
positions = range(start, end)
|
| 180 |
+
matched_idx = [i for i in positions if run[i].key == key_at(i)]
|
| 181 |
+
matched = len(matched_idx)
|
| 182 |
+
n = end - start
|
| 183 |
+
idf = sum(kb.quran_bm25.idf.get(run[i].skeleton, 0.0) for i in matched_idx)
|
| 184 |
+
ratio = matched / n if n else 0.0
|
| 185 |
+
needed_idf = self.min_idf if n >= 5 else self.min_idf + 6.0 # very short stretches must be rare phrases
|
| 186 |
+
if n < self.min_tokens or matched < self.min_tokens or ratio < self.min_ratio or idf < needed_idf:
|
| 187 |
+
return None
|
| 188 |
+
record = kb.quran[ayah]
|
| 189 |
+
return _Region(start, end, "Ayah", matched, ratio, idf, record["surah_id"], record["ayah_id"], record["ayah_id"])
|
| 190 |
+
|
| 191 |
+
@staticmethod
|
| 192 |
+
def _closest(run: List[Token], candidates, target: str):
|
| 193 |
+
"""Among candidate token ranges, the one whose joined phonetic key best resembles ``target``; near-ties prefer the
|
| 194 |
+
longer range (words next to a changed word usually belong to the same altered quotation)."""
|
| 195 |
+
scored = [(SequenceMatcher(None, "".join(t.key for t in run[a:b]), target).ratio(), a, b) for a, b in candidates]
|
| 196 |
+
if not scored:
|
| 197 |
+
return None
|
| 198 |
+
top = max(score for score, _, _ in scored)
|
| 199 |
+
if top < 0.6:
|
| 200 |
+
return None
|
| 201 |
+
return max((c for c in scored if c[0] >= top - 0.2), key=lambda c: c[2] - c[1])
|
| 202 |
+
|
| 203 |
+
def _soft_left(self, run: List[Token], start: int, diagonal: int, source_keys: List[str]) -> int:
|
| 204 |
+
"""Pull in up to two words before the region that look like the source words missing at its beginning."""
|
| 205 |
+
missing = min(start + diagonal, 2)
|
| 206 |
+
if missing <= 0:
|
| 207 |
+
return start
|
| 208 |
+
target = "".join(source_keys[start + diagonal - missing : start + diagonal])
|
| 209 |
+
candidates = [(start - n, start) for n in range(max(1, missing - 1), missing + 2)
|
| 210 |
+
if start - n >= 0 and not any(run[i].boundary_after for i in range(start - n, start))]
|
| 211 |
+
best = self._closest(run, candidates, target)
|
| 212 |
+
return best[1] if best else start
|
| 213 |
+
|
| 214 |
+
def _soft_right(self, run: List[Token], end: int, diagonal: int, source_keys: List[str]) -> int:
|
| 215 |
+
missing = min(len(source_keys) - (end + diagonal), 2)
|
| 216 |
+
if missing <= 0 or end >= len(run) or run[end - 1].boundary_after:
|
| 217 |
+
return end
|
| 218 |
+
target = "".join(source_keys[end + diagonal : end + diagonal + missing])
|
| 219 |
+
candidates = [(end, end + n) for n in range(max(1, missing - 1), missing + 2)
|
| 220 |
+
if end + n <= len(run) and not any(run[i].boundary_after for i in range(end, end + n - 1))]
|
| 221 |
+
best = self._closest(run, candidates, target)
|
| 222 |
+
return best[2] if best else end
|
| 223 |
+
|
| 224 |
+
@staticmethod
|
| 225 |
+
def _non_overlapping(regions: List[_Region]) -> List[_Region]:
|
| 226 |
+
chosen: List[_Region] = []
|
| 227 |
+
for region in sorted(regions, key=lambda r: (r.matched, r.ratio), reverse=True):
|
| 228 |
+
if all(region.end <= c.start or region.start >= c.end for c in chosen):
|
| 229 |
+
chosen.append(region)
|
| 230 |
+
return sorted(chosen, key=lambda r: r.start)
|
| 231 |
+
|
| 232 |
+
@staticmethod
|
| 233 |
+
def _merge(regions: List[_Region]) -> List[_Region]:
|
| 234 |
+
"""Join regions of consecutive ayahs that follow each other in the text (a quotation spanning several ayahs)."""
|
| 235 |
+
merged: List[_Region] = []
|
| 236 |
+
for region in regions:
|
| 237 |
+
last = merged[-1] if merged else None
|
| 238 |
+
if (last and last.surah == region.surah and region.first_ayah == last.last_ayah + 1
|
| 239 |
+
and region.start - last.end <= 1):
|
| 240 |
+
last.end, last.matched = region.end, last.matched + region.matched
|
| 241 |
+
last.ratio = last.matched / (last.end - last.start)
|
| 242 |
+
last.idf += region.idf
|
| 243 |
+
last.last_ayah = region.last_ayah
|
| 244 |
+
else:
|
| 245 |
+
merged.append(region)
|
| 246 |
+
return merged
|
| 247 |
+
|
| 248 |
+
# ---- Hadith -----------------------------------------------------------------------------------------------
|
| 249 |
+
def _scan_hadith(self, run: List[Token], taken: Sequence[Tuple[int, int]]) -> List[_Region]:
|
| 250 |
+
kb = self.kb
|
| 251 |
+
regions: List[_Region] = []
|
| 252 |
+
segment_start = 0
|
| 253 |
+
for i, token in enumerate(run):
|
| 254 |
+
if token.boundary_after or i == len(run) - 1:
|
| 255 |
+
segment = (segment_start, i + 1)
|
| 256 |
+
segment_start = i + 1
|
| 257 |
+
if segment[1] - segment[0] < self.hadith_min_tokens:
|
| 258 |
+
continue
|
| 259 |
+
for a, b in self._windows(*segment):
|
| 260 |
+
region = self._match_hadith(run, a, b)
|
| 261 |
+
if region is not None:
|
| 262 |
+
regions.append(region)
|
| 263 |
+
return self._non_overlapping(regions)
|
| 264 |
+
|
| 265 |
+
@staticmethod
|
| 266 |
+
def _windows(start: int, end: int, size: int = 24, stride: int = 12):
|
| 267 |
+
if end - start <= 40:
|
| 268 |
+
yield start, end
|
| 269 |
+
else:
|
| 270 |
+
for a in range(start, end - 6, stride):
|
| 271 |
+
yield a, min(end, a + size)
|
| 272 |
+
|
| 273 |
+
def _match_hadith(self, run: List[Token], a: int, b: int) -> Optional[_Region]:
|
| 274 |
+
kb = self.kb
|
| 275 |
+
window = run[a:b]
|
| 276 |
+
words = content_words(t.skeleton for t in window)
|
| 277 |
+
if len(words) < 4:
|
| 278 |
+
return None
|
| 279 |
+
window_keys = [t.key for t in window]
|
| 280 |
+
best = None
|
| 281 |
+
for idx in kb.hadith_candidates(words, 3):
|
| 282 |
+
record = kb.hadith[idx]
|
| 283 |
+
source = [p[1] for p in aligned_words(record["matn"] or record["full"])]
|
| 284 |
+
matcher = SequenceMatcher(None, window_keys, [phonetic_key(w) for w in source], autojunk=False)
|
| 285 |
+
blocks = [blk for blk in matcher.get_matching_blocks() if blk.size >= 3]
|
| 286 |
+
matched = sum(blk.size for blk in blocks)
|
| 287 |
+
if matched >= self.hadith_min_tokens and (best is None or matched > best[0]):
|
| 288 |
+
best = (matched, blocks)
|
| 289 |
+
if best is None:
|
| 290 |
+
return None
|
| 291 |
+
matched, blocks = best
|
| 292 |
+
start, end = blocks[0].a, blocks[-1].a + blocks[-1].size
|
| 293 |
+
ratio = matched / (end - start)
|
| 294 |
+
idf = sum(kb.hadith_data["bm25"].idf.get(window[i].skeleton, 0.0) for blk in blocks for i in range(blk.a, blk.a + blk.size))
|
| 295 |
+
if ratio < self.min_ratio or idf < self.hadith_min_idf:
|
| 296 |
+
return None
|
| 297 |
+
return _Region(a + start, a + end, "Hadith", matched, ratio, idf)
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
class HybridDetector:
|
| 301 |
+
"""Rule-based detection first (quotation marks + corpus lookup; introductory phrases are optional), then the corpus scanner on the rest of the
|
| 302 |
+
text to find unannounced and embedded quotations. Rule spans always win overlaps: an author-delimited quotation is
|
| 303 |
+
verified exactly as written."""
|
| 304 |
+
|
| 305 |
+
def __init__(self, retriever: SourceRetriever, use_scanner: bool = True, rules_use_corpus: bool = True) -> None:
|
| 306 |
+
self.rules = RuleDetector(retriever if rules_use_corpus else None)
|
| 307 |
+
self.scanner = CorpusScanner(retriever) if use_scanner else None
|
| 308 |
+
|
| 309 |
+
def detect(self, text: str) -> List[DetectedSpan]:
|
| 310 |
+
spans = self.rules.detect(text)
|
| 311 |
+
if self.scanner is not None:
|
| 312 |
+
spans += self.scanner.scan(text, [(s.start, s.end) for s in spans])
|
| 313 |
+
return sorted(spans, key=lambda s: s.start)
|
similarity.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Similarity indicators between a quotation and a candidate source (the signals behind the verification score)."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import re
|
| 5 |
+
import unicodedata
|
| 6 |
+
from typing import Dict, List
|
| 7 |
+
|
| 8 |
+
from alignment import best_region, lcs_length
|
| 9 |
+
from normalization import normalize_lenient, normalize_strict, tokenize
|
| 10 |
+
|
| 11 |
+
try: # optional accelerator; identical formula (1 - distance / max_len)
|
| 12 |
+
from rapidfuzz.distance import Levenshtein as _RapidLevenshtein
|
| 13 |
+
except ImportError: # pragma: no cover
|
| 14 |
+
_RapidLevenshtein = None
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _edit_similarity(a: str, b: str, max_len: int = 600) -> float:
|
| 18 |
+
"""Normalised Levenshtein similarity in [0, 1] on the first ``max_len`` characters."""
|
| 19 |
+
a, b = a[:max_len], b[:max_len]
|
| 20 |
+
if a == b:
|
| 21 |
+
return 1.0
|
| 22 |
+
if not a or not b:
|
| 23 |
+
return 0.0
|
| 24 |
+
if _RapidLevenshtein is not None:
|
| 25 |
+
return float(_RapidLevenshtein.normalized_similarity(a, b))
|
| 26 |
+
prev = list(range(len(b) + 1))
|
| 27 |
+
for i, ca in enumerate(a):
|
| 28 |
+
curr = [i + 1]
|
| 29 |
+
for j, cb in enumerate(b):
|
| 30 |
+
curr.append(min(curr[j] + 1, prev[j + 1] + 1, prev[j] + (ca != cb)))
|
| 31 |
+
prev = curr
|
| 32 |
+
return 1.0 - prev[-1] / max(len(a), len(b))
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _light_normalize(text: str) -> str:
|
| 36 |
+
"""Keeps diacritics (so diacritic changes lower the score) but drops punctuation and tatweel."""
|
| 37 |
+
text = unicodedata.normalize("NFC", text)
|
| 38 |
+
text = re.sub(r"[،؛؟!.,:;'\"()\[\]{}<>«»\-_/\\|@#$%^&*+=~`\u0640]", " ", text)
|
| 39 |
+
return re.sub(r"\s+", " ", text).strip()
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def compute_signals(claim: str, candidate: str, content_type: str = "Ayah", local: bool = True) -> Dict[str, float]:
|
| 43 |
+
"""Word-, character- and sequence-level similarity indicators between a quotation and a candidate source."""
|
| 44 |
+
is_quran = content_type == "Ayah"
|
| 45 |
+
full_candidate = candidate
|
| 46 |
+
if local: # score the snippet against the region it refers to, not against the whole verse / Hadith
|
| 47 |
+
candidate = best_region(claim, candidate)
|
| 48 |
+
normalize = normalize_strict if is_quran else normalize_lenient
|
| 49 |
+
norm_claim, norm_cand = normalize(claim), normalize(candidate)
|
| 50 |
+
|
| 51 |
+
claim_set, cand_set = set(tokenize(norm_claim)), set(tokenize(norm_cand))
|
| 52 |
+
if claim_set and cand_set:
|
| 53 |
+
shared = claim_set & cand_set
|
| 54 |
+
token_overlap = len(shared) / len(claim_set | cand_set)
|
| 55 |
+
coverage = len(shared) / len(claim_set)
|
| 56 |
+
else:
|
| 57 |
+
token_overlap = coverage = 0.0
|
| 58 |
+
|
| 59 |
+
claim_tokens, cand_tokens = tokenize(norm_claim), tokenize(norm_cand)
|
| 60 |
+
lcs_ratio = lcs_length(claim_tokens, cand_tokens) / len(claim_tokens) if claim_tokens else 0.0
|
| 61 |
+
edit_sim = _edit_similarity(norm_claim, norm_cand)
|
| 62 |
+
|
| 63 |
+
claim_chars, cand_chars = set(norm_claim.replace(" ", "")), set(norm_cand.replace(" ", ""))
|
| 64 |
+
char_overlap = len(claim_chars & cand_chars) / len(claim_chars | cand_chars) if (claim_chars or cand_chars) else 0.0
|
| 65 |
+
|
| 66 |
+
claim_flat, cand_flat = norm_claim.replace(" ", ""), norm_cand.replace(" ", "")
|
| 67 |
+
is_substring = int(bool(claim_flat) and bool(cand_flat) and (claim_flat in cand_flat or cand_flat in claim_flat))
|
| 68 |
+
|
| 69 |
+
diacritic_sim = (
|
| 70 |
+
_edit_similarity(_light_normalize(claim), _light_normalize(candidate), max_len=800) if is_quran else edit_sim
|
| 71 |
+
)
|
| 72 |
+
short = len(claim_tokens) < 4
|
| 73 |
+
|
| 74 |
+
if is_quran:
|
| 75 |
+
w = (
|
| 76 |
+
dict(coverage=0.35, diacritic_sim=0.30, lcs_ratio=0.15, token_overlap=0.10, char_overlap=0.05, edit_sim=0.05)
|
| 77 |
+
if short
|
| 78 |
+
else dict(coverage=0.25, diacritic_sim=0.30, lcs_ratio=0.20, token_overlap=0.10, char_overlap=0.05, edit_sim=0.10)
|
| 79 |
+
)
|
| 80 |
+
composite = (
|
| 81 |
+
w["coverage"] * coverage + w["diacritic_sim"] * diacritic_sim + w["lcs_ratio"] * lcs_ratio
|
| 82 |
+
+ w["token_overlap"] * token_overlap + w["char_overlap"] * char_overlap + w["edit_sim"] * edit_sim
|
| 83 |
+
)
|
| 84 |
+
if is_substring and diacritic_sim >= 0.60:
|
| 85 |
+
composite = max(composite, 0.88)
|
| 86 |
+
elif is_substring:
|
| 87 |
+
composite = max(composite, 0.75)
|
| 88 |
+
else:
|
| 89 |
+
w = (
|
| 90 |
+
dict(coverage=0.40, lcs_ratio=0.20, token_overlap=0.20, char_overlap=0.10, edit_sim=0.10)
|
| 91 |
+
if short
|
| 92 |
+
else dict(coverage=0.30, lcs_ratio=0.28, token_overlap=0.18, char_overlap=0.12, edit_sim=0.12)
|
| 93 |
+
)
|
| 94 |
+
composite = (
|
| 95 |
+
w["coverage"] * coverage + w["lcs_ratio"] * lcs_ratio + w["token_overlap"] * token_overlap
|
| 96 |
+
+ w["char_overlap"] * char_overlap + w["edit_sim"] * edit_sim
|
| 97 |
+
)
|
| 98 |
+
if is_substring:
|
| 99 |
+
composite = max(composite, 0.82)
|
| 100 |
+
|
| 101 |
+
return {
|
| 102 |
+
"token_overlap": round(token_overlap, 4),
|
| 103 |
+
"coverage": round(coverage, 4),
|
| 104 |
+
"lcs_ratio": round(lcs_ratio, 4),
|
| 105 |
+
"edit_sim": round(edit_sim, 4),
|
| 106 |
+
"diacritic_sim": round(diacritic_sim, 4),
|
| 107 |
+
"char_overlap": round(char_overlap, 4),
|
| 108 |
+
"is_substring": is_substring,
|
| 109 |
+
"source_fraction": round(min(1.0, len(claim_tokens) / max(len(tokenize(normalize(full_candidate))), 1)), 4),
|
| 110 |
+
"composite": round(composite, 4),
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def best_match_score(claim: str, candidates: List[dict], content_type: str = "Ayah", local: bool = True):
|
| 115 |
+
"""Return ``(score, candidate_with_signals)`` for the best-scoring candidate."""
|
| 116 |
+
best_score, best_candidate = 0.0, None
|
| 117 |
+
for candidate in candidates:
|
| 118 |
+
signals = compute_signals(claim, candidate.get("text", ""), content_type, local)
|
| 119 |
+
if signals["composite"] > best_score:
|
| 120 |
+
best_score, best_candidate = signals["composite"], {**candidate, "signals": signals}
|
| 121 |
+
return best_score, best_candidate
|
styles.css
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
@import url('https://fonts.googleapis.com/css2?family=Cairo:wght@400;600;700&family=Amiri:wght@400;700&display=swap');
|
| 3 |
+
:root { --green:#0F4C3A; --green-2:#17694F; --gold:#B8912F; --gold-2:#E2C06E; --cream:#FBF6EA; --paper:#FFFFFF;
|
| 4 |
+
--ink:#1F2933; --muted:#55626D; --line:#E6DCC3; --ok:#1B7A4B; --ok-bg:#EAF6EF; --ok-line:#BFE3CD;
|
| 5 |
+
--bad:#B3261E; --bad-bg:#FDECEA; --bad-line:#F4B8B3; --warn:#7A5B0C; --warn-bg:#FFF6DA; --warn-line:#EBD28A; }
|
| 6 |
+
.gradio-container, body.icv-page { background:var(--cream) url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='88' height='88' viewBox='0 0 88 88'%3E%3Cg fill='none' stroke='%230F4C3A' stroke-opacity='0.07' stroke-width='1'%3E%3Crect x='22' y='22' width='44' height='44'/%3E%3Crect x='22' y='22' width='44' height='44' transform='rotate(45 44 44)'/%3E%3C/g%3E%3C/svg%3E") !important; font-family:'Cairo','Segoe UI',Tahoma,sans-serif; color:var(--ink); }
|
| 7 |
+
.gradio-container { max-width:1060px !important; --body-text-color:#1F2933; --block-background-fill:#FFFFFF; --block-border-color:#E6DCC3;
|
| 8 |
+
--input-background-fill:#FFFFFF; --block-label-text-color:#55626D; --button-primary-background-fill:#0F4C3A;
|
| 9 |
+
--button-primary-background-fill-hover:#17694F; --button-primary-text-color:#FFFFFF; --button-secondary-background-fill:#FFFFFF;
|
| 10 |
+
--button-secondary-text-color:#0F4C3A; --button-secondary-border-color:#B8912F; --color-accent:#B8912F; }
|
| 11 |
+
.icv { direction:rtl; text-align:right; color:var(--ink); line-height:1.9; }
|
| 12 |
+
.icv .hero { background:linear-gradient(135deg,#0F4C3A,#17694F); border-radius:20px; padding:30px 22px 26px; margin:10px 0 18px; text-align:center;
|
| 13 |
+
box-shadow:0 6px 22px rgba(15,76,58,.18); border-bottom:4px solid var(--gold); }
|
| 14 |
+
.icv .hero .mark { width:54px; height:54px; display:block; margin:0 auto 4px; }
|
| 15 |
+
.icv .hero h1 { font-size:2.3rem; margin:.1rem 0; color:#FFFFFF; font-weight:700; }
|
| 16 |
+
.icv .hero .tagline { color:var(--gold-2); font-size:1.2rem; font-weight:600; margin:.1rem 0 .6rem; }
|
| 17 |
+
.icv .hero .sub { max-width:720px; margin:0 auto; color:#E9F2EE; font-size:1rem; }
|
| 18 |
+
.icv .flow { display:flex; flex-wrap:wrap; justify-content:center; align-items:center; gap:8px; margin-top:16px; }
|
| 19 |
+
.icv .flow span { background:rgba(255,255,255,.12); border:1px solid rgba(226,192,110,.55); color:#FFF; border-radius:999px; padding:3px 16px; font-size:.9rem; }
|
| 20 |
+
.icv .flow i { color:var(--gold-2); font-style:normal; font-size:1.2rem; }
|
| 21 |
+
.icv .disclaimer { font-size:.85rem; color:var(--muted); text-align:center; padding:14px 8px; }
|
| 22 |
+
.icv .en-free { direction:rtl; }
|
| 23 |
+
.input-area textarea { direction:rtl; text-align:right; font-family:'Amiri','Cairo',serif !important; font-size:1.25rem !important; line-height:2.1 !important; background:#FFFFFF !important; color:#1F2933 !important; }
|
| 24 |
+
.llm-row label, .gradio-container label span { font-family:'Cairo',sans-serif; }
|
| 25 |
+
.results { direction:rtl; }
|
| 26 |
+
.icv .summary { display:grid; grid-template-columns:repeat(auto-fit,minmax(130px,1fr)); gap:10px; margin:8px 0 14px; }
|
| 27 |
+
.icv .tile { background:var(--paper); border:1px solid var(--line); border-radius:14px; padding:12px 8px; text-align:center; }
|
| 28 |
+
.icv .tile b { display:block; font-size:1.8rem; color:var(--green); line-height:1.3; } .icv .tile span { font-size:.92rem; color:var(--muted); }
|
| 29 |
+
.icv .tile.verified { background:var(--ok-bg); border-color:var(--ok-line); } .icv .tile.verified b { color:var(--ok); }
|
| 30 |
+
.icv .tile.mismatch { background:var(--bad-bg); border-color:var(--bad-line); } .icv .tile.mismatch b { color:var(--bad); }
|
| 31 |
+
.icv .tile.review { background:var(--warn-bg); border-color:var(--warn-line); } .icv .tile.review b { color:var(--warn); }
|
| 32 |
+
.icv .tile.zero { background:var(--paper); border-color:var(--line); opacity:.6; } .icv .tile.zero b { color:var(--muted); }
|
| 33 |
+
.icv .highlight, .icv .generated, .icv .final { background:var(--paper); border:1px solid var(--line); border-radius:14px; padding:14px 18px; margin-bottom:14px; }
|
| 34 |
+
.icv label { display:block; color:var(--muted); font-size:.82rem; font-weight:600; margin-bottom:4px; }
|
| 35 |
+
.icv .highlight p, .icv .generated p, .icv .final-text { font-family:'Amiri','Cairo',serif; font-size:1.2rem; line-height:2.2; margin:0; white-space:pre-wrap; }
|
| 36 |
+
.icv mark { color:var(--ink); border-radius:6px; padding:1px 5px; }
|
| 37 |
+
.icv mark.verified { background:#D5EEDF; } .icv mark.mismatch { background:#F8CFCB; } .icv mark.review { background:#F7E3A6; }
|
| 38 |
+
.icv .hl-legend { display:flex; flex-wrap:wrap; gap:8px; margin-top:10px; font-size:.78rem; }
|
| 39 |
+
.icv .qcard { background:var(--paper); border:1px solid var(--line); border-inline-start:6px solid var(--line); border-radius:16px; padding:16px 18px; margin:12px 0; }
|
| 40 |
+
.icv .qcard.verified { border-inline-start-color:var(--ok); } .icv .qcard.mismatch { border-inline-start-color:var(--bad); } .icv .qcard.review { border-inline-start-color:var(--gold); }
|
| 41 |
+
.icv .qhead { display:flex; flex-wrap:wrap; gap:8px; align-items:center; margin-bottom:8px; }
|
| 42 |
+
.icv .idx { background:var(--green); color:#FFF; border-radius:50%; width:28px; height:28px; display:inline-flex; align-items:center; justify-content:center; font-weight:700; font-size:.9rem; }
|
| 43 |
+
.icv .badge { padding:2px 14px; border-radius:999px; font-size:.84rem; background:#F3EEDD; border:1px solid var(--line); color:var(--ink); }
|
| 44 |
+
.icv .badge.st.verified { background:var(--ok-bg); border-color:var(--ok-line); color:var(--ok); font-weight:600; }
|
| 45 |
+
.icv .badge.st.mismatch { background:var(--bad-bg); border-color:var(--bad-line); color:var(--bad); font-weight:600; }
|
| 46 |
+
.icv .badge.st.review { background:var(--warn-bg); border-color:var(--warn-line); color:var(--warn); font-weight:600; }
|
| 47 |
+
.icv .badge.scan { background:#EEF3FB; border-color:#C9D8EE; color:#2F4F7F; }
|
| 48 |
+
.icv .conf { margin-inline-start:auto; color:var(--muted); font-size:.88rem; } .icv .conf b { color:var(--green); }
|
| 49 |
+
.icv .field { margin:8px 0; } .icv .field p { margin:0; }
|
| 50 |
+
.icv .quote { font-family:'Amiri','Cairo',serif; font-size:1.25rem; line-height:2.2; color:var(--ink); } .icv .quote.small { font-size:1.05rem; color:#3a4651; }
|
| 51 |
+
.icv .diff { font-family:'Amiri','Cairo',serif; font-size:1.2rem; line-height:2.2; }
|
| 52 |
+
.icv .w-extra { background:var(--bad-bg); color:var(--bad); border-radius:4px; padding:0 4px; text-decoration:line-through; }
|
| 53 |
+
.icv .w-missing { background:var(--ok-bg); color:var(--ok); border-radius:4px; padding:0 4px; font-weight:700; }
|
| 54 |
+
.icv .legend { color:var(--muted); font-size:.82rem; margin:4px 0 0; } .icv .legend span { text-decoration:none; font-size:.8rem; }
|
| 55 |
+
.icv .reason { color:var(--muted); font-size:.92rem; margin:6px 0 0; }
|
| 56 |
+
.icv .note { color:var(--warn); background:var(--warn-bg); border:1px solid var(--warn-line); border-radius:10px; padding:5px 12px; font-size:.88rem; margin:8px 0 0; }
|
| 57 |
+
.icv .action { border-radius:12px; padding:10px 14px; margin-top:10px; }
|
| 58 |
+
.icv .action.ok { background:var(--ok-bg); border:1px solid var(--ok-line); } .icv .action.ok b { color:var(--ok); }
|
| 59 |
+
.icv .action.warn { background:var(--warn-bg); border:1px solid var(--warn-line); } .icv .action.warn b { color:var(--warn); }
|
| 60 |
+
.icv .action.bad { background:var(--bad-bg); border:1px solid var(--bad-line); } .icv .action.bad b { color:var(--bad); }
|
| 61 |
+
.icv .evidence { margin-top:12px; border-top:1px dashed var(--line); padding-top:8px; }
|
| 62 |
+
.icv .evidence summary { cursor:pointer; color:var(--green); font-weight:700; }
|
| 63 |
+
.icv .ev-row { margin:10px 0; } .icv .ev-row b { color:var(--gold); font-size:.88rem; } .icv .ev-row p { margin:2px 0; }
|
| 64 |
+
.icv .indicators { display:grid; gap:6px; margin-top:6px; }
|
| 65 |
+
.icv .ind { display:grid; grid-template-columns:150px 1fr 48px; gap:10px; align-items:center; font-size:.88rem; }
|
| 66 |
+
.icv .bar { background:#EFE8D3; border-radius:999px; height:9px; overflow:hidden; direction:rtl; } .icv .bar span { display:block; height:100%; background:linear-gradient(270deg,var(--green),var(--gold)); }
|
| 67 |
+
.icv .ind-val { text-align:left; direction:ltr; color:var(--ink); } .icv .ind-val.wide { grid-column:2 / span 2; text-align:right; direction:rtl; }
|
| 68 |
+
.icv .final { border-color:var(--gold); background:#FFFDF6; }
|
| 69 |
+
.icv .final-head { display:flex; justify-content:space-between; align-items:center; } .icv .final-head b { color:var(--green); font-size:1.05rem; }
|
| 70 |
+
.icv .copy { background:var(--green); color:#FFF; border:0; border-radius:10px; padding:5px 16px; font-family:inherit; cursor:pointer; }
|
| 71 |
+
.icv .notice { background:var(--paper); border:1px solid var(--line); border-radius:14px; padding:16px; }
|
| 72 |
+
.icv .notice.warn { background:var(--warn-bg); border-color:var(--warn-line); color:var(--warn); } .icv .notice.bad { background:var(--bad-bg); border-color:var(--bad-line); color:var(--bad); }
|
| 73 |
+
@media (max-width:640px){ .icv .hero h1{font-size:1.7rem;} .icv .ind{grid-template-columns:104px 1fr 40px;} }
|
| 74 |
+
|
| 75 |
+
body.icv-page{margin:0;min-height:100vh;} .page{max-width:1060px;margin:0 auto;padding:6px 16px 24px;box-sizing:border-box;}
|
| 76 |
+
.tabs{display:flex;gap:8px;margin:6px 0 14px;direction:rtl;}
|
| 77 |
+
.tab{flex:1;border:1px solid var(--line);background:#FFF;color:var(--green);border-radius:12px;padding:11px 14px;font:700 1rem 'Cairo',sans-serif;cursor:pointer;}
|
| 78 |
+
.tab.active{background:var(--green);color:#FFF;border-color:var(--green);box-shadow:0 3px 10px rgba(15,76,58,.2);}
|
| 79 |
+
section[hidden]{display:none;}
|
| 80 |
+
.form label{display:block;color:var(--muted);font-size:.88rem;font-weight:600;margin:10px 0 5px;direction:rtl;text-align:right;}
|
| 81 |
+
.form textarea,.form input,.form select{width:100%;box-sizing:border-box;background:#FFF;color:var(--ink);border:1px solid var(--line);border-radius:12px;padding:12px 14px;font:1rem 'Cairo',sans-serif;direction:rtl;text-align:right;outline:none;}
|
| 82 |
+
.form textarea{min-height:200px;resize:vertical;font-family:'Amiri','Cairo',serif;font-size:1.25rem;line-height:2.1;}
|
| 83 |
+
.form textarea.short{min-height:96px;}
|
| 84 |
+
.form textarea:focus,.form input:focus,.form select:focus{border-color:var(--gold);box-shadow:0 0 0 3px rgba(184,145,47,.18);}
|
| 85 |
+
.row{display:grid;grid-template-columns:1fr 1.4fr 1.6fr;gap:12px;} @media (max-width:760px){.row{grid-template-columns:1fr;}}
|
| 86 |
+
.btns{display:flex;gap:12px;margin:14px 0 6px;flex-wrap:wrap;}
|
| 87 |
+
.btns button{flex:1 1 200px;border:0;border-radius:12px;padding:13px 18px;font:700 1.02rem 'Cairo',sans-serif;cursor:pointer;}
|
| 88 |
+
.btn-main{background:var(--green);color:#FFF;} .btn-main:hover{background:var(--green-2);}
|
| 89 |
+
.btn-alt{background:#FFF;color:var(--green);border:1.5px solid var(--gold)!important;} .btn-alt:hover{background:#FFFBEF;}
|
| 90 |
+
.btns button:disabled{opacity:.45;cursor:not-allowed;}
|
| 91 |
+
.hint{font-size:.85rem;color:var(--muted);direction:rtl;margin:6px 0;}
|
| 92 |
+
#status{direction:rtl;text-align:center;color:var(--muted);font-size:.93rem;margin:8px 0;min-height:1.5em;}
|
| 93 |
+
.spinner{display:inline-block;width:14px;height:14px;border:2px solid var(--line);border-top-color:var(--green);border-radius:50%;animation:spin 1s linear infinite;vertical-align:middle;margin-inline-end:8px;}
|
| 94 |
+
@keyframes spin{to{transform:rotate(360deg)}}
|
| 95 |
+
|
| 96 |
+
/* ===== Executive layer: glassmorphism, dynamic gradients, RTL hardening (overrides _base.css) ===== */
|
| 97 |
+
:root{ --glass:rgba(255,255,255,.62); --glass-strong:rgba(255,255,255,.78); --glass-line:rgba(255,255,255,.7);
|
| 98 |
+
--shadow-1:0 8px 30px rgba(15,76,58,.10); --shadow-2:0 18px 48px rgba(15,76,58,.16); }
|
| 99 |
+
@media (prefers-color-scheme:dark){ :root{ --glass:rgba(24,38,33,.62); --glass-strong:rgba(24,38,33,.8); --glass-line:rgba(255,255,255,.12); } }
|
| 100 |
+
html{scroll-behavior:smooth}
|
| 101 |
+
body.icv-page{ background:#F4EFE0 !important; position:relative; overflow-x:hidden; }
|
| 102 |
+
body.icv-page::before{ content:""; position:fixed; inset:-20%; z-index:-1; pointer-events:none;
|
| 103 |
+
background: radial-gradient(40% 40% at 15% 20%, rgba(23,105,79,.28), transparent 70%),
|
| 104 |
+
radial-gradient(35% 35% at 85% 15%, rgba(226,192,110,.38), transparent 70%),
|
| 105 |
+
radial-gradient(45% 45% at 70% 90%, rgba(15,76,58,.22), transparent 70%);
|
| 106 |
+
animation:drift 22s ease-in-out infinite alternate; }
|
| 107 |
+
@keyframes drift{ to{ transform:translate3d(-3%,2%,0) scale(1.06) } }
|
| 108 |
+
|
| 109 |
+
/* RTL hardening: nothing clips or spills out of labels, cards or buttons */
|
| 110 |
+
.icv, .form, .page{ direction:rtl; text-align:right; }
|
| 111 |
+
.icv *, .form *{ min-width:0; overflow-wrap:anywhere; word-break:normal; }
|
| 112 |
+
.form label{ white-space:normal; line-height:1.6; }
|
| 113 |
+
.btns button, .tab, .copy, .chip{ white-space:normal; line-height:1.5; }
|
| 114 |
+
.en, .form select{ direction:ltr; text-align:left; font-family:'Inter','Segoe UI',system-ui,sans-serif; unicode-bidi:plaintext; }
|
| 115 |
+
.page{ padding-inline:clamp(10px,3vw,16px); }
|
| 116 |
+
|
| 117 |
+
/* Hero */
|
| 118 |
+
.icv .hero{ background:linear-gradient(120deg,#0F4C3A,#17694F,#1C7C5C,#0F4C3A); background-size:300% 300%; animation:flow 14s ease infinite;
|
| 119 |
+
border:1px solid var(--glass-line); border-bottom:4px solid var(--gold); box-shadow:var(--shadow-2); position:relative; overflow:hidden; }
|
| 120 |
+
.icv .hero::after{ content:""; position:absolute; inset:0; background:radial-gradient(60% 80% at 80% 0%, rgba(226,192,110,.25), transparent 60%); pointer-events:none; }
|
| 121 |
+
.icv .hero h1{ font-size:clamp(1.35rem,4.2vw,2.1rem); line-height:1.5; text-wrap:balance; }
|
| 122 |
+
.icv .hero .tagline{ font-size:clamp(1rem,2.6vw,1.2rem); }
|
| 123 |
+
@keyframes flow{ 50%{ background-position:100% 50% } }
|
| 124 |
+
|
| 125 |
+
/* Glass surfaces */
|
| 126 |
+
.icv .qcard,.icv .summary .tile,.icv .generated,.icv .final,.icv .notice,.icv .highlight,.icv .evidence,.form.glass,.panel{
|
| 127 |
+
background:var(--glass) !important; backdrop-filter:blur(14px) saturate(140%); -webkit-backdrop-filter:blur(14px) saturate(140%);
|
| 128 |
+
border:1px solid var(--glass-line) !important; border-radius:18px; box-shadow:var(--shadow-1); }
|
| 129 |
+
.icv .qcard{ transition:transform .25s ease, box-shadow .25s ease; animation:rise .45s ease both; }
|
| 130 |
+
.icv .qcard:hover{ transform:translateY(-3px); box-shadow:var(--shadow-2); }
|
| 131 |
+
@keyframes rise{ from{ opacity:0; transform:translateY(10px) } }
|
| 132 |
+
.form.glass{ padding:18px; margin-bottom:12px; }
|
| 133 |
+
.panel{ padding:14px 16px; margin:10px 0; }
|
| 134 |
+
|
| 135 |
+
/* Tabs / buttons / inputs */
|
| 136 |
+
.tabs{ background:var(--glass); backdrop-filter:blur(12px); border:1px solid var(--glass-line); border-radius:16px; padding:6px; box-shadow:var(--shadow-1); }
|
| 137 |
+
.tab{ border:0 !important; background:transparent !important; transition:all .2s ease; }
|
| 138 |
+
.tab.active{ background:linear-gradient(135deg,var(--green),var(--green-2)) !important; color:#fff !important; }
|
| 139 |
+
.btn-main{ background:linear-gradient(135deg,#0F4C3A,#17694F) !important; box-shadow:0 6px 18px rgba(15,76,58,.28); transition:transform .15s ease, box-shadow .2s; }
|
| 140 |
+
.btn-main:hover:not(:disabled){ transform:translateY(-1px); box-shadow:0 10px 24px rgba(15,76,58,.34); }
|
| 141 |
+
.btn-alt{ background:var(--glass-strong) !important; }
|
| 142 |
+
.form textarea,.form input,.form select{ background:var(--glass-strong) !important; border-radius:14px; }
|
| 143 |
+
.form select{ appearance:none; padding-inline:14px 38px; cursor:pointer;
|
| 144 |
+
background-image:url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='8'%3E%3Cpath d='M1 1l5 5 5-5' fill='none' stroke='%230F4C3A' stroke-width='2'/%3E%3C/svg%3E") !important;
|
| 145 |
+
background-repeat:no-repeat !important; background-position:right 14px center !important; }
|
| 146 |
+
.engine-row{ display:grid; grid-template-columns:minmax(0,1fr); gap:6px; margin-top:12px; }
|
| 147 |
+
.hint{ font-size:.82rem; color:var(--muted); }
|
| 148 |
+
.icv .copy{ background:linear-gradient(135deg,var(--green),var(--green-2)); box-shadow:0 4px 12px rgba(15,76,58,.25); transition:transform .15s; }
|
| 149 |
+
.icv .copy:active{ transform:scale(.96); }
|
| 150 |
+
.icv .final-head{ display:flex; align-items:center; justify-content:space-between; gap:10px; flex-wrap:wrap; }
|
| 151 |
+
|
| 152 |
+
/* Boot progress + skeletons */
|
| 153 |
+
.boot{ display:flex; flex-direction:column; gap:10px; }
|
| 154 |
+
.boot .steps{ display:flex; gap:8px; flex-wrap:wrap; }
|
| 155 |
+
.boot .step{ font-size:.82rem; padding:3px 12px; border-radius:999px; border:1px solid var(--line); background:var(--glass-strong); color:var(--muted); }
|
| 156 |
+
.boot .step.on{ border-color:var(--gold); color:var(--green); font-weight:700; }
|
| 157 |
+
.boot .step.done{ background:var(--ok-bg); border-color:var(--ok-line); color:var(--ok); }
|
| 158 |
+
.progress{ height:8px; border-radius:99px; background:rgba(15,76,58,.12); overflow:hidden; }
|
| 159 |
+
.progress i{ display:block; height:100%; width:6%; border-radius:99px; background:linear-gradient(90deg,var(--gold),var(--green-2)); transition:width .4s ease; }
|
| 160 |
+
.progress.indeterminate i{ width:35%; animation:slide 1.3s ease-in-out infinite; }
|
| 161 |
+
@keyframes slide{ 0%{ margin-inline-start:-35% } 100%{ margin-inline-start:100% } }
|
| 162 |
+
.skeleton{ position:relative; overflow:hidden; background:rgba(15,76,58,.08); border-radius:10px; }
|
| 163 |
+
.skeleton::after{ content:""; position:absolute; inset:0; transform:translateX(-100%);
|
| 164 |
+
background:linear-gradient(90deg,transparent,rgba(255,255,255,.65),transparent); animation:shimmer 1.4s infinite; }
|
| 165 |
+
[dir=rtl] .skeleton::after{ animation-direction:reverse; }
|
| 166 |
+
@keyframes shimmer{ to{ transform:translateX(100%) } }
|
| 167 |
+
.sk-card{ padding:16px; margin:12px 0; }
|
| 168 |
+
.sk-line{ height:14px; margin:10px 0; } .sk-line.w1{width:92%} .sk-line.w2{width:76%} .sk-line.w3{width:55%} .sk-title{height:22px;width:40%;margin-bottom:14px}
|
| 169 |
+
.spinner{ width:16px; height:16px; border-radius:50%; border:3px solid rgba(15,76,58,.2); border-top-color:var(--green); display:inline-block; vertical-align:middle; margin-inline-end:8px; animation:spin .8s linear infinite; }
|
| 170 |
+
@keyframes spin{ to{ transform:rotate(360deg) } }
|
| 171 |
+
#status{ min-height:1.6em; color:var(--muted); font-size:.9rem; margin:6px 2px; }
|
| 172 |
+
|
| 173 |
+
/* Model card (CAMeLBERT) */
|
| 174 |
+
.model-card .head{ display:flex; gap:8px; align-items:center; justify-content:space-between; flex-wrap:wrap; margin-bottom:8px; }
|
| 175 |
+
.model-card .title{ font-weight:700; color:var(--green); }
|
| 176 |
+
.badge-sim{ font-size:.75rem; padding:2px 10px; border-radius:99px; background:var(--warn-bg); border:1px solid var(--warn-line); color:var(--warn); direction:ltr; }
|
| 177 |
+
.badge-live{ font-size:.75rem; padding:2px 10px; border-radius:99px; background:var(--ok-bg); border:1px solid var(--ok-line); color:var(--ok); direction:ltr; }
|
| 178 |
+
.chips{ display:flex; flex-wrap:wrap; gap:4px; line-height:2; }
|
| 179 |
+
.chip{ display:inline-flex; flex-direction:column; align-items:center; padding:1px 8px 2px; border-radius:8px; border:1px solid var(--line); background:var(--glass-strong); font-family:'Amiri',serif; font-size:1.05rem; }
|
| 180 |
+
.chip small{ font:600 .62rem 'Inter',system-ui,sans-serif; direction:ltr; color:var(--muted); }
|
| 181 |
+
.chip.Ayah{ border-color:#8fd0b0; background:#EAF6EF; } .chip.Hadith{ border-color:#e5c871; background:#FFF6DA; }
|
| 182 |
+
|
| 183 |
+
.export{ display:flex; gap:8px; flex-wrap:wrap; margin:10px 0 0; }
|
| 184 |
+
.export button{ flex:0 1 auto; border:1px solid var(--gold); background:var(--glass-strong); color:var(--green); border-radius:12px; padding:8px 16px; font:700 .9rem 'Cairo',sans-serif; cursor:pointer; direction:ltr; }
|
| 185 |
+
|
| 186 |
+
/* Toast */
|
| 187 |
+
#toast{ position:fixed; inset-inline:0; bottom:calc(22px + env(safe-area-inset-bottom,0px)); margin-inline:auto; width:max-content; max-width:calc(100% - 32px); z-index:50;
|
| 188 |
+
background:linear-gradient(135deg,#0F4C3A,#17694F); color:#fff; padding:12px 22px; border-radius:14px; font:700 .98rem 'Cairo',sans-serif; text-align:center;
|
| 189 |
+
box-shadow:0 14px 36px rgba(15,76,58,.35); border:1px solid rgba(226,192,110,.6); opacity:0; transform:translateY(16px); pointer-events:none; transition:all .3s ease; }
|
| 190 |
+
#toast.show{ opacity:1; transform:none; } #toast.err{ background:linear-gradient(135deg,#8C1D18,#B3261E); }
|
| 191 |
+
@media (prefers-reduced-motion:reduce){ *,*::before,*::after{ animation:none !important; transition:none !important; } }
|
| 192 |
+
|
| 193 |
+
[hidden]{display:none !important}
|
ui.py
ADDED
|
@@ -0,0 +1,396 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Arabic-only presentation layer shared by the Gradio app and the static (in-browser) page.
|
| 2 |
+
|
| 3 |
+
Colour rules: green = verified, red = a real mismatch or a missing reference, amber = needs a human. Verified items
|
| 4 |
+
never use red. Every user-facing string is Arabic.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import html
|
| 9 |
+
from typing import List, Optional
|
| 10 |
+
|
| 11 |
+
_e = html.escape
|
| 12 |
+
|
| 13 |
+
GROUP_LABEL = {"verified": "موثّق", "mismatch": "غير مطابق", "review": "تحتاج مراجعة بشرية"}
|
| 14 |
+
TYPE_LABEL = {"Ayah": "آية قرآنية", "Hadith": "حديث نبوي"}
|
| 15 |
+
INDICATORS = [
|
| 16 |
+
("composite", "الدرجة المركبة"),
|
| 17 |
+
("coverage", "تغطية الكلمات"),
|
| 18 |
+
("lcs_ratio", "التسلسل النصي"),
|
| 19 |
+
("token_overlap", "تداخل الكلمات"),
|
| 20 |
+
("edit_sim", "تشابه الحروف"),
|
| 21 |
+
("diacritic_sim", "التشكيل"),
|
| 22 |
+
]
|
| 23 |
+
METHODS = {
|
| 24 |
+
"substring_match": "الاقتباس واردٌ كاملًا في المصدر",
|
| 25 |
+
"threshold_pass": "تجاوز عتبة التطابق",
|
| 26 |
+
"threshold_fail": "دون عتبة التطابق",
|
| 27 |
+
"borderline_multi_cov": "حالة حدّية: عدة مصادر متقاربة",
|
| 28 |
+
"borderline_default": "حالة حدّية",
|
| 29 |
+
"borderline_low_retrieval": "حالة حدّية: استرجاع ضعيف",
|
| 30 |
+
"no_candidates": "لا توجد مصادر مرشحة",
|
| 31 |
+
"empty_span": "نص فارغ",
|
| 32 |
+
"error": "تعذّرت المعالجة",
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _pct(value: float) -> str:
|
| 37 |
+
return f"{round(value * 100)}%"
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def source_label(source: dict) -> str:
|
| 41 |
+
if source["type"] == "Quran":
|
| 42 |
+
start, end = source["ayah_start"], source["ayah_end"]
|
| 43 |
+
verses = f"{start}" if start == end else f"{start}–{end}"
|
| 44 |
+
return f"سورة {source['surah_name']} — الآية {verses}"
|
| 45 |
+
return f"حديث رقم {source['hadithID']} — {source['title']}"
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def reason_text(reason: dict) -> str:
|
| 49 |
+
"""Arabic explanation of a decision code produced by the pipeline."""
|
| 50 |
+
code = reason["code"]
|
| 51 |
+
src = reason.get("source", "")
|
| 52 |
+
if code == "exact_match":
|
| 53 |
+
return f"تطابق تام مع {src} بعد تجاهل اختلافات الرسم والتشكيل."
|
| 54 |
+
if code == "close_match":
|
| 55 |
+
return f"تطابق شبه كامل مع {src}."
|
| 56 |
+
if code == "altered_passage":
|
| 57 |
+
n = reason.get("n", 0)
|
| 58 |
+
how = "تغيّر ترتيب الكلمات" if reason.get("reordered") else f"{n} موضع مختلف"
|
| 59 |
+
return f"النص يخالف {src} ({how}). التصحيح المقترح هو نص المصدر حرفيًا ولم يُولَّد."
|
| 60 |
+
if code == "weak_match":
|
| 61 |
+
return "وُجد مصدر قريب لكن التطابق غير كافٍ للحكم الآلي."
|
| 62 |
+
if code == "no_source":
|
| 63 |
+
return "لا يوجد في المراجع المضمّنة نصٌّ يشبه هذا الاقتباس، وقد يكون مختلَقًا."
|
| 64 |
+
if code == "insufficient_evidence":
|
| 65 |
+
return "الأدلة غير كافية: لا مصدر واضح، ودرجة اليقين منخفضة."
|
| 66 |
+
if code == "candidate_not_strong":
|
| 67 |
+
return f"وُجد مرشح ({src}، قوة المطابقة {round(reason.get('strength', 0) * 100)}%) لكن الدليل لا يكفي للتصحيح الآلي."
|
| 68 |
+
if code == "hadith_candidate":
|
| 69 |
+
return f"وُجدت رواية مشابهة ({src}) لكن لا يُصحَّح الحديث آليًا لاختلاف الروايات؛ يلزم الرجوع إلى مختص."
|
| 70 |
+
return "تعذّرت معالجة هذا الاقتباس آليًا."
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def note_text(note: dict) -> str:
|
| 74 |
+
n = note.get("n", 0)
|
| 75 |
+
code = note["code"]
|
| 76 |
+
if code == "diacritic_conflict":
|
| 77 |
+
return f"تنبيه: تشكيل {n} كلمة يخالف المصحف (الكلمات صحيحة لكن الحركات مختلفة)."
|
| 78 |
+
if code == "orthographic_variant":
|
| 79 |
+
return f"ملاحظة: {n} اختلاف إملائي (رسم أو مسافات) لا يغيّر الكلمة."
|
| 80 |
+
if code == "hadith_minor_diffs":
|
| 81 |
+
return f"ملاحظة: {n} اختلاف طفيف عن أقرب رواية في المراجع."
|
| 82 |
+
return ""
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _diff_html(comparison: dict) -> str:
|
| 86 |
+
"""Red = words only in the quotation; green = the source words that should be there."""
|
| 87 |
+
parts = []
|
| 88 |
+
for op in comparison["word_diff"]:
|
| 89 |
+
if op["op"] == "equal":
|
| 90 |
+
parts.append(f'<span class="w-eq">{_e(op["span"])}</span>')
|
| 91 |
+
continue
|
| 92 |
+
if op["span"]:
|
| 93 |
+
parts.append(f'<span class="w-extra">{_e(op["span"])}</span>')
|
| 94 |
+
if op["source"]:
|
| 95 |
+
parts.append(f'<span class="w-missing">{_e(op["source"])}</span>')
|
| 96 |
+
return " ".join(parts)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _legend(group: str, comparison: Optional[dict]) -> str:
|
| 100 |
+
if group == "verified" or not comparison or all(op["op"] == "equal" for op in comparison["word_diff"]):
|
| 101 |
+
return ""
|
| 102 |
+
return '<p class="legend"><span class="w-extra">كلمات في الاقتباس تخالف المصدر</span> <span class="w-missing">الصواب من المصدر</span></p>'
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _indicator_table(signals: Optional[dict]) -> str:
|
| 106 |
+
if not signals:
|
| 107 |
+
return ""
|
| 108 |
+
rows = []
|
| 109 |
+
for key, label in INDICATORS:
|
| 110 |
+
value = float(signals.get(key, 0.0))
|
| 111 |
+
rows.append(
|
| 112 |
+
f'<div class="ind"><div class="ind-name">{label}</div>'
|
| 113 |
+
f'<div class="bar"><span style="width:{_pct(value)}"></span></div><div class="ind-val">{_pct(value)}</div></div>'
|
| 114 |
+
)
|
| 115 |
+
rows.append(f'<div class="ind"><div class="ind-name">وروده كاملًا في المصدر</div><div class="ind-val wide">{"نعم" if signals.get("is_substring") else "لا"}</div></div>')
|
| 116 |
+
return '<div class="indicators">' + "".join(rows) + "</div>"
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _match_percent(span: dict) -> int:
|
| 120 |
+
evidence = span["evidence"]
|
| 121 |
+
if evidence and evidence.get("comparison"):
|
| 122 |
+
return round(evidence["comparison"]["word_similarity"] * 100)
|
| 123 |
+
if evidence and evidence.get("signals"):
|
| 124 |
+
return round(evidence["signals"]["composite"] * 100)
|
| 125 |
+
return 0
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def _evidence_panel(span: dict) -> str:
|
| 129 |
+
evidence, verification = span["evidence"], span["verification"]
|
| 130 |
+
rows = [
|
| 131 |
+
f'<div class="ev-row"><b>الاقتباس المكتشف</b><p class="quote">{_e(span["text"])}</p></div>',
|
| 132 |
+
f'<div class="ev-row"><b>النوع</b><p>{TYPE_LABEL[span["type"]]}</p></div>',
|
| 133 |
+
]
|
| 134 |
+
if evidence:
|
| 135 |
+
comparison = evidence["comparison"]
|
| 136 |
+
rows += [
|
| 137 |
+
f'<div class="ev-row"><b>المصدر المرشح</b><p>{_e(source_label(evidence["source"]))}</p></div>',
|
| 138 |
+
f'<div class="ev-row"><b>نص المصدر الأصلي</b><p class="quote">{_e(comparison["source_excerpt"])}</p></div>',
|
| 139 |
+
f'<div class="ev-row"><b>المقارنة كلمةً بكلمة</b><p class="diff">{_diff_html(comparison)}</p>{_legend(span["group"], comparison)}</div>',
|
| 140 |
+
f'<div class="ev-row"><b>مؤشرات التحقق</b>{_indicator_table(evidence.get("signals"))}</div>',
|
| 141 |
+
]
|
| 142 |
+
if comparison["diacritic_notes"]:
|
| 143 |
+
items = "، ".join(f"{_e(n['word'])} ← {_e(n['source_word'])}" for n in comparison["diacritic_notes"])
|
| 144 |
+
rows.append(f'<div class="ev-row"><b>اختلافات التشكيل</b><p>{items}</p></div>')
|
| 145 |
+
else:
|
| 146 |
+
rows.append('<div class="ev-row"><b>المصدر المرشح</b><p>لم يُسترجع أي مصدر.</p></div>')
|
| 147 |
+
rows += [
|
| 148 |
+
f'<div class="ev-row"><b>درجة اليقين في الحكم</b><p>{_pct(verification["confidence"])} · {METHODS.get(verification["method"], "")}</p></div>',
|
| 149 |
+
f'<div class="ev-row"><b>القرار</b><p>{_e(span["status_ar"])}</p></div>',
|
| 150 |
+
f'<div class="ev-row"><b>سبب القرار</b><p>{_e(reason_text(span["reason"]))}</p></div>',
|
| 151 |
+
]
|
| 152 |
+
return "".join(rows)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def _card(span: dict) -> str:
|
| 156 |
+
group, evidence = span["group"], span["evidence"]
|
| 157 |
+
label = GROUP_LABEL[group]
|
| 158 |
+
chip = f'<span class="conf">نسبة المطابقة <b>{_match_percent(span)}%</b></span>'
|
| 159 |
+
scan_badge = ('<span class="badge scan" title="اكتُشف بمطابقة النص مع المراجع دون علامات تنصيص">اقتباس غير معلَن</span>'
|
| 160 |
+
if span["detection"]["backend"] == "scan" else "")
|
| 161 |
+
|
| 162 |
+
fields = [f'<div class="field"><label>الاقتباس المكتشف</label><p class="quote">{_e(span["text"])}</p></div>']
|
| 163 |
+
if evidence:
|
| 164 |
+
comparison = evidence["comparison"]
|
| 165 |
+
fields.append(f'<div class="field"><label>المصدر المرشح</label><p>{_e(source_label(evidence["source"]))}</p></div>')
|
| 166 |
+
if group != "verified": # a verified quotation equals its source: no need to print it twice
|
| 167 |
+
fields.append(
|
| 168 |
+
f'<div class="field"><label>المقارنة بالمصدر</label><p class="diff">{_diff_html(comparison)}</p>{_legend(group, comparison)}</div>'
|
| 169 |
+
)
|
| 170 |
+
else:
|
| 171 |
+
fields.append('<div class="field"><label>المصدر المرشح</label><p>لا يوجد</p></div>')
|
| 172 |
+
fields.append(f'<p class="reason">{_e(reason_text(span["reason"]))}</p>')
|
| 173 |
+
|
| 174 |
+
notes = "".join(f'<p class="note">{_e(note_text(n))}</p>' for n in span["notes"] if note_text(n))
|
| 175 |
+
action = ""
|
| 176 |
+
correction, suggestion = span["correction"], span["suggestion"]
|
| 177 |
+
if correction:
|
| 178 |
+
action = (
|
| 179 |
+
'<div class="action ok"><b>التصحيح المقترح من المصدر</b>'
|
| 180 |
+
f'<p class="quote">{_e(correction["display_text"])}</p>'
|
| 181 |
+
f'<p class="legend">{_e(source_label(correction["source"]))}</p></div>'
|
| 182 |
+
)
|
| 183 |
+
elif group == "review":
|
| 184 |
+
closest = ""
|
| 185 |
+
if suggestion:
|
| 186 |
+
closest = (f'<p class="legend">أقرب مصدر وُجد للمراجِع، وليس تصحيحًا آليًا: {_e(source_label(suggestion["source"]))}</p>'
|
| 187 |
+
f'<p class="quote small">{_e(suggestion["display_text"][:420])}</p>')
|
| 188 |
+
action = ('<div class="action warn"><b>الأدلة غير كافية للتصحيح الآلي.</b> يُوصى بالمرا��عة البشرية.' + closest + "</div>")
|
| 189 |
+
elif span["status"] == "UNSUPPORTED":
|
| 190 |
+
action = '<div class="action bad"><b>لا يوجد مصدر مطابق في المراجع المتاحة.</b> لم يُقترح أي نص بديل.</div>'
|
| 191 |
+
|
| 192 |
+
return f"""
|
| 193 |
+
<div class="qcard {group}">
|
| 194 |
+
<div class="qhead">
|
| 195 |
+
<span class="idx">{span["id"]}</span>
|
| 196 |
+
<span class="badge type">{TYPE_LABEL[span["type"]]}</span>{scan_badge}
|
| 197 |
+
<span class="badge st {group}">{label}</span>
|
| 198 |
+
{chip}
|
| 199 |
+
</div>
|
| 200 |
+
{"".join(fields)}
|
| 201 |
+
{notes}
|
| 202 |
+
{action}
|
| 203 |
+
<details class="evidence"><summary>عرض الدليل</summary><div class="ev-body">{_evidence_panel(span)}</div></details>
|
| 204 |
+
</div>"""
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def _summary(summary: dict) -> str:
|
| 208 |
+
tiles = [
|
| 209 |
+
(summary["n_spans"], "إجمالي الاقتباسات", ""),
|
| 210 |
+
(summary["n_ayah"], "آيات قرآنية", ""),
|
| 211 |
+
(summary["n_hadith"], "أحاديث نبوية", ""),
|
| 212 |
+
(summary["VERIFIED"], "موثّق", "verified"),
|
| 213 |
+
(summary["CORRECTED"] + summary["UNSUPPORTED"], "غير مطابق", "mismatch"),
|
| 214 |
+
(summary["HUMAN_REVIEW"], "تحتاج مراجعة", "review"),
|
| 215 |
+
]
|
| 216 |
+
return '<div class="summary">' + "".join(
|
| 217 |
+
f'<div class="tile {cls}{" zero" if value == 0 and cls else ""}"><b>{value}</b><span>{label}</span></div>'
|
| 218 |
+
for value, label, cls in tiles
|
| 219 |
+
) + "</div>"
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def _highlighted_text(result: dict, title: str) -> str:
|
| 223 |
+
text, pieces, cursor = result["input_text"], [], 0
|
| 224 |
+
for span in result["spans"]:
|
| 225 |
+
pieces.append(_e(text[cursor:span["start"]]))
|
| 226 |
+
pieces.append(f'<mark class="{span["group"]}">{_e(text[span["start"]:span["end"]])}</mark>')
|
| 227 |
+
cursor = span["end"]
|
| 228 |
+
pieces.append(_e(text[cursor:]))
|
| 229 |
+
legend = ('<div class="hl-legend"><mark class="verified">موثّق</mark><mark class="mismatch">غير مطابق</mark>'
|
| 230 |
+
'<mark class="review">مراجعة بشرية</mark></div>')
|
| 231 |
+
return f'<div class="highlight"><label>{title}</label><p>' + "".join(pieces) + "</p>" + legend + "</div>"
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def final_text(result: dict) -> str:
|
| 235 |
+
"""Corrected text with an inline Arabic flag after every quotation that still needs attention."""
|
| 236 |
+
text, reports = result["input_text"], result["spans"]
|
| 237 |
+
for report in sorted(reports, key=lambda r: r["start"], reverse=True):
|
| 238 |
+
if report["status"] == "CORRECTED" and report["correction"]:
|
| 239 |
+
text = text[: report["start"]] + report["correction"]["display_text"] + text[report["end"]:]
|
| 240 |
+
elif report["status"] == "UNSUPPORTED":
|
| 241 |
+
text = text[: report["end"]] + " [⚠ لا يوجد مصدر مطابق]" + text[report["end"]:]
|
| 242 |
+
elif report["status"] == "HUMAN_REVIEW":
|
| 243 |
+
text = text[: report["end"]] + " [⚠ يحتاج مراجعة بشرية]" + text[report["end"]:]
|
| 244 |
+
return text
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def _final_block(result: dict) -> str:
|
| 248 |
+
summary = result["summary"]
|
| 249 |
+
fixed = summary["CORRECTED"]
|
| 250 |
+
open_items = summary["UNSUPPORTED"] + summary["HUMAN_REVIEW"]
|
| 251 |
+
if fixed == 0 and open_items == 0:
|
| 252 |
+
headline = "كل الاقتباسات المكتشفة مطابقة للمصادر."
|
| 253 |
+
else:
|
| 254 |
+
headline = f"صُحِّح {fixed} اقتباس من نص المصدر، وبقي {open_items} اقتباس يحتاج مراجعة بشرية ومعلَّم بعلامة تنبيه."
|
| 255 |
+
text = final_text(result)
|
| 256 |
+
return (
|
| 257 |
+
'<div class="final"><div class="final-head"><b>النسخة المصحّحة</b>'
|
| 258 |
+
f'<button class="copy copy-btn" type="button" data-text="{_e(text, quote=True)}" '
|
| 259 |
+
'onclick="window.icvCopy?window.icvCopy(this):(navigator.clipboard&&navigator.clipboard.writeText(this.dataset.text))">نسخ النص</button></div>'
|
| 260 |
+
f'<p class="legend">{headline}</p><p class="final-text">{_e(text)}</p></div>'
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
def render_results(result: dict, generated_answer: Optional[str] = None) -> str:
|
| 265 |
+
"""HTML report for a pipeline result. Pass the model's answer (mode B) to show it first and label the text as generated."""
|
| 266 |
+
header = render_generated_header(generated_answer) if generated_answer is not None else ""
|
| 267 |
+
if not result["spans"]:
|
| 268 |
+
body = (
|
| 269 |
+
'<div class="notice">لم يُعثر على اقتباسات قرآنية أو حديثية في هذا النص. يكتشف النظام الاقتباسات بمطابقة النص مع مراجع القرآن والحديث، '
|
| 270 |
+
'سواء وُضعت بين علامات تنصيص أو لم توضع، وسواء سبقتها عبارة تمهيدية أو لم تسبقها.</div>'
|
| 271 |
+
)
|
| 272 |
+
return f'<div class="icv">{header}{body}</div>'
|
| 273 |
+
generated = generated_answer is not None
|
| 274 |
+
title = "إجابة النموذج مع الاقتباسات المكتشفة" if generated else "النص مع الاقتباسات المكتشفة"
|
| 275 |
+
parts = [_summary(result["summary"]), _highlighted_text(result, title), "".join(_card(s) for s in result["spans"])]
|
| 276 |
+
summary = result["summary"]
|
| 277 |
+
if generated or summary["CORRECTED"] or summary["UNSUPPORTED"] or summary["HUMAN_REVIEW"]:
|
| 278 |
+
parts.append(_final_block(result))
|
| 279 |
+
return f'<div class="icv">{header}{"".join(parts)}</div>'
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
def render_generated_header(answer: str) -> str:
|
| 283 |
+
return f'<div class="generated"><label>إجابة النموذج اللغوي كما وصلت</label><p>{_e(answer)}</p></div>'
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def render_message(message: str, kind: str = "info") -> str:
|
| 287 |
+
return f'<div class="icv"><div class="notice {kind}">{_e(message)}</div></div>'
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
# --------------------------------------------------------------------------------------------------------------
|
| 291 |
+
# Static markup and CSS
|
| 292 |
+
# --------------------------------------------------------------------------------------------------------------
|
| 293 |
+
STAR = (
|
| 294 |
+
'<svg class="mark" viewBox="0 0 64 64" fill="none" stroke="#E2C06E" stroke-width="1.8" aria-hidden="true">'
|
| 295 |
+
'<rect x="14" y="14" width="36" height="36"/><rect x="14" y="14" width="36" height="36" transform="rotate(45 32 32)"/>'
|
| 296 |
+
'<circle cx="32" cy="32" r="7" fill="#E2C06E" stroke="none"/></svg>'
|
| 297 |
+
)
|
| 298 |
+
|
| 299 |
+
HERO = f"""
|
| 300 |
+
<div class="icv"><div class="hero">
|
| 301 |
+
{STAR}
|
| 302 |
+
<h1>التحقق من هلوسة القرآن والحديث وتصحيحها</h1>
|
| 303 |
+
<p class="tagline">تحقّق من آيات القرآن والأحاديث النبوية بالدليل</p>
|
| 304 |
+
<p class="sub">يكتشف النظام الاقتباسات داخل أي نص، ويسترجع نصوصها من المصادر، ويقارنها كلمةً بكلمة، ثم يعرض الدليل.
|
| 305 |
+
وإذا لم تكفِ الأدلة فلن يختلق تصحيحًا، بل يحيل الحالة إلى المراجعة البشرية.</p>
|
| 306 |
+
<div class="flow"><span>كشف</span><i>‹</i><span>استرجاع</span><i>‹</i><span>محاذاة</span><i>‹</i><span>دليل</span><i>‹</i><span>قرار</span></div>
|
| 307 |
+
</div></div>
|
| 308 |
+
"""
|
| 309 |
+
|
| 310 |
+
DISCLAIMER = """<div class="icv"><div class="disclaimer">أداة مساعدة للتدقيق النصي وليست فتوى ولا بديلًا عن المراجعة المتخصصة.
|
| 311 |
+
النتائج مبنية على مراجع القرآن الكريم والكتب الستة المضمّنة فقط.</div></div>"""
|
| 312 |
+
|
| 313 |
+
PLACEHOLDER = "الصق هنا أي نص يتضمن آيات أو أحاديث، سواء كانت بين علامات تنصيص أو بدونها، ومع أي صياغة تمهيدية أو بدونها…"
|
| 314 |
+
PROMPT_PLACEHOLDER = "اكتب سؤالك، مثل: اشرح لي فضل الصبر في القرآن والسنة مع ذكر الأدلة."
|
| 315 |
+
|
| 316 |
+
_PATTERN = (
|
| 317 |
+
"url(\"data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='88' height='88' viewBox='0 0 88 88'%3E"
|
| 318 |
+
"%3Cg fill='none' stroke='%230F4C3A' stroke-opacity='0.07' stroke-width='1'%3E"
|
| 319 |
+
"%3Crect x='22' y='22' width='44' height='44'/%3E%3Crect x='22' y='22' width='44' height='44' transform='rotate(45 44 44)'/%3E"
|
| 320 |
+
"%3C/g%3E%3C/svg%3E\")"
|
| 321 |
+
)
|
| 322 |
+
|
| 323 |
+
CSS = """
|
| 324 |
+
@import url('https://fonts.googleapis.com/css2?family=Cairo:wght@400;600;700&family=Amiri:wght@400;700&display=swap');
|
| 325 |
+
:root { --green:#0F4C3A; --green-2:#17694F; --gold:#B8912F; --gold-2:#E2C06E; --cream:#FBF6EA; --paper:#FFFFFF;
|
| 326 |
+
--ink:#1F2933; --muted:#55626D; --line:#E6DCC3; --ok:#1B7A4B; --ok-bg:#EAF6EF; --ok-line:#BFE3CD;
|
| 327 |
+
--bad:#B3261E; --bad-bg:#FDECEA; --bad-line:#F4B8B3; --warn:#7A5B0C; --warn-bg:#FFF6DA; --warn-line:#EBD28A; }
|
| 328 |
+
.gradio-container, body.icv-page { background:var(--cream) PATTERN !important; font-family:'Cairo','Segoe UI',Tahoma,sans-serif; color:var(--ink); }
|
| 329 |
+
.gradio-container { max-width:1060px !important; --body-text-color:#1F2933; --block-background-fill:#FFFFFF; --block-border-color:#E6DCC3;
|
| 330 |
+
--input-background-fill:#FFFFFF; --block-label-text-color:#55626D; --button-primary-background-fill:#0F4C3A;
|
| 331 |
+
--button-primary-background-fill-hover:#17694F; --button-primary-text-color:#FFFFFF; --button-secondary-background-fill:#FFFFFF;
|
| 332 |
+
--button-secondary-text-color:#0F4C3A; --button-secondary-border-color:#B8912F; --color-accent:#B8912F; }
|
| 333 |
+
.icv { direction:rtl; text-align:right; color:var(--ink); line-height:1.9; }
|
| 334 |
+
.icv .hero { background:linear-gradient(135deg,#0F4C3A,#17694F); border-radius:20px; padding:30px 22px 26px; margin:10px 0 18px; text-align:center;
|
| 335 |
+
box-shadow:0 6px 22px rgba(15,76,58,.18); border-bottom:4px solid var(--gold); }
|
| 336 |
+
.icv .hero .mark { width:54px; height:54px; display:block; margin:0 auto 4px; }
|
| 337 |
+
.icv .hero h1 { font-size:2.3rem; margin:.1rem 0; color:#FFFFFF; font-weight:700; }
|
| 338 |
+
.icv .hero .tagline { color:var(--gold-2); font-size:1.2rem; font-weight:600; margin:.1rem 0 .6rem; }
|
| 339 |
+
.icv .hero .sub { max-width:720px; margin:0 auto; color:#E9F2EE; font-size:1rem; }
|
| 340 |
+
.icv .flow { display:flex; flex-wrap:wrap; justify-content:center; align-items:center; gap:8px; margin-top:16px; }
|
| 341 |
+
.icv .flow span { background:rgba(255,255,255,.12); border:1px solid rgba(226,192,110,.55); color:#FFF; border-radius:999px; padding:3px 16px; font-size:.9rem; }
|
| 342 |
+
.icv .flow i { color:var(--gold-2); font-style:normal; font-size:1.2rem; }
|
| 343 |
+
.icv .disclaimer { font-size:.85rem; color:var(--muted); text-align:center; padding:14px 8px; }
|
| 344 |
+
.icv .en-free { direction:rtl; }
|
| 345 |
+
.input-area textarea { direction:rtl; text-align:right; font-family:'Amiri','Cairo',serif !important; font-size:1.25rem !important; line-height:2.1 !important; background:#FFFFFF !important; color:#1F2933 !important; }
|
| 346 |
+
.llm-row label, .gradio-container label span { font-family:'Cairo',sans-serif; }
|
| 347 |
+
.results { direction:rtl; }
|
| 348 |
+
.icv .summary { display:grid; grid-template-columns:repeat(auto-fit,minmax(130px,1fr)); gap:10px; margin:8px 0 14px; }
|
| 349 |
+
.icv .tile { background:var(--paper); border:1px solid var(--line); border-radius:14px; padding:12px 8px; text-align:center; }
|
| 350 |
+
.icv .tile b { display:block; font-size:1.8rem; color:var(--green); line-height:1.3; } .icv .tile span { font-size:.92rem; color:var(--muted); }
|
| 351 |
+
.icv .tile.verified { background:var(--ok-bg); border-color:var(--ok-line); } .icv .tile.verified b { color:var(--ok); }
|
| 352 |
+
.icv .tile.mismatch { background:var(--bad-bg); border-color:var(--bad-line); } .icv .tile.mismatch b { color:var(--bad); }
|
| 353 |
+
.icv .tile.review { background:var(--warn-bg); border-color:var(--warn-line); } .icv .tile.review b { color:var(--warn); }
|
| 354 |
+
.icv .tile.zero { background:var(--paper); border-color:var(--line); opacity:.6; } .icv .tile.zero b { color:var(--muted); }
|
| 355 |
+
.icv .highlight, .icv .generated, .icv .final { background:var(--paper); border:1px solid var(--line); border-radius:14px; padding:14px 18px; margin-bottom:14px; }
|
| 356 |
+
.icv label { display:block; color:var(--muted); font-size:.82rem; font-weight:600; margin-bottom:4px; }
|
| 357 |
+
.icv .highlight p, .icv .generated p, .icv .final-text { font-family:'Amiri','Cairo',serif; font-size:1.2rem; line-height:2.2; margin:0; white-space:pre-wrap; }
|
| 358 |
+
.icv mark { color:var(--ink); border-radius:6px; padding:1px 5px; }
|
| 359 |
+
.icv mark.verified { background:#D5EEDF; } .icv mark.mismatch { background:#F8CFCB; } .icv mark.review { background:#F7E3A6; }
|
| 360 |
+
.icv .hl-legend { display:flex; flex-wrap:wrap; gap:8px; margin-top:10px; font-size:.78rem; }
|
| 361 |
+
.icv .qcard { background:var(--paper); border:1px solid var(--line); border-inline-start:6px solid var(--line); border-radius:16px; padding:16px 18px; margin:12px 0; }
|
| 362 |
+
.icv .qcard.verified { border-inline-start-color:var(--ok); } .icv .qcard.mismatch { border-inline-start-color:var(--bad); } .icv .qcard.review { border-inline-start-color:var(--gold); }
|
| 363 |
+
.icv .qhead { display:flex; flex-wrap:wrap; gap:8px; align-items:center; margin-bottom:8px; }
|
| 364 |
+
.icv .idx { background:var(--green); color:#FFF; border-radius:50%; width:28px; height:28px; display:inline-flex; align-items:center; justify-content:center; font-weight:700; font-size:.9rem; }
|
| 365 |
+
.icv .badge { padding:2px 14px; border-radius:999px; font-size:.84rem; background:#F3EEDD; border:1px solid var(--line); color:var(--ink); }
|
| 366 |
+
.icv .badge.st.verified { background:var(--ok-bg); border-color:var(--ok-line); color:var(--ok); font-weight:600; }
|
| 367 |
+
.icv .badge.st.mismatch { background:var(--bad-bg); border-color:var(--bad-line); color:var(--bad); font-weight:600; }
|
| 368 |
+
.icv .badge.st.review { background:var(--warn-bg); border-color:var(--warn-line); color:var(--warn); font-weight:600; }
|
| 369 |
+
.icv .badge.scan { background:#EEF3FB; border-color:#C9D8EE; color:#2F4F7F; }
|
| 370 |
+
.icv .conf { margin-inline-start:auto; color:var(--muted); font-size:.88rem; } .icv .conf b { color:var(--green); }
|
| 371 |
+
.icv .field { margin:8px 0; } .icv .field p { margin:0; }
|
| 372 |
+
.icv .quote { font-family:'Amiri','Cairo',serif; font-size:1.25rem; line-height:2.2; color:var(--ink); } .icv .quote.small { font-size:1.05rem; color:#3a4651; }
|
| 373 |
+
.icv .diff { font-family:'Amiri','Cairo',serif; font-size:1.2rem; line-height:2.2; }
|
| 374 |
+
.icv .w-extra { background:var(--bad-bg); color:var(--bad); border-radius:4px; padding:0 4px; text-decoration:line-through; }
|
| 375 |
+
.icv .w-missing { background:var(--ok-bg); color:var(--ok); border-radius:4px; padding:0 4px; font-weight:700; }
|
| 376 |
+
.icv .legend { color:var(--muted); font-size:.82rem; margin:4px 0 0; } .icv .legend span { text-decoration:none; font-size:.8rem; }
|
| 377 |
+
.icv .reason { color:var(--muted); font-size:.92rem; margin:6px 0 0; }
|
| 378 |
+
.icv .note { color:var(--warn); background:var(--warn-bg); border:1px solid var(--warn-line); border-radius:10px; padding:5px 12px; font-size:.88rem; margin:8px 0 0; }
|
| 379 |
+
.icv .action { border-radius:12px; padding:10px 14px; margin-top:10px; }
|
| 380 |
+
.icv .action.ok { background:var(--ok-bg); border:1px solid var(--ok-line); } .icv .action.ok b { color:var(--ok); }
|
| 381 |
+
.icv .action.warn { background:var(--warn-bg); border:1px solid var(--warn-line); } .icv .action.warn b { color:var(--warn); }
|
| 382 |
+
.icv .action.bad { background:var(--bad-bg); border:1px solid var(--bad-line); } .icv .action.bad b { color:var(--bad); }
|
| 383 |
+
.icv .evidence { margin-top:12px; border-top:1px dashed var(--line); padding-top:8px; }
|
| 384 |
+
.icv .evidence summary { cursor:pointer; color:var(--green); font-weight:700; }
|
| 385 |
+
.icv .ev-row { margin:10px 0; } .icv .ev-row b { color:var(--gold); font-size:.88rem; } .icv .ev-row p { margin:2px 0; }
|
| 386 |
+
.icv .indicators { display:grid; gap:6px; margin-top:6px; }
|
| 387 |
+
.icv .ind { display:grid; grid-template-columns:150px 1fr 48px; gap:10px; align-items:center; font-size:.88rem; }
|
| 388 |
+
.icv .bar { background:#EFE8D3; border-radius:999px; height:9px; overflow:hidden; direction:rtl; } .icv .bar span { display:block; height:100%; background:linear-gradient(270deg,var(--green),var(--gold)); }
|
| 389 |
+
.icv .ind-val { text-align:left; direction:ltr; color:var(--ink); } .icv .ind-val.wide { grid-column:2 / span 2; text-align:right; direction:rtl; }
|
| 390 |
+
.icv .final { border-color:var(--gold); background:#FFFDF6; }
|
| 391 |
+
.icv .final-head { display:flex; justify-content:space-between; align-items:center; } .icv .final-head b { color:var(--green); font-size:1.05rem; }
|
| 392 |
+
.icv .copy { background:var(--green); color:#FFF; border:0; border-radius:10px; padding:5px 16px; font-family:inherit; cursor:pointer; }
|
| 393 |
+
.icv .notice { background:var(--paper); border:1px solid var(--line); border-radius:14px; padding:16px; }
|
| 394 |
+
.icv .notice.warn { background:var(--warn-bg); border-color:var(--warn-line); color:var(--warn); } .icv .notice.bad { background:var(--bad-bg); border-color:var(--bad-line); color:var(--bad); }
|
| 395 |
+
@media (max-width:640px){ .icv .hero h1{font-size:1.7rem;} .icv .ind{grid-template-columns:104px 1fr 40px;} }
|
| 396 |
+
""".replace("PATTERN", _PATTERN)
|
verifier.py
ADDED
|
@@ -0,0 +1,503 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evidence-based verification and source-backed correction of Quran and Hadith quotations.
|
| 2 |
+
|
| 3 |
+
Pipeline (see ``IslamicContentVerifier``):
|
| 4 |
+
|
| 5 |
+
input text -> detection -> BM25 retrieval -> alignment (phonetic skeleton, sliding window, LCS, dynamic gap)
|
| 6 |
+
-> exact match : verified
|
| 7 |
+
-> altered Quran passage : mismatch with the exact source text as the correction
|
| 8 |
+
-> anything uncertain : human review (no correction is produced)
|
| 9 |
+
-> nothing similar found : no matching source
|
| 10 |
+
|
| 11 |
+
Safety rule: a correction is only ever the exact text of a retrieved source. Nothing is generated.
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import difflib
|
| 16 |
+
import logging
|
| 17 |
+
import time
|
| 18 |
+
from dataclasses import dataclass, field
|
| 19 |
+
from typing import Dict, List, Optional
|
| 20 |
+
|
| 21 |
+
from alignment import align
|
| 22 |
+
from detector import DetectedSpan
|
| 23 |
+
from scanner import HybridDetector
|
| 24 |
+
from idgham import apply_idgham
|
| 25 |
+
from normalization import content_words, normalize_for_matching
|
| 26 |
+
from retrieval import SourceRetriever
|
| 27 |
+
from similarity import best_match_score, compute_signals
|
| 28 |
+
|
| 29 |
+
logger = logging.getLogger(__name__)
|
| 30 |
+
|
| 31 |
+
MAX_INPUT_CHARS = 20_000
|
| 32 |
+
MIN_EXACT_TOKENS = 3 # shorter quotations are too generic to be verified by exact containment alone
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
# --------------------------------------------------------------------------------------------------------------
|
| 36 |
+
# Configuration (all tunable numbers live here)
|
| 37 |
+
# --------------------------------------------------------------------------------------------------------------
|
| 38 |
+
@dataclass
|
| 39 |
+
class VerifierConfig:
|
| 40 |
+
"""Thresholds calibrated in Subtask 1B of the research notebook."""
|
| 41 |
+
|
| 42 |
+
quran_correct_threshold: float = 0.94
|
| 43 |
+
quran_uncertain_low: float = 0.45
|
| 44 |
+
quran_min_coverage: float = 0.40
|
| 45 |
+
hadith_correct_threshold: float = 0.88
|
| 46 |
+
hadith_uncertain_low: float = 0.30
|
| 47 |
+
hadith_min_coverage: float = 0.70
|
| 48 |
+
quran_top_k: int = 25
|
| 49 |
+
hadith_top_k: int = 15
|
| 50 |
+
hadith_retrieval_guard: float = 0.20
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@dataclass
|
| 54 |
+
class CorrectorConfig:
|
| 55 |
+
"""Correction is proposed only when match strength >= ``*_strong``; between ``*_low`` and ``*_strong`` the
|
| 56 |
+
case goes to human review. Hadith is never auto-corrected (``hadith_strong`` > 1), by design: the exact
|
| 57 |
+
fragment boundaries of a Hadith quotation cannot be reproduced reliably, and narrations differ legitimately."""
|
| 58 |
+
|
| 59 |
+
max_window: int = 8
|
| 60 |
+
hadith_top_k: int = 40
|
| 61 |
+
quran_strong: float = 0.65
|
| 62 |
+
min_full_ratio: float = 0.40
|
| 63 |
+
quran_low: float = 0.55
|
| 64 |
+
hadith_strong: float = 1.01
|
| 65 |
+
hadith_low: float = 0.45
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
@dataclass
|
| 69 |
+
class PipelineConfig:
|
| 70 |
+
verifier: VerifierConfig = field(default_factory=VerifierConfig)
|
| 71 |
+
corrector: CorrectorConfig = field(default_factory=CorrectorConfig)
|
| 72 |
+
verified_min_conf: float = 0.75 # "Correct" verdicts weaker than this -> human review
|
| 73 |
+
unsupported_min_conf: float = 0.70 # "Incorrect + no source" needs this confidence to abstain confidently
|
| 74 |
+
unsupported_strength: float = 0.35 # ...or the best source match is this weak (nothing similar exists)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
# --------------------------------------------------------------------------------------------------------------
|
| 78 |
+
# Verification (Subtask 1B)
|
| 79 |
+
# --------------------------------------------------------------------------------------------------------------
|
| 80 |
+
@dataclass
|
| 81 |
+
class Verification:
|
| 82 |
+
verdict: str # 'Correct' | 'Incorrect'
|
| 83 |
+
confidence: float
|
| 84 |
+
best_score: float
|
| 85 |
+
method: str
|
| 86 |
+
source: Optional[dict] = None # best matching source record (with 'signals')
|
| 87 |
+
n_candidates: int = 0
|
| 88 |
+
retrieval_top: float = 0.0
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
class Verifier:
|
| 92 |
+
"""Scores a quotation against retrieved candidates and returns a verdict with its supporting source."""
|
| 93 |
+
|
| 94 |
+
def __init__(self, retriever: SourceRetriever, config: Optional[VerifierConfig] = None) -> None:
|
| 95 |
+
self.kb = retriever
|
| 96 |
+
self.cfg = config or VerifierConfig()
|
| 97 |
+
|
| 98 |
+
def verify(self, span_text: str, content_type: str) -> Verification:
|
| 99 |
+
if not span_text or not span_text.strip():
|
| 100 |
+
return self._result("Incorrect", 0.95, 0.0, None, 0, "empty_span")
|
| 101 |
+
if content_type == "Ayah":
|
| 102 |
+
return self._verify_quran(span_text)
|
| 103 |
+
if content_type == "Hadith":
|
| 104 |
+
return self._verify_hadith(span_text)
|
| 105 |
+
return self._result("Incorrect", 0.5, 0.0, None, 0, "unknown_type")
|
| 106 |
+
|
| 107 |
+
def _verify_quran(self, span: str) -> Verification:
|
| 108 |
+
cfg = self.cfg
|
| 109 |
+
candidates = self.kb.search_quran_ayahs(span, top_k=cfg.quran_top_k)
|
| 110 |
+
if not candidates:
|
| 111 |
+
return self._result("Incorrect", 0.8, 0.0, None, 0, "no_candidates")
|
| 112 |
+
score, best = best_match_score(span, candidates, "Ayah")
|
| 113 |
+
signals = best.get("signals", {}) if best else {}
|
| 114 |
+
coverage, is_substring = signals.get("coverage", 0.0), signals.get("is_substring", 0)
|
| 115 |
+
n = len(candidates)
|
| 116 |
+
|
| 117 |
+
if is_substring and coverage >= cfg.quran_min_coverage:
|
| 118 |
+
return self._result("Correct", min(0.98, 0.85 + score * 0.15), score, best, n, "substring_match")
|
| 119 |
+
if score >= cfg.quran_correct_threshold and coverage >= cfg.quran_min_coverage:
|
| 120 |
+
return self._result("Correct", min(0.95, 0.70 + score * 0.25), score, best, n, "threshold_pass")
|
| 121 |
+
if score <= cfg.quran_uncertain_low:
|
| 122 |
+
return self._result("Incorrect", min(0.95, 0.70 + (1 - score) * 0.25), score, best, n, "threshold_fail")
|
| 123 |
+
|
| 124 |
+
strong = sum(
|
| 125 |
+
1 for cand in candidates[:10]
|
| 126 |
+
if (s := compute_signals(span, cand.get("text", ""), "Ayah"))["coverage"] >= 0.80 and s["lcs_ratio"] >= 0.75
|
| 127 |
+
)
|
| 128 |
+
if strong >= 2:
|
| 129 |
+
return self._result("Correct", 0.60 + min(0.20, strong * 0.05), score, best, n, "borderline_multi_cov")
|
| 130 |
+
return self._result("Incorrect", 0.58, score, best, n, "borderline_default")
|
| 131 |
+
|
| 132 |
+
def _verify_hadith(self, span: str) -> Verification:
|
| 133 |
+
cfg = self.cfg
|
| 134 |
+
candidates = self.kb.search_hadith(span, top_k=cfg.hadith_top_k)
|
| 135 |
+
if not candidates:
|
| 136 |
+
return self._result("Incorrect", 0.75, 0.0, None, 0, "no_candidates")
|
| 137 |
+
top_retrieval = candidates[0].get("retrieval_score", 0.0)
|
| 138 |
+
score, best = best_match_score(span, candidates, "Hadith")
|
| 139 |
+
signals = best.get("signals", {}) if best else {}
|
| 140 |
+
coverage, is_substring = signals.get("coverage", 0.0), signals.get("is_substring", 0)
|
| 141 |
+
n = len(candidates)
|
| 142 |
+
|
| 143 |
+
if is_substring and coverage >= cfg.hadith_min_coverage and top_retrieval >= cfg.hadith_retrieval_guard:
|
| 144 |
+
return self._result("Correct", min(0.97, 0.80 + score * 0.17), score, best, n, "substring_match", top_retrieval)
|
| 145 |
+
if score >= cfg.hadith_correct_threshold and coverage >= cfg.hadith_min_coverage:
|
| 146 |
+
return self._result("Correct", min(0.92, 0.65 + score * 0.27), score, best, n, "threshold_pass", top_retrieval)
|
| 147 |
+
if score <= cfg.hadith_uncertain_low:
|
| 148 |
+
return self._result("Incorrect", min(0.90, 0.65 + (1 - score) * 0.25), score, best, n, "threshold_fail", top_retrieval)
|
| 149 |
+
|
| 150 |
+
moderate = sum(
|
| 151 |
+
1 for cand in candidates[:8]
|
| 152 |
+
if (s := compute_signals(span, cand.get("text", ""), "Hadith"))["coverage"] >= 0.65 and s["lcs_ratio"] >= 0.55
|
| 153 |
+
)
|
| 154 |
+
if moderate >= 2 and top_retrieval >= 0.30:
|
| 155 |
+
return self._result("Correct", 0.58 + min(0.22, moderate * 0.06), score, best, n, "borderline_multi_cov", top_retrieval)
|
| 156 |
+
if top_retrieval < 0.25 or score < 0.45:
|
| 157 |
+
return self._result("Incorrect", 0.60, score, best, n, "borderline_low_retrieval", top_retrieval)
|
| 158 |
+
return self._result("Incorrect", 0.55, score, best, n, "borderline_default", top_retrieval)
|
| 159 |
+
|
| 160 |
+
@staticmethod
|
| 161 |
+
def _result(verdict, confidence, score, best, n_candidates, method, top_retrieval=0.0) -> Verification:
|
| 162 |
+
return Verification(verdict, round(confidence, 4), round(score, 4), method, best, n_candidates, round(top_retrieval, 4))
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
# --------------------------------------------------------------------------------------------------------------
|
| 166 |
+
# Correction (Subtask 1C): locate the true ayah window / Hadith record and return its exact text
|
| 167 |
+
# --------------------------------------------------------------------------------------------------------------
|
| 168 |
+
@dataclass
|
| 169 |
+
class CorrectionMatch:
|
| 170 |
+
kind: str # 'Ayah' | 'Hadith'
|
| 171 |
+
strength: float # coverage (Quran) / symmetric containment (Hadith)
|
| 172 |
+
full_ratio: float
|
| 173 |
+
text: str # proposed correction in the official 1C format (idgham + '(n)' ayah markers)
|
| 174 |
+
source: dict # reference metadata
|
| 175 |
+
display: str = "" # clean human-readable version
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
class Corrector:
|
| 179 |
+
def __init__(self, retriever: SourceRetriever, config: Optional[CorrectorConfig] = None) -> None:
|
| 180 |
+
self.kb = retriever
|
| 181 |
+
self.cfg = config or CorrectorConfig()
|
| 182 |
+
|
| 183 |
+
def match(self, span_text: str, content_type: str) -> Optional[CorrectionMatch]:
|
| 184 |
+
return self.match_quran(span_text) if content_type == "Ayah" else self.match_hadith(span_text)
|
| 185 |
+
|
| 186 |
+
def match_quran(self, query_text: str) -> Optional[CorrectionMatch]:
|
| 187 |
+
"""Best window of 1-8 consecutive ayahs (dynamic sliding window over BM25 seeds, exact-result pruning)."""
|
| 188 |
+
kb = self.kb
|
| 189 |
+
query_norm = normalize_for_matching(query_text)
|
| 190 |
+
query_words = content_words(query_norm.split())
|
| 191 |
+
if not query_words:
|
| 192 |
+
return None
|
| 193 |
+
query_len = len(query_norm)
|
| 194 |
+
memo: Dict[tuple, tuple] = {}
|
| 195 |
+
best = None # (key, coverage, ratio, surah, start, end)
|
| 196 |
+
for seed in kb.quran_seed_ayahs(query_words, top_k=25):
|
| 197 |
+
surah, ayah = kb.quran[seed]["surah_id"], kb.quran[seed]["ayah_id"]
|
| 198 |
+
ayahs = kb.quran_by_surah[surah]
|
| 199 |
+
min_ayah, max_ayah = min(ayahs), max(ayahs)
|
| 200 |
+
for offset in range(3):
|
| 201 |
+
start = ayah - offset
|
| 202 |
+
if start < min_ayah:
|
| 203 |
+
continue
|
| 204 |
+
window_len = -1
|
| 205 |
+
for length in range(1, self.cfg.max_window + 1):
|
| 206 |
+
end = start + length - 1
|
| 207 |
+
if end > max_ayah:
|
| 208 |
+
break
|
| 209 |
+
window_len += len(kb.q_norm_match[ayahs[end]]) + 1
|
| 210 |
+
len_diff = abs(window_len - query_len)
|
| 211 |
+
upper_bound = min(1.0, window_len / max(query_len, 1))
|
| 212 |
+
if best is not None: # exact-result pruning
|
| 213 |
+
best_cov, best_neg = best[0][0], best[0][1]
|
| 214 |
+
if upper_bound < best_cov or (upper_bound == best_cov and -len_diff < best_neg):
|
| 215 |
+
continue
|
| 216 |
+
key_pos = (surah, start, end)
|
| 217 |
+
if key_pos in memo:
|
| 218 |
+
continue
|
| 219 |
+
window = " ".join(kb.q_norm_match[ayahs[a]] for a in range(start, end + 1))
|
| 220 |
+
matcher = difflib.SequenceMatcher(None, query_norm, window, autojunk=False)
|
| 221 |
+
matched = sum(b.size for b in matcher.get_matching_blocks() if b.size >= 4)
|
| 222 |
+
coverage = matched / max(query_len, 1)
|
| 223 |
+
key = (coverage, -len_diff, matcher.ratio())
|
| 224 |
+
memo[key_pos] = key
|
| 225 |
+
if best is None or key > best[0]:
|
| 226 |
+
best = (key, coverage, key[2], surah, start, end)
|
| 227 |
+
if best is None:
|
| 228 |
+
return None
|
| 229 |
+
_, coverage, ratio, surah, start, end = best
|
| 230 |
+
display = " ".join(kb.quran[kb.quran_by_surah[surah][a]]["text"] for a in range(start, end + 1))
|
| 231 |
+
return CorrectionMatch(
|
| 232 |
+
"Ayah", coverage, ratio, self._ayah_text(surah, start, end),
|
| 233 |
+
{"type": "Quran", "surah_id": surah, "surah_name": kb.quran[kb.quran_by_surah[surah][start]]["surah_name"],
|
| 234 |
+
"ayah_start": start, "ayah_end": end},
|
| 235 |
+
display,
|
| 236 |
+
)
|
| 237 |
+
|
| 238 |
+
def _ayah_text(self, surah: int, start: int, end: int) -> str:
|
| 239 |
+
kb, multi = self.kb, end > start
|
| 240 |
+
parts = []
|
| 241 |
+
for a in range(start, end + 1):
|
| 242 |
+
text = kb.quran[kb.quran_by_surah[surah][a]]["text"]
|
| 243 |
+
parts.append(f"{text} ({a})" if multi else text)
|
| 244 |
+
return apply_idgham(" ".join(parts)).replace("\u0640", "")
|
| 245 |
+
|
| 246 |
+
def match_hadith(self, query_text: str) -> Optional[CorrectionMatch]:
|
| 247 |
+
kb = self.kb
|
| 248 |
+
query_norm = normalize_for_matching(query_text)
|
| 249 |
+
query_words = content_words(query_norm.split())
|
| 250 |
+
if not query_words:
|
| 251 |
+
return None
|
| 252 |
+
query_len = len(query_norm)
|
| 253 |
+
best = None # (key, idx, field, ratio)
|
| 254 |
+
for idx in kb.hadith_candidates(query_words, self.cfg.hadith_top_k):
|
| 255 |
+
for field_name in ("matn", "full"):
|
| 256 |
+
text = kb.hadith_norm(idx, field_name)
|
| 257 |
+
if not text:
|
| 258 |
+
continue
|
| 259 |
+
upper_bound = min(1.0, query_len / max(len(text), 1))
|
| 260 |
+
if best is not None and upper_bound < best[0][0]:
|
| 261 |
+
continue
|
| 262 |
+
matcher = difflib.SequenceMatcher(None, query_norm, text, autojunk=False)
|
| 263 |
+
matched = sum(b.size for b in matcher.get_matching_blocks() if b.size >= 4)
|
| 264 |
+
coverage, candidate_cov = matched / max(query_len, 1), matched / max(len(text), 1)
|
| 265 |
+
key = (min(coverage, candidate_cov), matcher.ratio())
|
| 266 |
+
if best is None or key > best[0]:
|
| 267 |
+
best = (key, idx, field_name, key[1])
|
| 268 |
+
if best is None:
|
| 269 |
+
return None
|
| 270 |
+
key, idx, field_name, ratio = best
|
| 271 |
+
record = kb.hadith[idx]
|
| 272 |
+
text = record[field_name].strip()
|
| 273 |
+
return CorrectionMatch(
|
| 274 |
+
"Hadith", key[0], ratio, text,
|
| 275 |
+
{"type": "Hadith", "hadithID": record["hadithID"], "book": record["book"], "title": record["title"],
|
| 276 |
+
"field": "matn" if field_name == "matn" else "full_text"},
|
| 277 |
+
text,
|
| 278 |
+
)
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
# --------------------------------------------------------------------------------------------------------------
|
| 282 |
+
# End-to-end pipeline
|
| 283 |
+
# --------------------------------------------------------------------------------------------------------------
|
| 284 |
+
STATUS_INFO = {
|
| 285 |
+
"VERIFIED": {"ar": "موثّق", "group": "verified"},
|
| 286 |
+
"CORRECTED": {"ar": "غير مطابق — يوجد تصحيح من المصدر", "group": "mismatch"},
|
| 287 |
+
"UNSUPPORTED": {"ar": "غير مطابق — لا يوجد مصدر مطابق", "group": "mismatch"},
|
| 288 |
+
"HUMAN_REVIEW": {"ar": "يحتاج مراجعة بشرية", "group": "review"},
|
| 289 |
+
}
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
class IslamicContentVerifier:
|
| 293 |
+
"""Detect quotations, verify them against the corpora and decide: verified, corrected, unsupported or review."""
|
| 294 |
+
|
| 295 |
+
def __init__(self, retriever: Optional[SourceRetriever] = None, config: Optional[PipelineConfig] = None,
|
| 296 |
+
use_scanner: bool = True) -> None:
|
| 297 |
+
self.cfg = config or PipelineConfig()
|
| 298 |
+
self.retriever = retriever or SourceRetriever()
|
| 299 |
+
self.verifier = Verifier(self.retriever, self.cfg.verifier)
|
| 300 |
+
self.corrector = Corrector(self.retriever, self.cfg.corrector)
|
| 301 |
+
self.detector = HybridDetector(self.retriever, use_scanner)
|
| 302 |
+
self.detector_name = type(self.detector).__name__
|
| 303 |
+
|
| 304 |
+
# ---- public API -----------------------------------------------------------------------------------------
|
| 305 |
+
def detect(self, text: str) -> List[DetectedSpan]:
|
| 306 |
+
return self.detector.detect(self._validate(text)) if text.strip() else []
|
| 307 |
+
|
| 308 |
+
def needs_hadith(self, text: str) -> bool:
|
| 309 |
+
"""True if the text contains a Hadith quotation (so the large Hadith index must be loaded)."""
|
| 310 |
+
return any(span.label == "Hadith" for span in self.detect(text))
|
| 311 |
+
|
| 312 |
+
def analyze(self, text: str) -> dict:
|
| 313 |
+
"""Run the full pipeline on a generated text."""
|
| 314 |
+
text = self._validate(text)
|
| 315 |
+
started = time.time()
|
| 316 |
+
spans = self.detector.detect(text) if text.strip() else []
|
| 317 |
+
detect_seconds = time.time() - started
|
| 318 |
+
result = self._analyze_spans(text, spans)
|
| 319 |
+
result["timings"] = {"detect_s": round(detect_seconds, 3), "total_s": round(time.time() - started, 3)}
|
| 320 |
+
return result
|
| 321 |
+
|
| 322 |
+
def analyze_spans(self, text: str, spans: List[dict]) -> dict:
|
| 323 |
+
"""Skip detection and use given spans ``[{label, start, end}]`` (evaluation / oracle mode)."""
|
| 324 |
+
given = [DetectedSpan(s["start"], s["end"], s["label"], None, "given", text[s["start"]:s["end"]]) for s in spans]
|
| 325 |
+
return self._analyze_spans(self._validate(text), given)
|
| 326 |
+
|
| 327 |
+
# ---- internals ------------------------------------------------------------------------------------------
|
| 328 |
+
@staticmethod
|
| 329 |
+
def _validate(text: str) -> str:
|
| 330 |
+
if not isinstance(text, str):
|
| 331 |
+
raise TypeError("Input text must be a string")
|
| 332 |
+
if len(text) > MAX_INPUT_CHARS:
|
| 333 |
+
raise ValueError(f"Input is too long ({len(text)} characters); the limit is {MAX_INPUT_CHARS}")
|
| 334 |
+
return text
|
| 335 |
+
|
| 336 |
+
def _analyze_spans(self, text: str, spans: List[DetectedSpan]) -> dict:
|
| 337 |
+
reports = [self._process_span(i + 1, span) for i, span in enumerate(sorted(spans, key=lambda s: s.start))]
|
| 338 |
+
counts = {status: 0 for status in STATUS_INFO}
|
| 339 |
+
for report in reports:
|
| 340 |
+
counts[report["status"]] += 1
|
| 341 |
+
return {
|
| 342 |
+
"input_text": text,
|
| 343 |
+
"detector": self.detector_name,
|
| 344 |
+
"spans": reports,
|
| 345 |
+
"corrected_text": self._apply_corrections(text, reports),
|
| 346 |
+
"summary": {
|
| 347 |
+
"n_spans": len(reports),
|
| 348 |
+
"n_ayah": sum(r["type"] == "Ayah" for r in reports),
|
| 349 |
+
"n_hadith": sum(r["type"] == "Hadith" for r in reports),
|
| 350 |
+
**counts,
|
| 351 |
+
"needs_human_review": counts["HUMAN_REVIEW"] > 0,
|
| 352 |
+
},
|
| 353 |
+
}
|
| 354 |
+
|
| 355 |
+
def _process_span(self, index: int, span: DetectedSpan) -> dict:
|
| 356 |
+
try:
|
| 357 |
+
return self._decide(index, span)
|
| 358 |
+
except Exception: # a single failing quotation must not break the whole report
|
| 359 |
+
logger.exception("Failed to process span %d", index)
|
| 360 |
+
report = self._empty_report(index, span)
|
| 361 |
+
self._finalize(report, "HUMAN_REVIEW", {"code": "internal_error"})
|
| 362 |
+
return report
|
| 363 |
+
|
| 364 |
+
@staticmethod
|
| 365 |
+
def _empty_report(index: int, span: DetectedSpan) -> dict:
|
| 366 |
+
return {
|
| 367 |
+
"id": index, "type": span.label, "start": span.start, "end": span.end, "text": span.text,
|
| 368 |
+
"detection": {"backend": span.source, "confidence": None if span.confidence is None else round(span.confidence, 4)},
|
| 369 |
+
"verification": {"verdict": "Incorrect", "confidence": 0.0, "score": 0.0, "method": "error", "n_candidates": 0},
|
| 370 |
+
"evidence": None, "correction": None, "suggestion": None, "notes": [],
|
| 371 |
+
}
|
| 372 |
+
|
| 373 |
+
@staticmethod
|
| 374 |
+
def _finalize(report: dict, status: str, reason: dict) -> None:
|
| 375 |
+
info = STATUS_INFO[status]
|
| 376 |
+
report.update(status=status, status_ar=info["ar"], group=info["group"], reason=reason)
|
| 377 |
+
|
| 378 |
+
@staticmethod
|
| 379 |
+
def source_label(source: dict) -> str:
|
| 380 |
+
"""Plain reference used in reasons, e.g. ``سورة البقرة 153`` or ``حديث رقم 5``."""
|
| 381 |
+
if source["type"] == "Quran":
|
| 382 |
+
start, end = source["ayah_start"], source["ayah_end"]
|
| 383 |
+
return f"سورة {source['surah_name']} {start}" + (f"–{end}" if end != start else "")
|
| 384 |
+
return f"حديث رقم {source['hadithID']}"
|
| 385 |
+
|
| 386 |
+
def _decide(self, index: int, span: DetectedSpan) -> dict:
|
| 387 |
+
report = self._empty_report(index, span)
|
| 388 |
+
verification = self.verifier.verify(span.text, span.label)
|
| 389 |
+
report["verification"] = {
|
| 390 |
+
"verdict": verification.verdict, "confidence": verification.confidence, "score": verification.best_score,
|
| 391 |
+
"method": verification.method, "n_candidates": verification.n_candidates,
|
| 392 |
+
}
|
| 393 |
+
if span.label == "Ayah":
|
| 394 |
+
self._decide_quran(report, span, verification)
|
| 395 |
+
else:
|
| 396 |
+
self._decide_hadith(report, span, verification)
|
| 397 |
+
return report
|
| 398 |
+
|
| 399 |
+
@staticmethod
|
| 400 |
+
def _proposal(span: DetectedSpan, match: Optional[CorrectionMatch]) -> Optional[dict]:
|
| 401 |
+
if match is None:
|
| 402 |
+
return None
|
| 403 |
+
return {"text": match.text, "display_text": match.display, "source": match.source,
|
| 404 |
+
"match_strength": round(match.strength, 4), "full_ratio": round(match.full_ratio, 4)}
|
| 405 |
+
|
| 406 |
+
def _decide_quran(self, report: dict, span: DetectedSpan, verification: Verification) -> None:
|
| 407 |
+
cfg, corr_cfg = self.cfg, self.cfg.corrector
|
| 408 |
+
match = self.corrector.match_quran(span.text)
|
| 409 |
+
if match is not None:
|
| 410 |
+
source_text, source_ref = match.display, match.source
|
| 411 |
+
elif verification.source:
|
| 412 |
+
candidate = verification.source
|
| 413 |
+
source_text = candidate["text"]
|
| 414 |
+
source_ref = {"type": "Quran", "surah_id": candidate["surah_id"], "surah_name": candidate["surah_name"],
|
| 415 |
+
"ayah_start": candidate["ayah_id"], "ayah_end": candidate["ayah_id"]}
|
| 416 |
+
else:
|
| 417 |
+
source_text, source_ref = "", None
|
| 418 |
+
alignment = align(span.text, source_text, self.retriever.quran_vocabulary) if source_text else None
|
| 419 |
+
if source_ref:
|
| 420 |
+
report["evidence"] = {"source": source_ref, "signals": compute_signals(span.text, source_text, "Ayah"),
|
| 421 |
+
"comparison": alignment}
|
| 422 |
+
proposal = self._proposal(span, match)
|
| 423 |
+
n_tokens = len(content_words(normalize_for_matching(span.text).split())) if span.text else 0
|
| 424 |
+
has_tokens = alignment is not None and alignment["exact"] and len(alignment["word_diff"]) >= 1
|
| 425 |
+
|
| 426 |
+
if has_tokens and sum(len(op["span"].split()) for op in alignment["word_diff"]) >= MIN_EXACT_TOKENS:
|
| 427 |
+
self._finalize(report, "VERIFIED", {"code": "exact_match", "source": self.source_label(source_ref)})
|
| 428 |
+
if alignment["diacritic_notes"]:
|
| 429 |
+
report["notes"].append({"code": "diacritic_conflict", "n": len(alignment["diacritic_notes"])})
|
| 430 |
+
if alignment["orthographic_variants"]:
|
| 431 |
+
report["notes"].append({"code": "orthographic_variant", "n": alignment["orthographic_variants"]})
|
| 432 |
+
return
|
| 433 |
+
|
| 434 |
+
strong = match is not None and match.strength >= corr_cfg.quran_strong and match.full_ratio >= corr_cfg.min_full_ratio
|
| 435 |
+
if strong:
|
| 436 |
+
self._finalize(report, "CORRECTED", {"code": "altered_passage", "source": self.source_label(source_ref),
|
| 437 |
+
"n": alignment["mismatches"] if alignment else 0,
|
| 438 |
+
"reordered": bool(alignment and alignment["reordered"])})
|
| 439 |
+
# replace only the passage the author quoted, not the whole ayah that contains it
|
| 440 |
+
excerpt = alignment["source_excerpt"] if alignment and alignment["source_excerpt"] else proposal["display_text"]
|
| 441 |
+
report["correction"] = {**proposal, "display_text": excerpt, "applied": True}
|
| 442 |
+
return
|
| 443 |
+
|
| 444 |
+
if verification.verdict == "Correct":
|
| 445 |
+
self._finalize(report, "HUMAN_REVIEW", {"code": "weak_match"})
|
| 446 |
+
report["suggestion"] = proposal
|
| 447 |
+
elif match is None or match.strength < corr_cfg.quran_low:
|
| 448 |
+
confident = verification.confidence >= cfg.unsupported_min_conf and not verification.method.startswith("borderline")
|
| 449 |
+
if match is None or match.strength < cfg.unsupported_strength or confident:
|
| 450 |
+
self._finalize(report, "UNSUPPORTED", {"code": "no_source"})
|
| 451 |
+
else:
|
| 452 |
+
self._finalize(report, "HUMAN_REVIEW", {"code": "insufficient_evidence"})
|
| 453 |
+
report["suggestion"] = proposal
|
| 454 |
+
else:
|
| 455 |
+
self._finalize(report, "HUMAN_REVIEW", {"code": "candidate_not_strong", "source": self.source_label(source_ref),
|
| 456 |
+
"strength": round(match.strength, 2)})
|
| 457 |
+
report["suggestion"] = proposal
|
| 458 |
+
|
| 459 |
+
def _decide_hadith(self, report: dict, span: DetectedSpan, verification: Verification) -> None:
|
| 460 |
+
cfg, corr_cfg = self.cfg, self.cfg.corrector
|
| 461 |
+
best = verification.source
|
| 462 |
+
alignment = None
|
| 463 |
+
if best:
|
| 464 |
+
alignment = align(span.text, best["text"])
|
| 465 |
+
report["evidence"] = {
|
| 466 |
+
"source": {"type": "Hadith", "hadithID": best["hadithID"], "book": best["book"], "title": best["title"]},
|
| 467 |
+
"signals": best.get("signals"), "comparison": alignment,
|
| 468 |
+
}
|
| 469 |
+
exact_tokens = alignment is not None and alignment["exact"] and sum(len(op["span"].split()) for op in alignment["word_diff"]) >= 4
|
| 470 |
+
|
| 471 |
+
if verification.verdict == "Correct" or exact_tokens:
|
| 472 |
+
if exact_tokens or (not verification.method.startswith("borderline") and verification.confidence >= cfg.verified_min_conf):
|
| 473 |
+
self._finalize(report, "VERIFIED", {"code": "exact_match" if exact_tokens else "close_match",
|
| 474 |
+
"source": f"حديث رقم {best['hadithID']}"})
|
| 475 |
+
if alignment and not alignment["exact"]:
|
| 476 |
+
report["notes"].append({"code": "hadith_minor_diffs", "n": alignment["mismatches"]})
|
| 477 |
+
if alignment and alignment["diacritic_notes"]:
|
| 478 |
+
report["notes"].append({"code": "diacritic_conflict", "n": len(alignment["diacritic_notes"])})
|
| 479 |
+
else:
|
| 480 |
+
self._finalize(report, "HUMAN_REVIEW", {"code": "weak_match"})
|
| 481 |
+
return
|
| 482 |
+
|
| 483 |
+
match = self.corrector.match_hadith(span.text)
|
| 484 |
+
proposal = self._proposal(span, match)
|
| 485 |
+
if match is None or match.strength < corr_cfg.hadith_low:
|
| 486 |
+
confident = verification.confidence >= cfg.unsupported_min_conf and not verification.method.startswith("borderline")
|
| 487 |
+
if match is None or match.strength < cfg.unsupported_strength or confident:
|
| 488 |
+
self._finalize(report, "UNSUPPORTED", {"code": "no_source"})
|
| 489 |
+
else:
|
| 490 |
+
self._finalize(report, "HUMAN_REVIEW", {"code": "insufficient_evidence"})
|
| 491 |
+
report["suggestion"] = proposal
|
| 492 |
+
else: # a similar narration exists, but Hadith is never corrected automatically
|
| 493 |
+
self._finalize(report, "HUMAN_REVIEW", {"code": "hadith_candidate", "source": self.source_label(match.source),
|
| 494 |
+
"strength": round(match.strength, 2)})
|
| 495 |
+
report["suggestion"] = proposal
|
| 496 |
+
|
| 497 |
+
@staticmethod
|
| 498 |
+
def _apply_corrections(text: str, reports: List[dict]) -> str:
|
| 499 |
+
out = text
|
| 500 |
+
for report in sorted(reports, key=lambda r: r["start"], reverse=True):
|
| 501 |
+
if report["status"] == "CORRECTED" and report["correction"] and report["correction"].get("applied"):
|
| 502 |
+
out = out[: report["start"]] + report["correction"]["display_text"] + out[report["end"]:]
|
| 503 |
+
return out
|
worker.js
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/* Python (Pyodide) and the whole verification pipeline run here, off the main thread. */
|
| 2 |
+
let py = null, hadithReady = null;
|
| 3 |
+
const post = (m) => self.postMessage(m);
|
| 4 |
+
const status = (text, step) => post({ type: "status", text, step });
|
| 5 |
+
|
| 6 |
+
async function fetchBytes(url, onProgress) {
|
| 7 |
+
let cache = null;
|
| 8 |
+
try { cache = await caches.open("icv-v2"); const hit = await cache.match(url); if (hit) return new Uint8Array(await hit.arrayBuffer()); } catch (e) { cache = null; }
|
| 9 |
+
const response = await fetch(url);
|
| 10 |
+
if (!response.ok) throw new Error("تعذّر تحميل " + url + " (" + response.status + ")");
|
| 11 |
+
if (cache) { try { await cache.put(url, response.clone()); } catch (e) { /* optimisation only */ } }
|
| 12 |
+
const total = Number(response.headers.get("Content-Length")) || 0;
|
| 13 |
+
if (!response.body || !onProgress) return new Uint8Array(await response.arrayBuffer());
|
| 14 |
+
const reader = response.body.getReader(), chunks = []; let received = 0;
|
| 15 |
+
for (;;) { const { done, value } = await reader.read(); if (done) break; chunks.push(value); received += value.length; onProgress(received, total); }
|
| 16 |
+
const bytes = new Uint8Array(received); let offset = 0;
|
| 17 |
+
for (const c of chunks) { bytes.set(c, offset); offset += c.length; }
|
| 18 |
+
return bytes;
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
// Accept both layouts: files inside folders (index/, demo/) or uploaded flat next to index.html.
|
| 22 |
+
async function fetchAny(file, base, onProgress) {
|
| 23 |
+
let lastError = null;
|
| 24 |
+
for (const path of [file, file.split("/").pop()]) {
|
| 25 |
+
try { return await fetchBytes(new URL(path, base).href, onProgress); } catch (e) { lastError = e; }
|
| 26 |
+
}
|
| 27 |
+
throw lastError;
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
async function boot(m) {
|
| 31 |
+
status("جارٍ تحميل بيئة بايثون…", 1);
|
| 32 |
+
importScripts("https://cdn.jsdelivr.net/pyodide/v" + m.pyodideVersion + "/full/pyodide.js");
|
| 33 |
+
py = await loadPyodide();
|
| 34 |
+
status("جارٍ تحميل الكود وفهرس القرآن…", 2);
|
| 35 |
+
for (const dir of ["/app", "/app/index", "/app/demo", "/app/data"]) py.FS.mkdir(dir);
|
| 36 |
+
for (const file of m.files) py.FS.writeFile("/app/" + file, await fetchAny(file, m.base));
|
| 37 |
+
py.runPython("import sys; sys.path.insert(0, '/app')");
|
| 38 |
+
py.runPython("from app import get_pipeline, verify_text, verify_generated_answer, analyze_benchmark, detect_spans, verify_given_spans; get_pipeline()");
|
| 39 |
+
post({ type: "ready" });
|
| 40 |
+
hadithReady = loadHadith(m); // background; verification waits for it
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
async function loadHadith(m) {
|
| 44 |
+
const file = m.lazy[0];
|
| 45 |
+
const bytes = await fetchAny(file, m.base, (got, total) => {
|
| 46 |
+
const mb = (got / 1048576).toFixed(1), of = total ? " / " + (total / 1048576).toFixed(1) : "";
|
| 47 |
+
post({ type: "progress", got, total });
|
| 48 |
+
status("جارٍ تحميل كتب الحديث… " + mb + of + " ميغابايت", 3);
|
| 49 |
+
});
|
| 50 |
+
py.FS.writeFile("/app/" + file, bytes);
|
| 51 |
+
status("جارٍ تجهيز كتب الحديث…", 3);
|
| 52 |
+
py.runPython("get_pipeline().retriever.warm()");
|
| 53 |
+
post({ type: "hadith" });
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
self.onmessage = async (event) => {
|
| 57 |
+
const m = event.data;
|
| 58 |
+
if (m.type === "boot") {
|
| 59 |
+
try { await boot(m); } catch (e) { post({ type: "error", text: String((e && e.message) || e) }); }
|
| 60 |
+
return;
|
| 61 |
+
}
|
| 62 |
+
try {
|
| 63 |
+
await hadithReady;
|
| 64 |
+
const call = (name, ...args) => py.globals.get(name)(...args);
|
| 65 |
+
let out;
|
| 66 |
+
if (m.type === "verify") out = call(m.generated ? "verify_generated_answer" : "verify_text", m.text);
|
| 67 |
+
else if (m.type === "detect") out = call("detect_spans", m.text);
|
| 68 |
+
else if (m.type === "verify_spans") out = call("verify_given_spans", m.text, m.spans);
|
| 69 |
+
else if (m.type === "benchmark") out = call("analyze_benchmark", m.text);
|
| 70 |
+
post({ type: "result", id: m.id, value: out });
|
| 71 |
+
} catch (e) { post({ type: "result", id: m.id, value: null }); }
|
| 72 |
+
};
|