QModel v8.5: accuracy release (code, 3,598-fatwa index, quran.com links)

.env.example

@@ -1,4 +1,4 @@
-# QModel 6 Configuration Template
+# QModel 8.5 Configuration Template
 # ==================================
 # Copy this to .env and update values for your environment
 
@@ -9,33 +9,31 @@ LLM_BACKEND=gguf
 # ─────────────────────────────────────────────────────────────────────
 # GGUF BACKEND (if LLM_BACKEND=gguf) — self-contained, no external daemon
 # ─────────────────────────────────────────────────────────────────────
-GGUF_MODEL_PATH=./models/Qwen3.6-27B-Q4_K_M.gguf
-GGUF_N_CTX=4096              # Context window size
+GGUF_MODEL_PATH=./models/Qwen2.5-14B-Instruct-Q8_0.gguf
+GGUF_N_CTX=8192              # Context window size
 GGUF_N_GPU_LAYERS=-1         # -1 = offload all layers to GPU (Metal on Mac)
 
 # ─────────────────────────────────────────────────────────────────────
 # OLLAMA BACKEND (if LLM_BACKEND=ollama)
 # ─────────────────────────────────────────────────────────────────────
 # OLLAMA_HOST=http://localhost:11434
-# OLLAMA_MODEL=qwen3.6:27b
+# OLLAMA_MODEL=qwen2.5:14b
 # Available models: llama3.1, mistral, neural-chat, openhermes
 
 # ─────────────────────────────────────────────────────────────────────
 # HUGGINGFACE BACKEND (if LLM_BACKEND=hf)
 # ─────────────────────────────────────────────────────────────────────
-# HF_MODEL_NAME=Qwen/Qwen3-8B
+# HF_MODEL_NAME=Qwen/Qwen2.5-14B-Instruct
 # HF_DEVICE=auto              # Options: auto, cuda, cpu
 # HF_MAX_NEW_TOKENS=2048
 # Popular models:
-#   - Qwen/Qwen3-8B (excellent Arabic)
-#   - mistralai/Mistral-7B-Instruct-v0.2
-#   - meta-llama/Llama-2-13b-chat-hf
+#   - Qwen/Qwen2.5-14B-Instruct (recommended)
 
 # ─────────────────────────────────────────────────────────────────────
 # LM STUDIO BACKEND (if LLM_BACKEND=lmstudio)
 # ─────────────────────────────────────────────────────────────────────
 # LMSTUDIO_URL=http://localhost:1234
-# LMSTUDIO_MODEL=qwen2.5-7b-instruct   # Model loaded in LM Studio
+# LMSTUDIO_MODEL=qwen2.5-14b-instruct   # Model loaded in LM Studio
 
 # ─────────────────────────────────────────────────────────────────────
 # EMBEDDING MODEL (shared by all backends)
@@ -63,11 +61,21 @@ MAX_TOKENS=2048         # Max output length
 # ─────────────────────────────────────────────────────────────────────
 # SAFETY & QUALITY
 # ─────────────────────────────────────────────────────────────────────
-# Confidence threshold: Below this score, skip LLM and return "not found"
-# Prevents hallucinations but may miss valid results
-# Range: 0.0-1.0 (default 0.30)
-# Tune up (0.50+) for stricter, tune down (0.20) for looser
-CONFIDENCE_THRESHOLD=0.30
+# Confidence gate: skip the LLM and return "not found" unless retrieval found
+# something genuinely relevant (direct reference / verbatim text match, or):
+# - USE_RERANKER=true  → best cross-encoder relevance (0-1) >= RERANK_MIN_SCORE
+# - USE_RERANKER=false → best dense cosine similarity >= DENSE_MIN_SCORE
+#   (multilingual-e5-large scale; recalibrate if EMBED_MODEL changes)
+# Calibrate with: python -m eval.run_eval --show-failures
+# On the golden set the reranker gate refuses 92% of off-topic questions vs 36% for
+# the dense gate, for ~1 s per answer; set false only where the model can't be loaded.
+USE_RERANKER=true
+
+# Rewrite queries without the LLM (regex intent + thesaurus terms), ~2 s faster per
+# question. Follow-ups with chat history always use the LLM rewrite.
+FAST_QUERY_REWRITE=true
+RERANK_MIN_SCORE=0.05
+DENSE_MIN_SCORE=0.80
 
 # Hadith boost: Score bonus when intent=hadith
 # Prevents Quran verses from outranking relevant Hadiths
@@ -99,19 +107,19 @@ ALLOWED_ORIGINS=*       # CORS origins (restrict in production: origin1.com,orig
 #
 # Self-contained (GGUF, e.g. HF Spaces):
 #   LLM_BACKEND=gguf
-#   GGUF_MODEL_PATH=./models/Qwen3.6-27B-Q4_K_M.gguf
+#   GGUF_MODEL_PATH=./models/Qwen2.5-14B-Instruct-Q8_0.gguf
 #
 # Development (Ollama):
 #   LLM_BACKEND=ollama
 #   OLLAMA_HOST=http://localhost:11434
-#   OLLAMA_MODEL=qwen3.6:27b
+#   OLLAMA_MODEL=qwen2.5:14b
 #
 # Production (HuggingFace GPU):
 #   LLM_BACKEND=hf
-#   HF_MODEL_NAME=Qwen/Qwen3-8B
+#   HF_MODEL_NAME=Qwen/Qwen2.5-14B-Instruct
 #   HF_DEVICE=cuda
 #
 # Production (HuggingFace CPU):
 #   LLM_BACKEND=hf
-#   HF_MODEL_NAME=Qwen/Qwen3-8B
+#   HF_MODEL_NAME=Qwen/Qwen2.5-14B-Instruct
 #   HF_DEVICE=cpu

.gitignore

@@ -214,3 +214,9 @@ metadata.json
 models/
 *.gguf
 scratch/
+
+# Eval run reports
+eval/results/
+QModel.fatwa-update.index
+metadata.fatwa-update.json
+*.prev-*

Dockerfile

@@ -1,5 +1,5 @@
 # name: logicsoft
-# QModel 7 - Islamic RAG API
+# QModel 8.5 - Islamic RAG API
 # =============================
 # Dockerfile for QModel API
 # Supports both Ollama and HuggingFace backends via .env configuration
@@ -42,8 +42,8 @@ FROM python:3.11-slim
 LABEL name="Logicsoft"
 LABEL author="Abdullah Elgendy"
 LABEL maintainer="Abdullah Elgendy"
-LABEL description="QModel v8 - Quran, Hadith, Tafsir and Fiqh RAG API"
-LABEL version="8.0.0"
+LABEL description="QModel v8.5 - Quran, Hadith, Tafsir and Fiqh RAG API"
+LABEL version="8.5.0"
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
@@ -72,8 +72,10 @@ USER appuser
 # Expose port for API
 EXPOSE 8000
 
-# Health check — /health returns 503 until the RAG pipeline finishes loading
-HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
+# Health check — /health returns 503 until the RAG pipeline finishes loading.
+# Loading takes ~100 s (embedding model, reranker, BM25 over 56k docs), longer on
+# first start while the models download.
+HEALTHCHECK --interval=30s --timeout=10s --start-period=300s --retries=3 \
     CMD curl -f http://localhost:8000/health || exit 1
 
 # Start application

README.md

@@ -19,12 +19,12 @@ language:
   - en
 ---
 
-# QModel v8.0 — Islamic RAG System
+# QModel v8.5 — Islamic RAG System
 **Specialized Qur'an, Hadith & Fatwa Knowledge System with Multi-Vector RAG & Dual LLM Support**
 
 > A production-ready Retrieval-Augmented Generation system specialized exclusively in authenticated Islamic knowledge. No hallucinations, no outside knowledge—only content from verified sources.
 
-![Version](https://img.shields.io/badge/version-8.0.0-blue)
+![Version](https://img.shields.io/badge/version-8.5.0-blue)
 ![Backend](https://img.shields.io/badge/backend-gguf%20%7C%20ollama%20%7C%20huggingface-green)
 ![Status](https://img.shields.io/badge/status-production--ready-success)
 
@@ -50,17 +50,17 @@ language:
 - **Collection Navigation**: Filter by Bukhari, Muslim, Abu Dawud, etc.
 
 ### ⚖️ Fatwa & Fiqh Capabilities
-- **Verified Scholarly Rulings**: Ingested fatwas from recognized scholars and councils (Bin Baz, Lajnah Da'imah, etc.)
+- **Verified Scholarly Rulings**: 3,598 fatwas from Islamweb's Fatwa Center (مركز الفتوى) and Imam Ibn Baz
 - **Semantic & Source Filtering**: Filter fatwas by scholar, council, or source
 - **Integrated RAG Context**: Rulings retrieved alongside Quran and Hadith evidence
 - **Direct Lookup**: Fast endpoint access via `/fatwa/{id}`, `/fatwa/search`, and `/fatwa/sources`
 
 ### 🛡️ Safety Features
-- **Dynamic Confidence Gating**: Intent-aware thresholds (stricter for authenticity)
+- **Relevance Gating**: The LLM is only called when retrieval found something genuinely relevant (exact reference/text match, else the cross-encoder's relevance score); out-of-scope requests get a fixed "not found" answer
 - **Reasoning Support**: Capture and stream model thinking/Chain-of-Thought
-- **Source Attribution**: Every answer cites exact verse, Hadith, or Fatwa reference
-- **Verbatim Quotes**: Text verified against raw sources with support for all quote formats (`«...»`, `“...”`, `❝...❞`, `"..."`)
-- **Anti-Hallucination**: Hardened prompts with pre/post-generation citation verification
+- **Source Attribution**: Every answer cites exact verse, Hadith, or Fatwa reference — hadith by collection and number, fatwas by source link (added automatically when the model omits it)
+- **Verbatim Scripture**: Quoted verses and hadith (`﴿...﴾`, `❝...❞`, `«...»`) are checked against the retrieved passages; near-copies are restored to the canonical text, ungrounded quotes are removed
+- **Language Guard**: Answers that drift into Chinese are regenerated (or cut off when streaming)
 
 ### 🚀 Integration
 - **OpenAI-Compatible API**: Use with Open-WebUI, Langchain, or any OpenAI client
@@ -73,7 +73,7 @@ language:
 - **Multi-Vector Dense Retrieval**: Independent Arabic and English query vectors fused adaptively
 - **Inverted BM25 Sparse Index**: Bilingual tokenization with stopword filtering and morphological root indexing
 - **Adaptive RRF Fusion**: Language-weighted Reciprocal Rank Fusion balancing dense & sparse signals
-- **Cross-Encoder Reranker**: Optional `BAAI/bge-reranker-v2-m3` rescoring
+- **Cross-Encoder Reranker**: `BAAI/bge-reranker-v2-m3` rescoring and confidence gating (on by default)
 - **Triple LLM Backend**: Local GGUF, Ollama (dev), or HuggingFace (prod)
 - **Async API & Caching**: FastAPI with TTL-based LRU response cache
 
@@ -90,8 +90,8 @@ language:
 ### Installation
 
 ```bash
-# Clone and enter project
-git clone https://github.com/Logicsoft/QModel.git && cd QModel
+# Download code + search index from Hugging Face (skips the 15 GB GGUF)
+huggingface-cli download aelgendy/QModel --exclude "models/*" --local-dir QModel && cd QModel
 python3 -m venv .venv && source .venv/bin/activate
 pip install -r requirements.txt
 
@@ -101,9 +101,10 @@ export LLM_BACKEND=gguf
 export GGUF_MODEL_PATH=./models/Qwen2.5-14B-Instruct-Q8_0.gguf
 huggingface-cli download bartowski/Qwen2.5-14B-Instruct-GGUF Qwen2.5-14B-Instruct-Q8_0.gguf --local-dir ./models
 
-# Option B — Ollama (local development):
+# Option B — Ollama:
+ollama pull hf.co/aelgendy/QModel
 export LLM_BACKEND=ollama
-export OLLAMA_MODEL=qwen2.5:14b
+export OLLAMA_MODEL=hf.co/aelgendy/QModel
 # Make sure Ollama is running: ollama serve
 
 # Option C — HuggingFace (production):
@@ -122,14 +123,23 @@ API docs: http://localhost:8000/docs
 ### Data & Index
 
 Pre-built data files are included:
-- `metadata.json` — 52,677 documents (6,236 Quran verses + 46,416 hadiths from 15 collections + Fatwas)
-- `QModel.index` — FAISS search index (205.77 MB, 52,677 vectors, 1024-d)
+- `metadata.json` — 56,250 documents (6,236 Quran verses + 46,416 hadiths from 15 collections + 3,598 fatwas)
+- `QModel.index` — FAISS search index (~220 MB, 56,250 vectors, 1024-d)
 
 To rebuild after dataset changes:
 ```bash
 python build_index.py
 ```
 
+Fatwas are read from `data/fatwas/` (hand-collected) and `data/fatwas_scraped/` (scraper output), de-duplicated by id and content; files listed in `FATWA_EXCLUDE` in `build_index.py` are skipped. To add fatwas without re-embedding the Qur'an and hadith:
+```bash
+python data_pipeline/scrape_fatwas.py --source islamweb --start 8001 --end 12000 --delay 1.0
+python scripts/update_fatwa_index.py              # writes QModel.fatwa-update.index / metadata.fatwa-update.json
+python scripts/update_fatwa_index.py --in-place   # or replace the live files (previous pair saved as .bak, overwriting older .bak)
+```
+
+Records without answer text, and scraped listing pages (e.g. audio-lecture indexes), are skipped.
+
 ---
 
 ## Example Queries
@@ -227,12 +237,16 @@ huggingface-cli download bartowski/Qwen2.5-14B-Instruct-GGUF Qwen2.5-14B-Instruc
 ### Ollama Backend (Development)
 
 ```bash
+ollama pull hf.co/aelgendy/QModel
+
 LLM_BACKEND=ollama
 OLLAMA_HOST=http://localhost:11434
-OLLAMA_MODEL=qwen2.5:14b
+OLLAMA_MODEL=hf.co/aelgendy/QModel
 ```
 
-Requires: `ollama serve` running and model pulled (`ollama pull qwen2.5:14b`).
+Requires `ollama serve` running. The pull brings the GGUF with QModel's chat template, system prompt and sampling settings (the `template`, `system` and `params` files, kept in sync with `Modelfile`).
+
+> `ollama run hf.co/aelgendy/QModel` on its own gives the bare language model, without retrieval, citation checks or verse links. For grounded answers, run the QModel API with `LLM_BACKEND=ollama` and send questions to it (or to Open-WebUI connected to it).
 
 ### HuggingFace Backend (Production)
 
@@ -265,7 +279,8 @@ HF_MAX_NEW_TOKENS=2048
 | `TOP_K_SEARCH` | `20` | Candidate pool (5–100) |
 | `TOP_K_RETURN` | `5` | Results shown to user (1–20) |
 | `RERANK_ALPHA` | `0.6` | Dense vs Sparse weight (0.0–1.0) |
-| `USE_RERANKER` | `false` | Enable cross-encoder reranker (true/false) |
+| `FAST_QUERY_REWRITE` | `true` | Rewrite queries without the LLM (regex intent + thesaurus terms), saving ~2 s per question. The LLM rewrite still runs for follow-up questions with chat history |
+| `USE_RERANKER` | `true` | Cross-encoder reranker (true/false). Also drives the confidence gate: off-topic questions refused 92% with it vs 36% without |
 | `RERANK_MODEL` | `BAAI/bge-reranker-v2-m3` | Cross-encoder model ID |
 | `AYAH_WINDOW_SIZE` | `1` | Number of neighboring verses for context expansion |
 | `ENABLE_HADITH_DEDUPLICATION` | `true` | Cluster near-identical Hadith narrations |
@@ -275,7 +290,8 @@ HF_MAX_NEW_TOKENS=2048
 | `MAX_TOKENS` | `2048` | Max response length |
 | `SHOW_THINKING` | `false` | Include chain-of-thought/reasoning output (`true`/`false`) |
 | **Safety** | | |
-| `CONFIDENCE_THRESHOLD` | `0.30` | Min score to call LLM (higher = fewer hallucinations) |
+| `RERANK_MIN_SCORE` | `0.05` | Confidence gate with reranker on: min cross-encoder relevance (0–1) of the best retrieved passage to call the LLM |
+| `DENSE_MIN_SCORE` | `0.80` | Confidence gate with reranker off: min dense cosine similarity (multilingual-e5-large scale) |
 | `HADITH_BOOST` | `0.08` | Score boost for hadith on hadith queries |
 | `QURAN_BOOST` | `0.12` | Score boost for Quran verses on tafsir/quran queries |
 | **Other** | | |
@@ -293,16 +309,14 @@ GGUF_MODEL_PATH=./models/Qwen2.5-14B-Instruct-Q8_0.gguf
 GGUF_N_CTX=8192
 GGUF_N_GPU_LAYERS=-1
 TEMPERATURE=0.2
-CONFIDENCE_THRESHOLD=0.30
 ```
 
 **Development (Ollama)**
 ```bash
 LLM_BACKEND=ollama
 OLLAMA_HOST=http://localhost:11434
-OLLAMA_MODEL=qwen2.5:14b
+OLLAMA_MODEL=hf.co/aelgendy/QModel
 TEMPERATURE=0.2
-CONFIDENCE_THRESHOLD=0.30
 ALLOWED_ORIGINS=*
 ```
 
@@ -313,15 +327,47 @@ HF_MODEL_NAME=Qwen/Qwen2.5-14B-Instruct
 HF_DEVICE=cuda
 TOP_K_SEARCH=30
 TEMPERATURE=0.1
-CONFIDENCE_THRESHOLD=0.35
+USE_RERANKER=true
 ALLOWED_ORIGINS=yourdomain.com,api.yourdomain.com
 ```
 
 ### Tuning Tips
 
-- **Better results**: Increase `TOP_K_SEARCH`, lower `CONFIDENCE_THRESHOLD`, use `TEMPERATURE=0.1`
+- **Better results**: keep `USE_RERANKER=true` (the default: better ranking and a far more reliable confidence gate, ~1 s per answer), increase `TOP_K_SEARCH`, use `TEMPERATURE=0.1`
 - **Faster performance**: Lower `TOP_K_SEARCH` and `TOP_K_RETURN`, reduce `MAX_TOKENS`, use Ollama
-- **More conservative**: Increase `CONFIDENCE_THRESHOLD`, lower `TEMPERATURE`
+- **More conservative**: Increase `RERANK_MIN_SCORE` / `DENSE_MIN_SCORE`, lower `TEMPERATURE` — check the trade-off with `python -m eval.run_eval --show-failures`
+
+### Measuring Retrieval Accuracy
+
+`eval/golden.jsonl` holds 147 labelled queries (Qur'an topic/text/reference, Hadith topic/text/authenticity, Fatwa, and out-of-scope negatives). Each case lists the dataset `id`s that count as a correct hit; where one source is decisive (5:90 for "is alcohol forbidden?", while 2:219 alone would mislead), it is also listed under `required` and scored as `complete@k`. `eval/run_eval.py` runs them through retrieval (no LLM generation) using the local index:
+
+```bash
+python -m eval.run_eval --out eval/results/baseline.json        # full pipeline, k = TOP_K_RETURN
+python -m eval.run_eval --mode hybrid --k 20                     # hybrid_search() alone
+python -m eval.run_eval --only fatwa,negative --show-failures    # subset + list misses
+python -m eval.run_eval --compare eval/results/baseline.json     # diff against a previous run
+```
+
+It reports Hit@1/3/k, MRR and Recall@k overall, by category and by language, plus the confidence gate's false-reject rate (answer found but refused) and false-accept rate (out-of-scope queries let through). Run it before and after any retrieval or config change; env vars such as `USE_RERANKER=false` or `TOP_K_SEARCH=100` apply as usual.
+
+### Measuring Answer Quality
+
+`eval/run_answers.py` runs the same golden set through the full pipeline **with the LLM** and checks what users read:
+
+```bash
+python -m eval.run_answers --limit 5                           # smoke test
+python -m eval.run_answers --only hadith_text,fatwa            # subset
+python -m eval.run_answers --out eval/results/answers.json     # full run (~1 h at ~25 s/answer)
+```
+
+- **Refusals**: out-of-scope questions get the "not found" answer; real ones don't
+- **Citations**: every cited verse / hadith / fatwa was retrieved, at least one is a golden-relevant source, and all `required` sources are cited
+- **Quotes**: scripture quoted by the model, before verification — verbatim, altered copy of a real verse, or fabricated — and how many the verifier restored or removed
+- **Language**: answer matches the question's language, no CJK leakage
+
+Every answer is saved in the report for manual review.
+
+The labels themselves need expert review: `eval/label_review.csv` lists every golden label (question, labelled source and its text) with columns for a reviewer's verdict. Categories with 5–11 cases move 10–20 points on a single case, so read per-category numbers with that in mind.
 
 ---
 
@@ -384,7 +430,7 @@ this backend suitable for bundling directly into an HF Space.
 # .env
 LLM_BACKEND=ollama
 OLLAMA_HOST=http://host.docker.internal:11434
-OLLAMA_MODEL=qwen2.5:14b
+OLLAMA_MODEL=hf.co/aelgendy/QModel
 ```
 
 Requires Ollama running on the host (`ollama serve`).
@@ -537,21 +583,22 @@ tests/
 1. **Conversational Coreference Rewriting**: Query rewriter accepts dialogue `history` to resolve ambiguous pronouns and anaphoric references into standalone topical queries.
 2. **Dual-Vector Query Embedding**: Encodes both Arabic (`query_ar`) and English (`query_en`) query vectors using `multilingual-e5-large` with the mandatory `query: ` prefix.
 3. **Multi-Vector Dense Retrieval**: Independent FAISS searches against Arabic and English dense vectors.
-4. **Inverted BM25 Sparse Retrieval**: BM25 scoring with bilingual stopword removal and Quranic morphological root expansion.
+4. **Inverted BM25 Sparse Retrieval**: BM25 scoring with bilingual stopword removal and Quranic morphological root expansion. The light stemmer only strips an affix when at least 3 letters remain, so short words keep their root letters (فضل, الربا).
 5. **Adaptive Reciprocal Rank Fusion (RRF)**: Merges Arabic dense, English dense, and BM25 sparse hits with dynamic language-weighted multipliers.
 6. **Hadith Deduplication & Ayah Windowing**:
    - Clusters duplicate Hadith narrations ($\ge 0.85$ text similarity), tags *Muttafaq 'Alayh* (Bukhari + Muslim), and aggregates alternative collection references.
    - Expands top Quran verses with $\pm N$ adjacent verses (`AYAH_WINDOW_SIZE`) for complete theological context.
-7. **Cross-Encoder Reranking (Optional)**: Rescores top candidates using `BAAI/bge-reranker-v2-m3` when `USE_RERANKER=true`.
-8. **Confidence Gating & Context Assembly**: Gated by intent-aware confidence thresholds before assembling verified, citation-attributed LLM context.
+7. **Cross-Encoder Reranking**: Rescores the top 25 candidates using `BAAI/bge-reranker-v2-m3` (`USE_RERANKER`, on by default). Five of those slots are reserved for the best BM25-only hits, which RRF would otherwise rank below anything both dense searches found. Each passage is scored with its text in the query's language first, since the reranker truncates at 512 tokens and a long diacritized verse fills that on its own.
+8. **Confidence Gating & Context Assembly**: Direct references and verbatim text matches pass; otherwise the best reranker relevance must reach `RERANK_MIN_SCORE` (or dense similarity `DENSE_MIN_SCORE` without the reranker). Context carries copyable citation lines: `[REF]` (verse + URLs), `[HADITH_REF]` (collection and number in English and Arabic, grade), `[FATWA_REF]` (source, scholar, URL).
 
 ### Anti-Hallucination Measures
 
-- Few-shot examples including "not found" refusal path
-- Hardcoded citation format rules
-- Verbatim copy rules (no text reconstruction)
-- Confidence threshold gating (default: 0.30)
-- Post-generation citation verification
+- Relevance gate before generation (see step 8 above)
+- Citation format rules and copyable `[REF]` / `[HADITH_REF]` / `[FATWA_REF]` lines in the context
+- Scope rule: non-Islamic requests (poems, sports, trivia) get the fixed "not found" answer; the model must either answer from the context or refuse, never both
+- Quote verification: a quote must appear in a retrieved passage; a near-copy of a retrieved verse or hadith (≥80% of its consonant skeleton) is replaced by the canonical text, anything else by a warning marker. LLMs corrupt scripture while copying it (foreign letters, changed word forms, truncation), so this runs on every answer, streamed or not
+- Fatwa links: when the answer uses a retrieved fatwa (by title or by naming its source) without linking it, the link is appended
+- CJK guard: an answer containing Chinese/Japanese/Korean text is regenerated once at temperature 0, then replaced by a list of the retrieved sources; when streaming, CJK is stripped and generation stops after sustained drift
 - Grade inference from collection name
 
 ### Performance
@@ -587,7 +634,7 @@ export HF_TOKEN=hf_xxxxxxxxxxxxx  # Set token for gated models
 ### "Assistant returns 'Not found'"
 This is expected — QModel rejects low-confidence queries. Try:
 - More specific queries
-- Lower `CONFIDENCE_THRESHOLD` in `.env`
+- Lower `RERANK_MIN_SCORE` (reranker on) or `DENSE_MIN_SCORE` (reranker off) in `.env`
 - Check raw scores: `GET /debug/scores?q=your+query`
 
 ### "Port already in use"
@@ -600,21 +647,6 @@ docker-compose down && docker system prune
 
 ## Roadmap
 
-- [x] Grade-based filtering
-- [x] Streaming responses (SSE)
-- [x] Modular architecture (5 routers, 19 endpoints)
-- [x] Triple LLM backend (GGUF + Ollama + HuggingFace)
-- [x] Text search (exact substring + fuzzy matching)
-- [x] Chain of narrators (Isnad highlighting)
-- [x] Synonym expansion (curated Islamic thesaurus)
-- [x] Root-word analysis (thematic Arabic roots & dictionary)
-- [x] Fatwa & scholarly rulings corpus integration
-- [x] Asymmetric E5 embedding prefixing (`passage: ` / `query: `)
-- [x] Multi-vector dual-language dense retrieval with adaptive RRF
-- [x] Conversational multi-turn coreference resolution
-- [x] Hadith matn deduplication & *Muttafaq 'Alayh* clustering
-- [x] Quranic parent-document Ayah windowing
-- [x] Cross-encoder reranking support (`BAAI/bge-reranker-v2-m3`)
 - [ ] Batch processing (multiple questions per request)
 - [ ] Islamic calendar integration (Hijri dates)
 
@@ -625,64 +657,7 @@ docker-compose down && docker system prune
 - **Qur'an**: [risan/quran-json](https://github.com/risan/quran-json) & [semarketir/quranjson](https://github.com/semarketir/quranjson) — 114 Surahs, 6,236 verses with Arabic text, transliteration, and English translation.
 - **Tafsir**: [Tafseer Dataset & Quran.com](https://api.quran.com) — 5,430 verses enriched with classical Arabic commentaries (*Tafsir al-Sa'di*, *Tafsir al-Mukhtasar*, *Tafsir al-Jalalayn*) + English *Tafsir Ibn Kathir*.
 - **Hadith**: [AhmedBaset/hadith-json](https://github.com/AhmedBaset/hadith-json) & [fawazahmed0/hadith-api](https://github.com/fawazahmed0/hadith-api) — 15 authoritative collections (9 canonical + *Riyad al-Salihin*, *Bulugh al-Maram*, *Al-Adab al-Mufrad*, *Ash-Shama'il al-Muhammadiyyah*, *An-Nawawi's 40 Hadith*, *Hadith Qudsi*), 46,416 hadiths with 21,264 scholar authenticity grades.
-- **Fatwa**: Verified rulings and answers from renowned scholars and councils (Ibn Baz, Lajnah Da'imah, Ibn Uthaymeen, Islamweb, Al-Badr).
-
----
-
-## What's New in v8
-
-✨ **15 Authoritative Hadith Collections (46,416 Narrations)** — Expanded beyond the 9 canonical books by ingesting core thematic fiqh and ethical collections: *Riyad al-Salihin* (1,896 hadiths), *Bulugh al-Maram* (1,767 hadiths), *Al-Adab al-Mufrad* (1,326 hadiths), *Ash-Shama'il al-Muhammadiyyah* (402 hadiths), *An-Nawawi's 40 Hadith* (42 hadiths), and *Hadith Qudsi* (40 hadiths), with 21,264 scholar authenticity grades.
-✨ **Classical Arabic Tafsir Enrichment (5,430 Verses)** — Enriched Quranic entries with verse-by-verse *Tafsir al-Sa'di* (تفسير السعدي) and *Tafsir al-Mukhtasar* (التفسير المختصر) alongside English Tafsir snippets, enabling precise semantic retrieval for theological and interpretative queries.
-✨ **Fatwa & Fiqh Corpus Integration** — Full integration of authenticated Islamic legal rulings and scholarly answers with dedicated `/fatwa` search, source, and ID lookup endpoints.
-✨ **Asymmetric E5 Embedding Alignment** — Prefixed indexed passages with `passage: ` while keeping queries prefixed with `query: `, maximizing dense semantic retrieval accuracy with `multilingual-e5-large`.
-✨ **Multi-Vector Dual-Language Dense Retrieval** — Executes independent vector searches for Arabic and English translations and blends them via adaptive Reciprocal Rank Fusion (RRF) based on input script.
-✨ **Conversational Multi-Turn Coreference Resolution** — Query rewriter now ingests prior conversation history to resolve pronouns and implicit subject references into standalone, unambiguous queries.
-✨ **Hadith Matn Deduplication & *Muttafaq 'Alayh* Clustering** — Identifies near-identical Hadith narrations across canonical collections ($\ge 0.85$ text similarity), labels narrations recorded by both Bukhari and Muslim as *Muttafaq 'Alayh*, and aggregates alternative cross-references in context.
-✨ **Quranic Parent-Document Ayah Windowing** — Automatically expands retrieved Quranic verses with surrounding $\pm 1$ verses (`AYAH_WINDOW_SIZE`) so the LLM receives complete theological context.
-✨ **Bilingual Stopwords & Curated Quranic Roots** — Built-in morphological Quranic root dictionary for primary Islamic concepts (`صوم`, `صلاة`, `زكاة`, `ربا`, `طهر`, etc.) and bilingual stopword filtering for high-precision BM25 scoring.
-✨ **Cross-Encoder Reranker Support** — Added optional second-stage neural reranking with `BAAI/bge-reranker-v2-m3` via `USE_RERANKER=true`.
-✨ **Hardware-Accelerated Index Builder** — Enhanced `build_index.py` with automatic Apple Silicon Metal (MPS) and CUDA detection, increasing indexing throughput to 64 items/batch.
-✨ **Extended Tafsir Capacity & Multi-Style Quote Verification** — Increased Tafsir context snippet size to 1,200 characters and hardened quote verification regex to support Arabic guillemets `«...»`, typographic quotes `“...”`, and standard quotes.
-✨ **Corpus Scale Expansion** — Total knowledge base expanded to **52,677 authenticated documents** across Quran, Hadith, and Fatwa, with a 205.77 MB FAISS inner-product index.
-
----
-
-## What's New in v7
-
-✨ **Inverted BM25 Index** — Full-dataset sparse retrieval built at startup (replaces per-candidate linear BM25 scoring)
-✨ **Reciprocal Rank Fusion** — Dense (FAISS) + sparse (BM25) results merged via RRF instead of fixed-weight scoring
-✨ **Quran Boost** — New `QURAN_BOOST` config counterbalances `HADITH_BOOST` so Hadiths don't outrank relevant Quran verses on tafsir/quran-intent queries
-✨ **Query Embedding Cache** — Repeated query vectors served from cache instead of re-encoding
-✨ **Fast-Path Query Rewrite** — Direct Quran references (e.g. "2:255", surah names) and verse-count questions skip the LLM rewrite roundtrip
-✨ **Improved Arabic Root Extraction** — `extract_arabic_root()` strips augmented letters for more accurate 3-letter root matching; root variants now included in tokenization
-✨ **Hardened Docker Image** — Multi-stage build (compiler toolchain isolated to builder stage), runs as non-root user, smaller runtime image
-✨ **Production-Safe Compose** — `docker-compose.yml` no longer bind-mounts the repo by default; data files mounted read-only; live-reload dev setup moved to `docker-compose.override.yml`
-
----
-
-## What's New in v6
-
-✨ **Reasoning Support** — Capture and stream model thinking/CoT
-✨ **Root Analysis** — Explore Quran via 3-letter Arabic roots
-✨ **Isnad Highlighting** — Narrators emphasized in Hadith responses
-✨ **Synonym Expansion** — Curated Islamic thesaurus for broader search
-✨ **Dynamic Gating** — Intent-aware confidence thresholds
-✨ **Dual LLM Backend** — Ollama (dev) + HuggingFace (prod)
-✨ **Grade Filtering** — Return only Sahih/Hasan authenticated Hadiths
-✨ **Source Filtering** — Quran-only or Hadith-only queries
-✨ **Hadith Verification** — `/hadith/verify` endpoint
-✨ **Enhanced Frequency** — Word counts by Surah
-✨ **OpenAI Compatible** — Use with any OpenAI client
-✨ **Production Ready** — Structured logging, error handling, async throughout
-
----
-
-## Next Steps
-
-1. **Get Started**: See [SETUP.md](SETUP.md)
-2. **Integrate with Open-WebUI**: See [OPEN_WEBUI.md](OPEN_WEBUI.md)
-3. **Deploy with Docker**: See [DOCKER.md](DOCKER.md)
-4. **Understand Architecture**: See [ARCHITECTURE.md](ARCHITECTURE.md)
+- **Fatwa**: 3,598 rulings — [Islamweb](https://www.islamweb.net/ar/fatwa/) Fatwa Center (3,534) and [Imam Ibn Baz](https://binbaz.org.sa/fatwas) (64).
 
 ---
 
@@ -698,4 +673,4 @@ See individual repositories for license details.
 
 **Made with ❤️ for Islamic scholarship.**
 
-Version 8.0.0 | October 2026 | Production-Ready
+Version 8.5.0 | October 2026 | Production-Ready

app/__init__.py

@@ -1 +1 @@
-"""QModel v6 — Islamic RAG API."""
+"""QModel v8.5 — Islamic RAG API."""

app/analysis.py

@@ -152,12 +152,12 @@ async def detect_analysis_intent(query: str, rewrite: dict) -> Optional[str]:
         kw_text = " ".join(kws)
         if any(w in kw_text for w in ("آيات", "آية", "verses", "ayat")):
             return None
-        # The rewriter is instructed to put the target word as first keyword
-        if kws:
-            return kws[0]
-        # Fallback: extract from query
+        # The question's own wording first: only the LLM rewrite puts the target word
+        # first in keywords — the fast rewrite's first keyword is just "كم" / "How".
         keyword = _extract_count_keyword(query)
-        return keyword
+        if keyword:
+            return keyword
+        return kws[0] if kws else None
 
     if not (_COUNT_EN.search(query) or _COUNT_AR.search(query)):
         return None

app/arabic_nlp.py

@@ -46,17 +46,34 @@ def normalize_arabic(text: str, *, aggressive: bool = False) -> str:
 _AR_PREFIXES = re.compile(
     r"^(ولل|فال|بال|كال|وال|ومن|وفي|وعن|وإلى|وعلى|لل|ال|و|ف|ب|ل)"
 )
+_PREFIX_SET = set(_AR_PREFIXES.pattern.strip("^()").split("|"))
 _AR_SUFFIXES = re.compile(
     r"(ون|ين|ان|ات|ها|هم|هن|كم|كن|نا|ني|تي|ي|ه|ك|ا|وا)$"
 )
 
 
+# An affix is only stripped when enough letters remain: in short words the "affix"
+# is usually a root letter (فضل is not ف+ضل, الربا is not ال+رب+ا).
+_MIN_STEM = 3
+_MIN_STEM_AFTER_1_SUFFIX = 3
+
+
 def light_stem(word: str) -> str:
     """Light stemming: remove common Arabic prefixes and suffixes."""
-    w = _AR_PREFIXES.sub("", word)
-    if len(w) >= 3:
-        w = _AR_SUFFIXES.sub("", w)
-    return w if len(w) >= 2 else word
+    w = word
+    m = _AR_PREFIXES.match(w)
+    if m:
+        # Longest prefix that leaves a real stem (e.g. وال → و when "ال" is part of the word)
+        for k in range(len(m.group(1)), 0, -1):
+            if w[:k] in _PREFIX_SET and len(w) - k >= _MIN_STEM:
+                w = w[k:]
+                break
+    m = _AR_SUFFIXES.search(w)
+    if m:
+        need = _MIN_STEM_AFTER_1_SUFFIX if len(m.group(1)) == 1 else _MIN_STEM
+        if len(w) - len(m.group(1)) >= need:
+            w = w[: m.start()]
+    return w
 
 
 

app/config.py

@@ -58,7 +58,7 @@ class Config:
     HADITH_BOOST:         float = float(os.getenv("HADITH_BOOST", 0.08))
     QURAN_BOOST:          float = float(os.getenv("QURAN_BOOST", 0.12))
     FATWA_BOOST:          float = float(os.getenv("FATWA_BOOST", 0.10))
-    USE_RERANKER:         bool  = os.getenv("USE_RERANKER", "false").lower() in ("true", "1", "yes")
+    USE_RERANKER:         bool  = os.getenv("USE_RERANKER", "true").lower() in ("true", "1", "yes")
     RERANK_MODEL:         str   = os.getenv("RERANK_MODEL", "BAAI/bge-reranker-v2-m3")
     AYAH_WINDOW_SIZE:     int   = int(os.getenv("AYAH_WINDOW_SIZE", 1))
     ENABLE_HADITH_DEDUPLICATION: bool = os.getenv("ENABLE_HADITH_DEDUPLICATION", "true").lower() in ("true", "1", "yes")
@@ -67,18 +67,13 @@ class Config:
     FATWA_MAX_CHARS:      int   = int(os.getenv("FATWA_MAX_CHARS", 1500))
     CONTEXT_MAX_CHARS:    int   = int(os.getenv("CONTEXT_MAX_CHARS", 14000))
 
-    # Safety
-    CONFIDENCE_THRESHOLD: float = float(os.getenv("CONFIDENCE_THRESHOLD", 0.30))
+    # Safety — confidence gate. A question reaches the LLM only when retrieval shows
+    # real relevance: a direct reference / verbatim text match, else the reranker's
+    # relevance probability (when USE_RERANKER), else raw dense cosine similarity.
+    RERANK_MIN_SCORE:     float = float(os.getenv("RERANK_MIN_SCORE", 0.05))
+    # Cosine scale of multilingual-e5-large; recalibrate (eval/run_eval.py) if EMBED_MODEL changes.
+    DENSE_MIN_SCORE:      float = float(os.getenv("DENSE_MIN_SCORE", 0.80))
     ENABLE_CITATION_VERIFICATION: bool = os.getenv("ENABLE_CITATION_VERIFICATION", "true").lower() in ("true", "1", "yes")
-    INTENT_THRESHOLDS: Dict[str, float] = {
-        "auth":        0.45,  # Stricter for authenticity checks
-        "hadith":      0.30,
-        "tafsir":      0.30,
-        "surah_info":  0.20,  # More lenient for metadata
-        "count":       0.25,
-        "general":     0.28,
-        "fatwa":       0.28,
-    }
 
     # CORS
     ALLOWED_ORIGINS:      str   = os.getenv("ALLOWED_ORIGINS", "*")

app/prompts.py

@@ -30,7 +30,7 @@ TASK_INSTRUCTIONS: Dict[str, str] = {
         "   ﴿{Arabic verse text copied VERBATIM from context}﴾ [[1]({tafsir_url})]\n"
         "   [ثم أضف شرحاً وتفسيراً للآية مع بيان معانيها المستفادة بناءً على التفسير المرفق ومعنى الآية]\n\n"
         "   Where {surah_url} and {tafsir_url} are taken directly from the [REF] line for this verse:\n"
-        "   Surah URL: https://surahquran.com/aya-{verse_number}-sora-{surah_number}.html\n"
+        "   Surah URL: https://quran.com/{surah_number}/{verse_number}\n"
         "   Tafsir URL: https://quran.ksu.edu.sa/tafseer/qortobi/sura{surah_number}-aya{verse_number}.html\n\n"
         "3. If the user's question is in English, format your response as follows:\n"
         "   This holy verse is Ayah {verse_number} of Surah {surah_name_ar} ({surah_name_en}) in the Holy Quran: [[1]({surah_url})]\n"
@@ -48,9 +48,12 @@ TASK_INSTRUCTIONS: Dict[str, str] = {
         "1. Find the best matching Hadith from the RETRIEVED RESULTS in the ISLAMIC CONTEXT.\n"
         "2. Quote the Hadith text EXACTLY — both Arabic and English VERBATIM from the results.\n"
         "   IMPORTANT: Highlight narrator markers (e.g., **حَدَّثَنَا**, **عَنْ**, **Narrated**, **on the authority of**) in bold.\n"
-        "3. State the full reference: collection name, book/chapter, hadith number, and grade from the results.\n"
+        "3. State the full reference — collection name AND hadith number, copied EXACTLY from the [HADITH_REF] line "
+        "(use its Arabic form in Arabic answers) — plus chapter and grade.\n"
         "4. If the user searched by partial text, present the complete Hadith found in the results.\n"
         "5. Explain the meaning, context, and scholarly implications.\n"
+        "Before saying a Hadith is missing, check EVERY Hadith item in the context: one with the same meaning in "
+        "different wording counts as found.\n"
         "CRITICAL: If the Hadith is NOT in the results, say so clearly — do NOT generate or fabricate from memory."
     ),
     "auth": (
@@ -61,7 +64,7 @@ TASK_INSTRUCTIONS: Dict[str, str] = {
         "   b. Use the 'grade' field from the result as the AUTHORITATIVE grade. "
         "NEVER infer or guess a grade from the collection name alone.\n"
         "   c. If no grade field exists in the result, state: 'Grade not recorded in dataset.'\n"
-        "   d. Provide the full reference: collection, hadith number, chapter.\n"
+        "   d. Provide the full reference — collection AND hadith number copied EXACTLY from the [HADITH_REF] line — and chapter.\n"
         "   e. Quote the full Hadith text from the results.\n"
         "      IMPORTANT: Highlight narrator markers (e.g., **عَنْ**, **Narrated**) in bold.\n"
         "   f. Explain what the grade means (e.g. Sahih = rigorously authenticated chain).\n"
@@ -95,7 +98,8 @@ TASK_INSTRUCTIONS: Dict[str, str] = {
         "2. For Quranic verses found, format as:\n"
         "   هذه الآية الكريمة هي الآية رقم {verse_number} من سورة {surah_name_ar} في القرآن الكريم: [[1]({surah_url})]\n"
         "   ﴿{Arabic text copied VERBATIM from context}﴾ [[1]({tafsir_url})]\n"
-        "3. For Hadith found: state collection name, hadith number, chapter, and grade, with narrator chain highlighted.\n"
+        "3. For Hadith found: state collection name and hadith number EXACTLY as in the [HADITH_REF] line, chapter, and grade, "
+        "with narrator chain highlighted.\n"
         "4. For Fatwa found: state scholar/council name, title, and link [[1]({source_url})].\n"
         "5. NEVER generate or guess verses, hadiths, or fatwas from memory.\n"
         "6. Provide context or explanation based strictly on the retrieved results."
@@ -127,7 +131,9 @@ CITATION FORMAT RULES:
 
 2. For Hadiths:
    - Highlight narrator markers in bold (e.g. **عَنْ**, **حَدَّثَنَا**, **Narrated**).
-   - State the full reference (Collection, Book, Number, Grade).
+   - State the full reference (Collection, Number, Chapter, Grade). Copy the collection and number EXACTLY
+     from the [HADITH_REF] line — e.g. "Sahih al-Bukhari 6116", or "صحيح البخاري، رقم 6116" in Arabic.
+     Never mention a collection without its number.
 
 3. For Fatwas:
    - State the scholar or council name, fatwa title, answer summary, and source link [[1]({source_url})].
@@ -140,7 +146,16 @@ ABSOLUTE ZERO-HALLUCINATION RULES (CRITICAL):
     "هذا الحديث/الآية غير موجود في قاعدة البيانات." (Arabic)
     or "This Hadith/verse is not in the available dataset." (English)
 • Do NOT substitute another verse or hadith from memory.
-• Copy references, Surah names, Ayah numbers, and URLs EXACTLY from the [REF] / [FATWA_REF] metadata.
+• NEVER both refuse and answer: either answer from the context, or give ONLY the not-found statement.
+  Do not say something is missing from the dataset and then explain it from general knowledge.
+• Copy references, Surah names, Ayah numbers, and URLs EXACTLY from the [REF] / [HADITH_REF] / [FATWA_REF] metadata.
+
+SCOPE RULE (CRITICAL):
+• You only answer questions about Islam (Quran, Hadith, Fiqh/Fatwa, Islamic belief and practice) from the context.
+• For any other request — creative writing (poems, stories, songs), sports, trivia, coding, general knowledge —
+  even if the context contains loosely related verses, do NOT comply. Reply ONLY with:
+    "لم أجد في قاعدة البيانات ما يكفي للإجابة على هذا السؤال بدقة." (Arabic)
+    or "The available dataset does not contain sufficient information to answer this question accurately." (English)
 
 LANGUAGE RULE (CRITICAL — MUST FOLLOW):
 • You MUST answer in the SAME language as the user's question.

app/routers/fatwa.py

@@ -39,6 +39,7 @@ async def search_fatwas(
                 dataset=state.dataset,
                 top_n=top_k * 3,
                 source_type="fatwa",
+                type_indexes=state.type_indexes,
                 bm25_index=state.bm25_index,
             )
         except Exception:

app/routers/hadith.py

@@ -15,6 +15,7 @@ from app.models import (
 )
 from app.search import (
     filter_results_by_collection,
+    fuse_search_results,
     hybrid_search,
     lookup_hadith_references,
     merge_search_results,
@@ -42,12 +43,15 @@ async def hadith_text_search(
     Use this to find a hadith when you know part of the text.
     """
     check_ready()
-    direct_results = lookup_hadith_references(q, state.dataset, collection=collection, limit=limit, hadith_idx=state.hadith_ref_idx)
-    text_results = text_search(q, state.dataset, source_type="hadith", limit=limit)
-    results = filter_results_by_collection(
-        merge_search_results(direct_results, text_results, limit=limit),
+    direct_results = filter_results_by_collection(
+        lookup_hadith_references(q, state.dataset, collection=collection, limit=limit, hadith_idx=state.hadith_ref_idx),
         collection,
     )
+    text_results = filter_results_by_collection(
+        text_search(q, state.dataset, source_type="hadith", limit=limit),
+        collection,
+    )
+    results = merge_search_results(direct_results, text_results, limit=limit)
 
     # If text search returns few results, augment with semantic search
     if len(results) < 3:
@@ -55,10 +59,10 @@ async def hadith_text_search(
         sem_results = await hybrid_search(
             q, rewrite,
             state.embed_model, state.faiss_index, state.dataset,
-            top_n=limit, source_type="hadith",
+            top_n=limit, source_type="hadith", type_indexes=state.type_indexes,
         )
         sem_results = filter_results_by_collection(sem_results, collection)
-        results = merge_search_results(results, sem_results, limit=limit)
+        results = fuse_search_results(direct_results, sem_results, [(q, text_results)], limit=limit)
 
     return TextSearchResponse(
         query=q,
@@ -99,6 +103,7 @@ async def hadith_topic_search(
         topic, rewrite,
         state.embed_model, state.faiss_index, state.dataset,
         top_n=top_k, source_type="hadith", grade_filter=grade_filter,
+        type_indexes=state.type_indexes,
     )
     return TextSearchResponse(
         query=topic,
@@ -148,20 +153,17 @@ async def verify_hadith(
     semantic_results = await hybrid_search(
         q, rewrite,
         state.embed_model, state.faiss_index, state.dataset,
-        top_n=10, source_type="hadith",
+        top_n=10, source_type="hadith", type_indexes=state.type_indexes,
     )
     semantic_results = filter_results_by_collection(semantic_results, collection)
-    merged_results = merge_search_results(direct_results, text_results, semantic_results, limit=10)
-
-    # 3. Pick best result — prefer high-confidence text matches,
-    #    then semantic results, then lower-confidence text matches
-    best = None
-    if direct_results:
-        best = direct_results[0]
-    elif text_results and text_results[0].get("_score", 0) > 2.0:
-        best = text_results[0]
-    elif merged_results:
-        best = merged_results[0]
+
+    # 3. Pick best result by rank fusion: direct references and verbatim text first,
+    #    then semantic and phrase-level text matches. Raw scores aren't comparable
+    #    across these sources, so a weak word-overlap hit must not win on score.
+    merged_results = fuse_search_results(
+        direct_results, semantic_results, [(q, text_results)], limit=10,
+    )
+    best = merged_results[0] if merged_results else None
 
     if best:
         return HadithVerifyResponse(

app/routers/ops.py

@@ -27,11 +27,12 @@ def health(response: Response):
         response.status_code = 503
     return {
         "status":               "ok" if state.ready else "initialising",
-        "version":              "7.0.0",
+        "version":              "8.5.0",
         "llm_backend":          cfg.LLM_BACKEND,
         "dataset_size":         len(state.dataset) if state.dataset else 0,
         "faiss_total":          state.faiss_index.ntotal if state.faiss_index else 0,
-        "confidence_threshold": cfg.CONFIDENCE_THRESHOLD,
+        "confidence_gate":      (f"reranker>={cfg.RERANK_MIN_SCORE}" if state.reranker
+                                 else f"dense>={cfg.DENSE_MIN_SCORE}"),
     }
 
 
@@ -55,19 +56,20 @@ async def debug_scores(
 ):
     """Inspect raw retrieval scores without calling the LLM.
 
-    Use this to calibrate CONFIDENCE_THRESHOLD and debug search quality.
+    `_dense` is the cosine similarity the confidence gate compares to
+    DENSE_MIN_SCORE when the reranker is off (calibrate with eval/run_eval.py).
     """
     check_ready()
     rewrite = await rewrite_query(q, state.llm)
     results = await hybrid_search(
         q, rewrite,
         state.embed_model, state.faiss_index, state.dataset,
-        top_n=top_k, source_type=source_type,
+        top_n=top_k, source_type=source_type, type_indexes=state.type_indexes,
     )
     return {
         "query":     q,
         "intent":    rewrite.get("intent", "general"),
-        "threshold": cfg.CONFIDENCE_THRESHOLD,
+        "dense_min_score": cfg.DENSE_MIN_SCORE,
         "count":     len(results),
         "results": [
             {

app/routers/quran.py

@@ -21,7 +21,14 @@ from app.models import (
     VerseItem,
     WordFrequencyResponse,
 )
-from app.search import hybrid_search, lookup_quran_verses, merge_search_results, rewrite_query, text_search
+from app.search import (
+    fuse_search_results,
+    hybrid_search,
+    lookup_quran_verses,
+    merge_search_results,
+    rewrite_query,
+    text_search,
+)
 from app.state import check_ready, state
 
 router = APIRouter(prefix="/quran", tags=["quran"])
@@ -64,9 +71,9 @@ async def quran_text_search(
             semantic_results = await hybrid_search(
                 q, rewrite,
                 state.embed_model, state.faiss_index, state.dataset,
-                top_n=limit, source_type="quran",
+                top_n=limit, source_type="quran", type_indexes=state.type_indexes,
             )
-            results = merge_search_results(results, semantic_results, limit=limit)
+            results = fuse_search_results(direct_results, semantic_results, [(q, text_results)], limit=limit)
 
     return TextSearchResponse(
         query=q,
@@ -106,7 +113,7 @@ async def quran_topic_search(
     semantic_results = await hybrid_search(
         topic, rewrite,
         state.embed_model, state.faiss_index, state.dataset,
-        top_n=top_k, source_type="quran",
+        top_n=top_k, source_type="quran", type_indexes=state.type_indexes,
     )
     results = merge_search_results(direct_results, semantic_results, limit=top_k)
     return TextSearchResponse(

app/search.py

@@ -24,6 +24,8 @@ except ImportError:
     SentenceTransformer = None
 
 from app.arabic_nlp import (
+    ARABIC_STOPWORDS,
+    ENGLISH_STOPWORDS,
     detect_language,
     extract_arabic_root,
     light_stem,
@@ -35,7 +37,7 @@ from app.cache import embed_cache, rewrite_cache, search_cache
 from app.config import cfg
 from app.hadith_utils import highlight_narrators
 from app.llm import LLMProvider
-from app.synonyms import expand_synonyms
+from app.synonyms import expand_synonyms, synonyms_in_text
 
 logger = logging.getLogger("qmodel.search")
 
@@ -57,6 +59,7 @@ class BM25InvertedIndex:
 
     def build(self, dataset: List[dict]) -> None:
         self.doc_count = len(dataset)
+        self.doc_types = [doc.get("type") for doc in dataset]
         self.doc_lengths = [0] * self.doc_count
         posting: Dict[str, Dict[int, int]] = {}
         total_len = 0
@@ -77,7 +80,9 @@ class BM25InvertedIndex:
         self.df = {term: len(docs) for term, docs in posting.items()}
         self.avg_dl = total_len / max(self.doc_count, 1)
 
-    def search(self, query_terms: List[str], top_k: int = 50) -> List[Tuple[int, float]]:
+    def search(
+        self, query_terms: List[str], top_k: int = 50, doc_type: Optional[str] = None,
+    ) -> List[Tuple[int, float]]:
         scores: Dict[int, float] = Counter()
         for term in query_terms:
             if term not in self.posting_list:
@@ -91,7 +96,10 @@ class BM25InvertedIndex:
                 num = tf * (self.k1 + 1.0)
                 scores[doc_idx] += idf * (num / denom)
 
-        return sorted(scores.items(), key=lambda x: x[1], reverse=True)[:top_k]
+        items = scores.items()
+        if doc_type is not None:
+            items = [(d, sc) for d, sc in items if self.doc_types[d] == doc_type]
+        return sorted(items, key=lambda x: x[1], reverse=True)[:top_k]
 
 
 
@@ -100,7 +108,7 @@ class BM25InvertedIndex:
 # ═══════════════════════════════════════════════════════════════════════
 _REWRITE_SYSTEM = """\
 You are an Islamic-scholarship search query optimizer.
-Your ONLY job: rewrite the user's question to maximise retrieval from a Quranic + Hadith dataset.
+Your ONLY job: rewrite the user's question to maximise retrieval from a Quran + Hadith + Fatwa dataset.
 
 Reply ONLY with a valid JSON object — no markdown, no preamble:
 {
@@ -176,6 +184,11 @@ _QURAN_REF_EN_VERSE_FIRST = re.compile(
     r"(?:ayah|verse|ayat)\s*(\d{1,3})\s+(?:of|in)\s+(?:surah|sura)\s+([A-Za-z'\- ]+)",
     re.I,
 )
+# Whole query is "<surah name> <verse>" with no keyword, e.g. "Al-Baqarah 183" / "البقرة 183".
+# Only honoured when the name is exactly a surah name (see _surah_name_equals).
+_QURAN_REF_BARE = re.compile(
+    r"^\s*([A-Za-zء-ۿ][A-Za-zء-ۿ'\- ]{1,30}?)\s*[,:،\s]\s*(\d{1,3})\s*$",
+)
 
 _COLLECTION_ALIASES = {
     "sahih al-bukhari": "Sahih al-Bukhari",
@@ -291,6 +304,41 @@ def _extract_json_object(text: str) -> Optional[str]:
     return _find_balanced_json(cleaned)
 
 
+# Quote marks must sit at word boundaries so apostrophes ("Prophet's") don't count.
+_AUTH_QUOTED = re.compile(r"(?:^|(?<=\s)|(?<=:))[\"“”«»❝❞']([^\"“”«»❝❞]{8,}?)[\"“”«»❝❞'](?=\s|$|[?.,؟:])")
+_AUTH_LEADING = re.compile(
+    r"^\s*(?:"
+    r"(?:هل|ما\s+(?:صحة|درجة|مدى\s+صحة)|صحة|درجة)\s+(?:هذا\s+)?(?:ال)?حديث(?:\s+(?:صحيح|ضعيف))?"
+    r"|هل\s+(?:صحيح|يصح|ثبت)\s+(?:ان\s+|أن\s+)?(?:(?:النبي|الرسول|رسول\s+الله)\S*\s+(?:\S+\s+){0,4}?قال\s*)?"
+    r"|(?:is|was|how\s+authentic\s+is)\s+(?:the\s+|this\s+|there\s+a\s+)?hadith(?:\s+that\s+says)?"
+    r"|is\s+it\s+(?:true|authentic)\s+that\s+(?:the\s+prophet\S*\s+(?:said|says)\s*(?:that\s+)?)?"
+    r"|verify\s+(?:the\s+|this\s+)?hadith"
+    r")\s*[:：]?\s*",
+    re.I,
+)
+_AUTH_TRAILING = re.compile(
+    r"\s*(?:"
+    r"(?:هل\s+)?(?:هو\s+)?(?:حديث\s+)?(?:صحيح|ضعيف|موضوع|ثابت)(?:\s+ام\s+(?:ضعيف|موضوع))?"
+    r"|(?:is\s+(?:it|this)\s+)?(?:authentic|sahih|weak|da'?if|fabricated|reliable|true)"
+    r")\s*[؟?]*\s*$",
+    re.I,
+)
+
+
+def strip_auth_phrasing(text: str) -> str:
+    """Reduce an authenticity question to the hadith text being asked about.
+
+    "هل حديث طلب العلم فريضة على كل مسلم صحيح؟" → "طلب العلم فريضة على كل مسلم"
+    "Is the hadith 'actions are by intentions' authentic?" → "actions are by intentions"
+    The framing words otherwise dominate embeddings and text search.
+    """
+    quoted = _AUTH_QUOTED.search(text)
+    if quoted:
+        return quoted.group(1).strip()
+    core = _AUTH_TRAILING.sub("", _AUTH_LEADING.sub("", text)).strip(" ؟?:：،,")
+    return core if len(core) >= 6 else text
+
+
 def _detect_intent_regex(query: str) -> str:
     """Detect intent from raw query using regex when LLM rewrite is unavailable."""
     # surah_info: asking about surah metadata (verse count, type, etc.)
@@ -319,7 +367,8 @@ def _detect_intent_regex(query: str) -> str:
 
     # auth: hadith authenticity check
     if re.search(
-        r"صحيح[؟?]|هل صحيح|درجة الحديث|هل هذا حديث|is.+authentic|verify hadith|hadith.+grade",
+        r"صحيح[؟?]|هل صحيح|درجة الحديث|درجة حديث|صحة حديث|صحة الحديث|هل هذا حديث"
+        r"|is.+authentic|is it true that|verify hadith|hadith.+grade|hadith.+\b(?:sahih|weak|da'?if|fabricated)\b",
         query, re.I,
     ):
         return "auth"
@@ -457,10 +506,21 @@ async def rewrite_query(
     # Strip common query prefixes like "آية:", "حديث:", "فتوى:" so text/BM25 searches aren't distorted
     cleaned_raw = re.sub(r"^\s*(?:آية|اية|الآية|الاية|حديث|الحديث|فتوى|الفتوى|سؤال|السؤال)\s*[:：]\s*", "", raw).strip()
     target_text = cleaned_raw if cleaned_raw else raw
+    if detected_intent == "auth":
+        target_text = strip_auth_phrasing(target_text)
+    synonyms = synonyms_in_text(target_text)
+    en_query = target_text
+    if detect_language(target_text) == "arabic":
+        # No translation without the LLM; give the English vector the thesaurus terms
+        english = [w for w in synonyms if w.isascii()]
+        if english:
+            en_query = f"{target_text} ({', '.join(english)})"
     fallback = {
         "ar_query": normalize_arabic(target_text),
-        "en_query": target_text,
-        "keywords": [light_stem(t) for t in tokenize_bilingual(target_text)][:7],
+        "en_query": en_query,
+        # Surface words, not stems: hybrid_search and count analysis stem them themselves,
+        # and stemming twice strips root letters (الوضوء → وضوء → ضوء).
+        "keywords": ([w for w in target_text.split() if tokenize_bilingual(w)][:7] + synonyms)[:15],
         "intent":   detected_intent,
     }
 
@@ -519,6 +579,9 @@ async def rewrite_query(
 
         result["ar_query"] = str(result.get("ar_query") or fallback["ar_query"]).strip()[:400]
         result["en_query"] = str(result.get("en_query") or fallback["en_query"]).strip()[:400]
+        if result["intent"] == "auth":
+            result["ar_query"] = strip_auth_phrasing(result["ar_query"])
+            result["en_query"] = strip_auth_phrasing(result["en_query"])
 
         await rewrite_cache.set(result, *cache_key)
         logger.info("Rewrite: intent=%s ar=%s", result["intent"], result["ar_query"][:60])
@@ -565,6 +628,73 @@ def merge_search_results(*result_groups: list, limit: Optional[int] = None) -> l
     return results[:limit] if limit is not None else results
 
 
+def _content_word_count(query: str) -> int:
+    words = normalize_arabic(query, aggressive=True).lower().split()
+    return len({w for w in words if w not in ARABIC_STOPWORDS and w not in ENGLISH_STOPWORDS})
+
+
+def fuse_search_results(
+    direct: list,
+    hybrid: list,
+    text_hits: List[Tuple[str, list]],
+    limit: Optional[int] = None,
+    rrf_k: int = 60,
+) -> list:
+    """Merge direct lookups, hybrid search and text search by rank, not raw score.
+
+    The three sources score on unrelated scales (direct 9.x, text 1–5, hybrid
+    RRF ≈0.2–0.9), so sorting on ``_score`` lets any weak text overlap outrank
+    the best semantic hit. Instead:
+      1. direct reference lookups stay first (explicit "2:255"-style requests);
+      2. exact substring hits for a query with ≥3 content words come next
+         (the user quoted the text verbatim);
+      3. everything else is ordered by reciprocal-rank fusion of the hybrid
+         ranking and the phrase/fuzzy/short-exact text rankings. Word-overlap
+         ("partial") text hits are dropped — they match on common words.
+
+    ``text_hits`` is a list of (query, hits) pairs, one per text_search call.
+    Items keep their original ``_score``; the returned order is the ranking.
+    Items from tiers 1–2 are marked ``_pinned`` (lexical evidence of relevance).
+    """
+    ordered: dict[tuple, dict] = {}
+
+    def add(item: dict, pinned: bool = False) -> None:
+        key = result_key(item)
+        if key not in ordered:
+            ordered[key] = {**item, "_pinned": True} if pinned else item
+
+    for item in sorted(direct, key=lambda r: r.get("_score", 0.0), reverse=True):
+        add(item, pinned=True)
+
+    rrf: Dict[tuple, float] = {}
+    items: Dict[tuple, dict] = {}
+
+    def accumulate(ranked: list) -> None:
+        for rank, item in enumerate(ranked, 1):
+            key = result_key(item)
+            rrf[key] = rrf.get(key, 0.0) + 1.0 / (rrf_k + rank)
+            if key not in items or item.get("_score", 0.0) > items[key].get("_score", 0.0):
+                items[key] = item
+
+    accumulate(hybrid)
+    for query, hits in text_hits:
+        verbatim = _content_word_count(query) >= 3
+        ranked = []
+        for hit in hits:
+            match = hit.get("_match", "partial")
+            if match == "exact" and verbatim:
+                add(hit, pinned=True)
+            elif match != "partial":
+                ranked.append(hit)
+        accumulate(ranked)
+
+    for key in sorted(rrf, key=rrf.get, reverse=True):
+        add(items[key])
+
+    results = list(ordered.values())
+    return results[:limit] if limit is not None else results
+
+
 def normalize_collection_name(text: str) -> Optional[str]:
     """Resolve a collection alias to the canonical dataset collection name."""
     if not text:
@@ -619,6 +749,19 @@ def _surah_matches(item: dict, surah_query: str) -> bool:
     return False
 
 
+def _surah_name_equals(item: dict, name: str) -> bool:
+    """Strict variant of _surah_matches: the whole name must equal a surah name."""
+    def clean(text: str) -> str:
+        text = normalize_arabic(text, aggressive=True).lower().strip()
+        return re.sub(r"^(ال|al[\-\s']*|the\s+)", "", text, flags=re.I).replace("-", " ").strip()
+
+    target = clean(name)
+    return bool(target) and any(
+        clean(item.get(field) or "") == target
+        for field in ("surah_name_ar", "surah_name_en", "surah_name_transliteration")
+    )
+
+
 _FAMOUS_AYAH_MAP = {
     "ايه الكرسي": (2, 255),
     "ayat al-kursi": (2, 255),
@@ -703,6 +846,17 @@ def lookup_quran_verses(
         if matches:
             break
 
+    bare = None if matches else _QURAN_REF_BARE.match(query)
+    if bare:
+        surah_query, verse_num = bare.group(1), int(bare.group(2))
+        for item in dataset:
+            if (
+                item.get("type") == "quran"
+                and (item.get("ayah_number") or item.get("verse_number")) == verse_num
+                and _surah_name_equals(item, surah_query)
+            ):
+                matches.append({**item, "_score": 9.0})
+
     return matches[:limit]
 
 
@@ -751,16 +905,19 @@ def _collection_rank(col_name: str) -> int:
 
 
 def cluster_hadith_candidates(results: list) -> list:
-    """Cluster near-identical Hadith narrations (matn deduplication) across collections."""
+    """Cluster near-identical Hadith narrations (matn deduplication) across collections.
+
+    Preserves input order: each cluster sits where its first narration appeared.
+    """
     if not results or not cfg.ENABLE_HADITH_DEDUPLICATION:
         return results
 
-    non_hadiths = []
+    ordered: list = []
     hadith_clusters: List[dict] = []
 
     for item in results:
         if item.get("type") != "hadith":
-            non_hadiths.append(item)
+            ordered.append(item)
             continue
 
         item_ar = normalize_arabic(item.get("arabic", ""), aggressive=True).lower()
@@ -838,14 +995,13 @@ def cluster_hadith_candidates(results: list) -> list:
             item_copy["other_collections"] = []
             item_copy["muttafaq_alayh"] = False
             hadith_clusters.append(item_copy)
+            ordered.append(item_copy)
 
-    final_hadiths = []
     for h in hadith_clusters:
         h.pop("_matn", None)
         h.pop("_words", None)
-        final_hadiths.append(h)
 
-    return sorted(non_hadiths + final_hadiths, key=lambda x: x.get("_score", 0.0), reverse=True)
+    return ordered
 
 
 def lookup_hadith_references(
@@ -959,6 +1115,67 @@ def expand_quran_context(
     return expanded
 
 
+def _cross_doc_text(c: dict, query: str = "") -> str:
+    # The reranker truncates at 512 tokens and a long diacritized verse (2:282 is ~1,000
+    # tokens) fills that alone, so put the text in the query's language first.
+    ar, en = c.get("arabic", ""), c.get("english", "")
+    doc_text = f"{en} {ar}" if detect_language(query) == "english" else f"{ar} {en}"
+    if c.get("type") == "quran" and c.get("tafsir_en"):
+        doc_text += f" {c['tafsir_en'][:300]}"
+    return doc_text.strip()
+
+
+# "في القرآن" says where to search (source routing handles it), not what to find; left
+# in, the reranker favours verses about the Qur'an itself (27:1 for "الوضوء في القرآن").
+# English keeps "in the Quran": without it "Is alcohol forbidden?" is too vague and
+# scored below the confidence gate.
+_SOURCE_PHRASE = re.compile(r"\s*(?:في|من|حسب)\s+(?:ال)?قر[اآ]ن(?:\s+الكريم)?")
+
+
+def _reranker_query(query: str) -> str:
+    """Query as the cross-encoder sees it: without "في القرآن", plus thesaurus terms in
+    the query's own script.
+
+    The reranker doesn't know that "alcohol" means the Qur'an's "wine" (khamr); without
+    the hint it scored 5:90 below a verse that merely mentions "the Qur'an".
+    """
+    query = _SOURCE_PHRASE.sub("", query).strip() or query
+    arabic = detect_language(query) == "arabic"
+    terms = [t for t in synonyms_in_text(query) if (not t.isascii()) == arabic]
+    return f"{query} ({', '.join(terms)})" if terms else query
+
+
+async def _cross_probabilities(cross_encoder_model: Any, pairs: list) -> List[float]:
+    # CrossEncoder.predict already applies its default activation (sigmoid for
+    # single-label rerankers such as bge-reranker), so outputs are probabilities.
+    loop = asyncio.get_event_loop()
+    raw = await loop.run_in_executor(None, lambda: cross_encoder_model.predict(pairs))
+    return [float(s) for s in raw]
+
+
+async def score_relevance(query: str, candidates: list, cross_encoder_model: Any) -> List[float]:
+    """Cross-encoder relevance probability (0–1) of each candidate to *query*.
+
+    Reuses ``_cross_score`` values already attached by hybrid_search's reranking.
+    """
+    missing = [i for i, c in enumerate(candidates) if "_cross_score" not in c]
+    probs = [c.get("_cross_score", 0.0) for c in candidates]
+    query = _reranker_query(query)
+    if missing:
+        pairs = [[query, _cross_doc_text(candidates[i], query)] for i in missing]
+        for i, p in zip(missing, await _cross_probabilities(cross_encoder_model, pairs)):
+            probs[i] = p
+    return probs
+
+
+# Candidates the cross-encoder scores, and how many of them are reserved for the best
+# BM25-only hits: in RRF a document found by both dense searches outranks any document
+# only BM25 found, so exact-word matches would otherwise never reach the reranker.
+_RERANK_POOL = 25
+_SPARSE_POOL_SLOTS = 5
+_RERANK_WEIGHT = 0.8      # share of the reranker in the blended hybrid score (0.8–0.95 tie on the golden set; 0.7 dropped 5:90 for "is alcohol forbidden?")
+
+
 async def rerank_with_cross_encoder(
     query: str,
     candidates: list,
@@ -970,28 +1187,18 @@ async def rerank_with_cross_encoder(
         return candidates[:top_k]
 
     try:
-        pool_size = min(len(candidates), 25)
+        pool_size = min(len(candidates), _RERANK_POOL)
         pool = candidates[:pool_size]
         tail = candidates[pool_size:]
 
-        pairs = []
-        for c in pool:
-            doc_text = f"{c.get('arabic', '')} {c.get('english', '')}"
-            if c.get("type") == "quran" and c.get("tafsir_en"):
-                doc_text += f" {c['tafsir_en'][:300]}"
-            pairs.append([query, doc_text])
+        query = _reranker_query(query)
+        pairs = [[query, _cross_doc_text(c, query)] for c in pool]
+        probs = await _cross_probabilities(cross_encoder_model, pairs)
 
-        loop = asyncio.get_event_loop()
-        raw_scores = await loop.run_in_executor(
-            None, lambda: cross_encoder_model.predict(pairs)
-        )
-
-        for c, s in zip(pool, raw_scores):
-            logit = float(s)
-            prob = 1.0 / (1.0 + np.exp(-logit))
-            c["_cross_score"] = logit
-            # Blend normalized cross-encoder confidence (scaled to ~2.0) with base RRF score
-            c["_score"] = float(0.7 * (prob * 2.0) + 0.3 * c.get("_score", 1.0))
+        for c, prob in zip(pool, probs):
+            c["_cross_score"] = prob
+            # Blend cross-encoder probability (scaled to ~2.0) with base RRF score
+            c["_score"] = float(_RERANK_WEIGHT * (prob * 2.0) + (1 - _RERANK_WEIGHT) * c.get("_score", 1.0))
 
         pool.sort(key=lambda x: x.get("_score", 0.0), reverse=True)
         combined = pool + tail
@@ -1001,6 +1208,26 @@ async def rerank_with_cross_encoder(
         return candidates[:top_k]
 
 
+def build_type_indexes(index: Any, dataset: list) -> Dict[str, Tuple[Any, np.ndarray]]:
+    """Per-source flat sub-indexes: {type: (index, dataset positions)}.
+
+    Filtering a global top-k afterwards fails for the smaller sources — for an
+    ablution question the top few hundred neighbours are all hadith (46k of them
+    vs 6k verses), leaving a Quran-only query with nothing. Searching the
+    source's own vectors avoids that.
+    """
+    n = min(index.ntotal, len(dataset))
+    vectors = index.reconstruct_n(0, n)
+    out: Dict[str, Tuple[Any, np.ndarray]] = {}
+    for doc_type in ("quran", "hadith", "fatwa"):
+        positions = np.array([i for i in range(n) if dataset[i].get("type") == doc_type], dtype=np.int64)
+        if len(positions):
+            sub = faiss.IndexFlatIP(index.d)
+            sub.add(vectors[positions])
+            out[doc_type] = (sub, positions)
+    return out
+
+
 # ═══════════════════════════════════════════════════════════════════════
 # HYBRID SEARCH  — dense FAISS + BM25 re-ranking + filtering
 # ═══════════════════════════════════════════════════════════════════════
@@ -1015,8 +1242,13 @@ async def hybrid_search(
     grade_filter: Optional[str] = None,
     bm25_index: Optional[BM25InvertedIndex] = None,
     reranker_model: Optional[Any] = None,
+    type_indexes: Optional[Dict[str, Tuple[Any, np.ndarray]]] = None,
 ) -> list:
-    """Hybrid search: multi-vector dense FAISS + sparse BM25 with RRF fusion, deduplication, and reranking."""
+    """Hybrid search: multi-vector dense FAISS + sparse BM25 with RRF fusion, deduplication, and reranking.
+
+    With ``source_type`` and ``type_indexes`` (see build_type_indexes), dense search
+    runs over that source's vectors only.
+    """
     ar_str = rewrite.get("ar_query") or rewrite.get("query_ar") or raw_query
     en_str = rewrite.get("en_query") or rewrite.get("query_en") or raw_query
     intent = rewrite.get("intent", "general")
@@ -1049,11 +1281,24 @@ async def hybrid_search(
         emb_ar, emb_en = embeddings[0], embeddings[1]
         await embed_cache.set((emb_ar, emb_en), ar_q, en_q)
 
-    # Batched FAISS search for Arabic and English vectors
+    # Batched FAISS search for Arabic and English vectors. A source filter searches
+    # that source's own sub-index; other filters (grade) are applied afterwards, so
+    # widen the pool for them.
+    k_search = cfg.TOP_K_SEARCH * (5 if grade_filter else 1)
     vec_both = np.vstack([emb_ar.reshape(1, -1), emb_en.reshape(1, -1)]).astype("float32")
-    distances_both, indices_both = await loop.run_in_executor(
-        None, lambda: index.search(vec_both, cfg.TOP_K_SEARCH)
-    )
+    sub = type_indexes.get(source_type) if (type_indexes and source_type) else None
+    if sub is not None:
+        sub_index, positions = sub
+        distances_both, sub_idx = await loop.run_in_executor(
+            None, lambda: sub_index.search(vec_both, k_search)
+        )
+        indices_both = np.where(sub_idx >= 0, positions[np.clip(sub_idx, 0, None)], -1)
+    else:
+        if source_type:
+            k_search *= 5
+        distances_both, indices_both = await loop.run_in_executor(
+            None, lambda: index.search(vec_both, k_search)
+        )
     distances_ar, indices_ar = distances_both[0], indices_both[0]
     distances_en, indices_en = distances_both[1], indices_both[1]
 
@@ -1099,12 +1344,13 @@ async def hybrid_search(
             rrf_scores[doc_idx] = rrf_scores.get(doc_idx, 0.0) + w_en * (1.0 / (rrf_k + rank))
 
     # ── 3. Sparse Inverted BM25 Search + RRF Fusion ───────────────────
+    sparse_order: List[int] = []
     query_terms: List[str] = []
     for kw in rewrite.get("keywords", raw_query.split()):
         query_terms.extend(tokenize_bilingual(kw))
 
     if bm25_index is not None:
-        sparse_hits = bm25_index.search(query_terms, top_k=cfg.TOP_K_SEARCH)
+        sparse_hits = bm25_index.search(query_terms, top_k=k_search, doc_type=source_type)
         w_sparse = 1.0
         for rank, (doc_idx, s_score) in enumerate(sparse_hits, 1):
             if 0 <= doc_idx < len(dataset):
@@ -1117,6 +1363,7 @@ async def hybrid_search(
                 if doc_idx not in item_map:
                     item_map[doc_idx] = {**item, "_dense": 0.0, "_dataset_idx": doc_idx}
                 item_map[doc_idx]["_sparse"] = float(s_score)
+                sparse_order.append(doc_idx)
                 rrf_scores[doc_idx] = rrf_scores.get(doc_idx, 0.0) + w_sparse * (1.0 / (rrf_k + rank))
 
     fused_candidates = []
@@ -1136,6 +1383,7 @@ async def hybrid_search(
 
     # ── 4. Cross-Encoder Re-ranking (if configured) ───────────────────
     if reranker_model is not None and cfg.USE_RERANKER and len(fused_candidates) > 1:
+        fused_candidates = _reserve_sparse_slots(fused_candidates, sparse_order)
         fused_candidates = await rerank_with_cross_encoder(
             raw_query, fused_candidates, reranker_model, top_k=len(fused_candidates)
         )
@@ -1149,9 +1397,46 @@ async def hybrid_search(
 
 
 
+def _reserve_sparse_slots(candidates: list, sparse_order: List[int]) -> list:
+    """Reorder fused candidates so the top BM25 hits fall inside the rerank pool."""
+    head_size = _RERANK_POOL - _SPARSE_POOL_SLOTS
+    in_head = {c.get("_dataset_idx") for c in candidates[:head_size]}
+    promote = []
+    for doc_idx in sparse_order:
+        if len(promote) == _SPARSE_POOL_SLOTS:
+            break
+        if doc_idx not in in_head:
+            promote.append(doc_idx)
+    if not promote:
+        return candidates
+    promoted = set(promote)
+    by_idx = {c.get("_dataset_idx"): c for c in candidates}
+    rest = [c for c in candidates[head_size:] if c.get("_dataset_idx") not in promoted]
+    return candidates[:head_size] + [by_idx[i] for i in promote] + rest
+
+
 # ═══════════════════════════════════════════════════════════════════════
 # TEXT-BASED SEARCH  (exact substring + fuzzy matching)
 # ═══════════════════════════════════════════════════════════════════════
+_NORM_CACHE: Dict[int, Tuple[list, List[Tuple[str, str]]]] = {}
+
+
+def _normalized_texts(dataset: list) -> List[Tuple[str, str]]:
+    """(normalized Arabic, lowercased English) per document, computed once per dataset.
+
+    Normalizing ~50k documents takes ~1.5 s, and text_search runs 2–3 times per query.
+    """
+    cached = _NORM_CACHE.get(id(dataset))
+    if cached is None or cached[0] is not dataset or len(cached[1]) != len(dataset):
+        texts = [
+            (normalize_arabic(d.get("arabic", ""), aggressive=True).lower(), d.get("english", "").lower())
+            for d in dataset
+        ]
+        _NORM_CACHE.clear()
+        _NORM_CACHE[id(dataset)] = cached = (dataset, texts)
+    return cached[1]
+
+
 def text_search(
     query: str,
     dataset: list,
@@ -1161,23 +1446,27 @@ def text_search(
     """Search dataset by exact text match (Arabic or English).
 
     Returns items sorted by relevance: exact matches first, then partial.
+    Each hit carries ``_match`` — "exact" (whole query is a substring),
+    "phrase" (2–3 word content window), "fuzzy" (≥80% similar), or
+    "partial" (word overlap) — so callers can weigh match strength.
     """
     clean_query = re.sub(r"^\s*(?:آية|اية|الآية|الاية|حديث|الحديث|فتوى|الفتوى|سؤال|السؤال)\s*[:：]\s*", "", query).strip()
     q_target = clean_query if clean_query else query
     q_norm = normalize_arabic(q_target, aggressive=True).lower()
     q_lower = q_target.lower().strip()
+    q_content = [
+        w not in ARABIC_STOPWORDS and w not in ENGLISH_STOPWORDS for w in q_norm.split()
+    ]
 
     results = []
-    for item in dataset:
+    for item, (ar_norm, en_lower) in zip(dataset, _normalized_texts(dataset)):
         if source_type and item.get("type") != source_type:
             continue
 
         ar_raw = item.get("arabic", "")
-        en_raw = item.get("english", "")
-        ar_norm = normalize_arabic(ar_raw, aggressive=True).lower()
-        en_lower = en_raw.lower()
 
         score = 0.0
+        match = "partial"
 
         # Exact substring in raw Arabic (with diacritics) — highest priority
         if query.strip() in ar_raw:
@@ -1192,21 +1481,32 @@ def text_search(
         if q_lower and q_lower in en_lower:
             score = max(score, 2.0 + (1.0 / max(len(en_lower), 1)) * 100)
 
-        # N-gram phrase matching for partial Arabic text (3+ word sequences)
+        if score > 0.0:
+            match = "exact"
+
+        # N-gram phrase matching for partial Arabic text (3+ word sequences).
+        # N-grams made mostly of stopwords ("of the", "ما هو") match nearly
+        # every document, so require ≥2 content words per window.
         if score == 0.0 and q_norm:
             q_words = q_norm.split()
             if len(q_words) >= 3:
                 # Check sliding windows of 3 words from query against doc
                 for i in range(len(q_words) - 2):
+                    if sum(q_content[i:i+3]) < 2:
+                        continue
                     trigram = " ".join(q_words[i:i+3])
                     if trigram in ar_norm:
                         score = max(score, 2.0 + (i == 0) * 0.5)
+                        match = "phrase"
                         break
             if score == 0.0 and len(q_words) >= 2:
                 for i in range(len(q_words) - 1):
+                    if sum(q_content[i:i+2]) < 2:
+                        continue
                     bigram = " ".join(q_words[i:i+2])
                     if bigram in ar_norm or bigram in en_lower:
                         score = max(score, 1.5)
+                        match = "phrase"
                         break
 
         # Word-level overlap for lower-confidence matches
@@ -1232,9 +1532,10 @@ def text_search(
                 ratio = SequenceMatcher(None, q_norm, text).ratio()
                 if ratio >= 0.80:
                     score = max(score, 1.0 + ratio)  # 1.80–2.0 range
+                    match = "fuzzy"
 
         if score > 0:
-            results.append({**item, "_score": score})
+            results.append({**item, "_score": score, "_match": match})
 
     results.sort(key=lambda x: x["_score"], reverse=True)
     return results if limit is None else results[:limit]
@@ -1247,9 +1548,38 @@ def _clip(text: str, limit: int, marker: str) -> str:
     return text[:limit].rstrip() + marker
 
 
+def surah_url(surah, verse) -> str:
+    """Link to a verse on quran.com ("" when either number is missing)."""
+    return f"https://quran.com/{int(surah)}/{int(verse)}" if surah and verse else ""
+
+
 _AR_CUT = " ... [باقي النص مقتطع للاختصار]"
 _EN_CUT = " ... [remainder truncated for length]"
 
+# Arabic collection names for the [HADITH_REF] line, so Arabic answers cite in Arabic
+_COLLECTION_AR = {
+    "Sahih al-Bukhari": "صحيح البخاري", "Sahih Muslim": "صحيح مسلم",
+    "Sunan Abu Dawood": "سنن أبي داود", "Jami' at-Tirmidhi": "جامع الترمذي",
+    "Sunan an-Nasai": "سنن النسائي", "Sunan Ibn Majah": "سنن ابن ماجه",
+    "Muwatta Malik": "موطأ مالك", "Musnad Ahmad": "مسند أحمد", "Sunan al-Darimi": "سنن الدارمي",
+    "Riyad al-Salihin": "رياض الصالحين", "Bulugh al-Maram": "بلوغ المرام",
+    "Al-Adab al-Mufrad": "الأدب المفرد", "Ash-Shama'il al-Muhammadiyyah": "الشمائل المحمدية",
+    "An-Nawawi's 40 Hadith": "الأربعون النووية", "Hadith Qudsi": "الأحاديث القدسية",
+}
+
+
+def _hadith_ref_line(r: dict) -> str:
+    """Citation line for a hadith: the exact reference in English and Arabic, plus grade."""
+    collection, number = r.get("collection", ""), r.get("hadith_number", "")
+    reference = r.get("reference") or f"{collection} {number}".strip()
+    line = f"\n    [HADITH_REF] Cite as: {reference}"
+    if collection in _COLLECTION_AR and number != "":
+        line += f" | بالعربية: {_COLLECTION_AR[collection]}، رقم {number}"
+    line += f" | Grade: {r.get('grade') or 'not recorded in dataset'}"
+    if r.get("chapter"):
+        line += f" | Chapter: {r['chapter'][:120]}"
+    return line
+
 
 def build_context(results: list) -> str:
     """Format search results into context block for LLM."""
@@ -1315,13 +1645,13 @@ def build_context(results: list) -> str:
         if doc_type == "quran":
             sn = r.get("surah_number", "")
             vn = r.get("verse_number", "")
-            surah_url = f"https://surahquran.com/aya-{vn}-sora-{sn}.html" if sn and vn else ""
+            verse_url = surah_url(sn, vn)
             tafsir_url = f"https://quran.ksu.edu.sa/tafseer/qortobi/sura{sn}-aya{vn}.html" if sn and vn else ""
             block += (
                 f"\n    [REF] Surah: {r.get('surah_name_ar', '')} ({r.get('surah_name_en', '')}) "
                 f"| Surah Number: {sn} "
                 f"| Ayah: {vn}\n"
-                f"    Surah URL : {surah_url}\n"
+                f"    Surah URL : {verse_url}\n"
                 f"    Tafsir URL: {tafsir_url}"
             )
             # Neighboring verses context if present
@@ -1335,6 +1665,8 @@ def build_context(results: list) -> str:
             elif r.get("tafsir_en"):
                 max_chars = getattr(cfg, "TAFSIR_MAX_CHARS", 1200)
                 block += f"\n    Tafsir : {r['tafsir_en'][:max_chars]}"
+        elif doc_type == "hadith":
+            block += _hadith_ref_line(r)
         elif doc_type == "fatwa":
             block += (
                 f"\n    [FATWA_REF] Source: {r.get('source', '')} | Scholar: {r.get('scholar_or_council', '')} "

app/state.py

@@ -8,7 +8,8 @@ import logging
 import re
 import time
 from contextlib import asynccontextmanager
-from typing import Any, AsyncIterator, Dict, FrozenSet, List, Literal, Optional, Tuple
+from difflib import SequenceMatcher
+from typing import Any, AsyncIterator, Dict, List, Literal, Optional, Tuple
 
 try:
     import faiss
@@ -28,21 +29,26 @@ from app.analysis import (
     detect_surah_info,
     lookup_surah_info,
 )
-from app.arabic_nlp import detect_language, normalize_arabic, tokenize_bilingual
+from app.arabic_nlp import ARABIC_STOPWORDS, detect_language, normalize_arabic, tokenize_bilingual
 from app.config import cfg
 from app.cache import pipeline_cache
 from app.llm import LLMProvider, get_llm_provider
 from app.prompts import build_messages, not_found_answer
 from app.search import (
     BM25InvertedIndex,
+    _COLLECTION_AR,
+    _normalized_texts,
     build_context,
+    build_type_indexes,
     cluster_hadith_candidates,
     expand_quran_context,
+    fuse_search_results,
     hybrid_search,
     lookup_hadith_references,
     lookup_quran_verses,
-    merge_search_results,
     rewrite_query,
+    score_relevance,
+    surah_url,
     text_search,
 )
 
@@ -109,9 +115,9 @@ def _results_fallback(results: list, lang: str) -> str:
             surah_en = r.get("surah_name_en", "")
             surah_num = r.get("surah_number", "")
             verse_num = r.get("verse_number", "")
-            surah_url = f"https://surahquran.com/aya-{verse_num}-sora-{surah_num}.html" if surah_num and verse_num else ""
+            verse_url = surah_url(surah_num, verse_num)
             tafsir_url = f"https://quran.ksu.edu.sa/tafseer/qortobi/sura{surah_num}-aya{verse_num}.html" if surah_num and verse_num else ""
-            surah_link = f"[[1]({surah_url})]" if surah_url else ""
+            surah_link = f"[[1]({verse_url})]" if verse_url else ""
             tafsir_link = f"[[1]({tafsir_url})]" if tafsir_url else ""
 
             if lang == "arabic":
@@ -231,11 +237,177 @@ def _build_surah_name_map(results: list) -> dict:
     return name_map
 
 
+# How answers name each fatwa source (matched on normalized text)
+_FATWA_SOURCE_MENTIONS = {
+    "Islamweb": re.compile(r"اسلام ?ويب|مركز الفتوي|islam ?web", re.I),
+    "BinBaz":   re.compile(r"\bا?بن باز|\b(?:bin|ibn) ?baz", re.I),
+    "AlBadr":   re.compile(r"عبد الرزاق البدر|al-?badr", re.I),
+}
+_TITLE_MIN_COVERAGE = 0.6
+_STOP_NORM = {normalize_arabic(w) for w in ARABIC_STOPWORDS}
+
+
+def _content_words(text: str) -> set:
+    return {w for w in normalize_arabic(text).lower().split() if len(w) > 2 and w not in _STOP_NORM}
+
+
+# A fatwa also counts as used when the answer shares at least _FATWA_COPY_MIN runs of
+# _FATWA_COPY_RUN words with it (about 17 consecutive words). Quoted scripture is left
+# out first, since fatwas quote the same verses and hadith the answer does.
+_FATWA_COPY_RUN = 8
+_FATWA_COPY_MIN = 10
+_QUOTED_SPAN = re.compile(r"﴿[^﴾]*﴾|❝[^❞]*❞|«[^»]*»")
+
+
+def _word_runs(text: str, n: int = _FATWA_COPY_RUN) -> set:
+    words = re.findall(r"\w+", normalize_arabic(text).lower())
+    return {tuple(words[i:i + n]) for i in range(len(words) - n + 1)}
+
+
+def _copied_runs(answer: str, results: list) -> set:
+    """Word runs of the answer outside quoted scripture and retrieved verses/hadith."""
+    runs = _word_runs(_QUOTED_SPAN.sub(" ", answer))
+    for r in results:
+        if r.get("type") in ("quran", "hadith"):
+            runs -= _word_runs(r.get("arabic", ""))
+    return runs
+
+
+# The prompt's link template, copied literally: [[1]({source_url})] or [[1]({https://...})]
+_LINK_PLACEHOLDER = re.compile(r"\s*\[?\[\d*\]\(\{source_url\}\)\]?")
+_BRACED_URL = re.compile(r"\(\{(https?://[^\s{}]+)\}\)")
+# [1](!https://...) — the stray "!" breaks the link and the UI shows the raw URL with ")]"
+_BANG_URL = re.compile(r"\]\(\s*!+\s*(?=https?://)")
+
+
+def _clean_links(text: str) -> str:
+    text = _BANG_URL.sub("](", _BRACED_URL.sub(r"(\1)", text))
+    return _LINK_PLACEHOLDER.sub("", text)
+
+
+def _fatwa_links(answer: str, results: list, lang: str) -> str:
+    """Source-link lines for fatwas the answer relies on but doesn't link ("" if none).
+
+    The LLM reliably names the fatwa council but often drops the [FATWA_REF] URL.
+    A retrieved fatwa counts as used when the answer contains most of its title's
+    words or copies a run of its text, else when the answer credits its source and
+    it is that source's top-ranked fatwa. Nothing is added if the answer already
+    links a retrieved fatwa.
+    """
+    fatwas = [r for r in results if r.get("type") == "fatwa" and r.get("source_url")]
+    if not fatwas or any(r["source_url"].rstrip("/") in answer for r in fatwas):
+        return ""
+
+    answer_words = _content_words(answer)
+    answer_runs = _copied_runs(answer, results)
+    used = []
+    for r in fatwas:
+        title_words = _content_words(r.get("title", ""))
+        if len(title_words) >= 3 and len(title_words & answer_words) / len(title_words) >= _TITLE_MIN_COVERAGE:
+            used.append(r)
+        elif len(answer_runs & _word_runs(r.get("answer_ar") or r.get("arabic") or r.get("english", ""))) >= _FATWA_COPY_MIN:
+            used.append(r)
+    if not used:
+        answer_norm = normalize_arabic(answer)
+        for source, pattern in _FATWA_SOURCE_MENTIONS.items():
+            if pattern.search(answer_norm):
+                top = next((r for r in fatwas if r.get("source") == source), None)
+                if top:
+                    used.append(top)
+    used = list({r["source_url"]: r for r in reversed(used)}.values())[::-1]
+    label = "المصدر" if lang == "arabic" else "Source"
+    return "\n".join(f"{label}: [{r.get('title') or r['source_url']}]({r['source_url']})" for r in used[:3])
+
+
+# Word that identifies each collection when an answer names it
+_COLLECTION_NAMES = {
+    "Sahih al-Bukhari": ("bukhari", "البخاري"), "Sahih Muslim": ("muslim", "مسلم"),
+    "Sunan Abu Dawood": ("dawud", "dawood", "داود"), "Jami' at-Tirmidhi": ("tirmidhi", "الترمذي"),
+    "Sunan an-Nasai": ("nasai", "nasa'i", "النسائي"), "Sunan Ibn Majah": ("majah", "ماجه"),
+    "Muwatta Malik": ("malik", "muwatta", "مالك", "الموطأ"), "Musnad Ahmad": ("ahmad", "أحمد"),
+    "Sunan al-Darimi": ("darimi", "الدارمي"), "Riyad al-Salihin": ("riyad", "الصالحين"),
+    "Bulugh al-Maram": ("bulugh", "المرام"), "Al-Adab al-Mufrad": ("adab", "المفرد"),
+    "Ash-Shama'il al-Muhammadiyyah": ("shama'il", "shamail", "الشمائل"),
+    "An-Nawawi's 40 Hadith": ("nawawi", "النووية"), "Hadith Qudsi": ("qudsi", "القدسية"),
+}
+# A hadith counts as quoted when the answer reproduces most of its matn, or a long run
+# of its English translation. The isnad (shared narrators, so shared between unrelated
+# hadith) ends at the first mention of the Prophet; without one, the last 60% is used.
+_PROPHET_MENTION = re.compile(
+    r"(?:قال|عن|ان|سمعت|يقول)\s+(?:رسول\s+الله|النبي)(?:\s+صلي\s+الله\s+عليه\s+وسلم)?"
+)
+_MATN_MAX = 150           # consonants of the matn compared
+_MIN_MATN_COVERAGE = 0.6
+_MATN_MIN_RUN = 8         # ~2 words; shorter runs match by chance in a long answer
+
+
+def _matn_skeleton(arabic: str) -> str:
+    norm = normalize_arabic(arabic)
+    m = _PROPHET_MENTION.search(norm)
+    matn = _skeleton(norm[m.end():]) if m else _skeleton(norm)[int(len(_skeleton(norm)) * 0.4):]
+    return matn[:_MATN_MAX]
+_MIN_QUOTED_EN = 80       # characters shared in one run with the English translation
+
+
+def _longest_shared_run(a: str, b: str) -> int:
+    if not a or not b:
+        return 0
+    return SequenceMatcher(None, a, b, autojunk=False).find_longest_match(0, len(a), 0, len(b)).size
+
+
+def _hadith_refs(answer: str, results: list, lang: str) -> str:
+    """Reference lines for retrieved hadith the answer quotes without citing ("" if none).
+
+    The model often quotes the hadith but drops its number, despite [HADITH_REF].
+    """
+    lines = []
+    answer_lower = answer.lower()
+    answer_skel = _skeleton(answer)
+    for r in results:
+        number, collection = str(r.get("hadith_number", "")), r.get("collection", "")
+        if r.get("type") != "hadith" or not number or not collection:
+            continue
+        names = _COLLECTION_NAMES.get(collection, (collection.lower(),))
+        if re.search(rf"(?<!\d){re.escape(number)}(?!\d)", answer) and any(n in answer_lower for n in names):
+            continue  # already cited
+        matn = _matn_skeleton(r.get("arabic", ""))
+        quoted = (len(matn) >= 15 and _skeleton_coverage(matn, answer_skel, _MATN_MIN_RUN) >= _MIN_MATN_COVERAGE
+                  or _longest_shared_run(answer_lower, r.get("english", "").lower()) >= _MIN_QUOTED_EN)
+        if quoted:
+            ref = (f"{_COLLECTION_AR[collection]}، رقم {number}" if lang == "arabic" and collection in _COLLECTION_AR
+                   else r.get("reference") or f"{collection} {number}")
+            lines.append(f"{'المرجع' if lang == 'arabic' else 'Reference'}: {ref}")
+    return "\n".join(lines[:3])
+
+
+def _source_refs(answer: str, results: list, lang: str) -> str:
+    """Citation lines to append for sources the answer uses but doesn't cite."""
+    return "\n".join(x for x in (_hadith_refs(answer, results, lang), _fatwa_links(answer, results, lang)) if x)
+
+
+def _attach_source_refs(answer: str, results: list, lang: str) -> str:
+    answer = _clean_links(answer)
+    links = _source_refs(answer, results, lang)
+    if not links:
+        return answer
+    # Keep the closing formula last
+    closing = re.search(r"\n*\s*(والله أعلم\.?|And Allah knows best\.?)\s*$", answer)
+    if closing:
+        return f"{answer[:closing.start()].rstrip()}\n\n{links}\n\n{closing.group(1)}"
+    return f"{answer.rstrip()}\n\n{links}"
+
+
+_SURAH_URL = re.compile(r"https://quran\.com/(\d+)/(\d+)\b")
+# Verse links the model may still write from memory, in the old format
+_OLD_SURAH_URL = re.compile(r"https?://(?:www\.)?surahquran\.com/aya-(\d+)-sora-(\d+)\.html")
+
+
 def _verify_references(answer: str, results: list) -> str:
     """Check that surah/verse references in the answer match retrieved results.
 
     Replaces hallucinated references with corrected ones or warnings.
     """
+    answer = _OLD_SURAH_URL.sub(lambda m: surah_url(m.group(2), m.group(1)), answer)
     valid_refs = _build_valid_refs(results)
     if not valid_refs:
         return answer  # No quran results — nothing to verify
@@ -334,7 +506,7 @@ def _verify_references(answer: str, results: list) -> str:
         primary = quran_results[0]
         p_sn = int(primary["surah_number"])
         p_vn = int(primary["verse_number"])
-        actual_surah_url = f"https://surahquran.com/aya-{p_vn}-sora-{p_sn}.html"
+        actual_surah_url = surah_url(p_sn, p_vn)
         actual_tafsir_url = f"https://quran.ksu.edu.sa/tafseer/qortobi/sura{p_sn}-aya{p_vn}.html"
         answer = answer.replace("{surah_url}", actual_surah_url).replace("{tafsir_url}", actual_tafsir_url)
 
@@ -342,10 +514,10 @@ def _verify_references(answer: str, results: list) -> str:
         correct_sn, correct_vn = next(iter(valid_refs))
 
         def _check_surah_url(m: re.Match) -> str:
-            vn, sn = int(m.group(1)), int(m.group(2))
+            sn, vn = int(m.group(1)), int(m.group(2))
             if (sn, vn) not in valid_refs:
                 logger.warning("Correcting hallucinated surah URL %d:%d -> %d:%d", sn, vn, correct_sn, correct_vn)
-                return f"https://surahquran.com/aya-{correct_vn}-sora-{correct_sn}.html"
+                return surah_url(correct_sn, correct_vn)
             return m.group(0)
 
         def _check_ksu_url(m: re.Match) -> str:
@@ -355,37 +527,52 @@ def _verify_references(answer: str, results: list) -> str:
                 return f"https://quran.ksu.edu.sa/tafseer/qortobi/sura{correct_sn}-aya{correct_vn}.html"
             return m.group(0)
 
-        answer = re.sub(r"https://surahquran\.com/aya-(\d+)-sora-(\d+)\.html", _check_surah_url, answer)
+        answer = _SURAH_URL.sub(_check_surah_url, answer)
         answer = re.sub(r"https://quran\.ksu\.edu\.sa/tafseer/qortobi/sura(\d+)-aya(\d+)\.html", _check_ksu_url, answer)
 
     return answer
 
 
 def _verify_citations(answer: str, results: list) -> str:
-    """Check that quoted Arabic scripture in the answer actually appears in retrieved results or database.
-
-    Only applies to explicit sacred quote markers (❝...❞ or «...») containing substantive Arabic text.
-    If an Arabic scripture quote is not found in results or the Quran/Hadith dataset, it is flagged.
+    """Check that quoted Arabic scripture in the answer appears in the retrieved results.
+
+    Only applies to explicit sacred quote markers (﴿...﴾, ❝...❞, «...») containing substantive
+    Arabic text. Grounding is strict: a genuine verse that was not retrieved still fails,
+    since the answer may only quote what it was given.
+
+    1. A quote found verbatim (after normalization) in a retrieved passage is kept.
+    2. A quote that nearly matches a retrieved verse or hadith (≥ _SCRIPTURE_MIN_COVERAGE of
+       its consonant skeleton) is replaced by that passage's canonical text. LLMs corrupt
+       scripture while copying it: foreign letters, changed word forms, truncation.
+    3. Other quotes are kept only if they largely overlap a retrieved non-scripture text
+       (fatwa, tafsir, translation); otherwise they are replaced with a warning marker.
     """
     if not getattr(cfg, "ENABLE_CITATION_VERIFICATION", True):
         return answer
 
-    source_texts_raw = []
+    source_texts_raw = []      # every retrieved text: verbatim check
+    commentary_raw = []        # non-scripture texts: word-overlap check
+    scripture = []             # (canonical arabic, open, close, skeleton)
     for r in results:
         for field in ("arabic", "english", "text", "tafsir", "tafsir_ar", "tafsir_en", "explanation"):
             val = r.get(field, "")
             if val:
-                source_texts_raw.append(re.sub(r"\s+", " ", str(val).strip()))
+                val = re.sub(r"\s+", " ", str(val).strip())
+                source_texts_raw.append(val)
+                if not (field == "arabic" and r.get("type") in ("quran", "hadith")):
+                    commentary_raw.append(val)
+        if r.get("type") in ("quran", "hadith") and r.get("arabic"):
+            marks = ("﴿", "﴾") if r["type"] == "quran" else ("❝ ", " ❞")
+            scripture.append((r["arabic"], *marks, _skeleton(r["arabic"])))
         if r.get("context_ayat"):
             for ctx in r["context_ayat"]:
                 if ctx.get("arabic"):
                     source_texts_raw.append(re.sub(r"\s+", " ", str(ctx["arabic"]).strip()))
+                    scripture.append((ctx["arabic"], "﴿", "﴾", _skeleton(ctx["arabic"])))
 
     # Pre-compute normalized versions + word sets for diacritics-insensitive comparison
-    source_entries = []
-    for s in source_texts_raw:
-        s_norm = normalize_arabic(s)
-        source_entries.append((s, s_norm, set(s_norm.split())))
+    source_entries = [(s, normalize_arabic(s)) for s in source_texts_raw]
+    commentary_words = [set(normalize_arabic(s).split()) for s in commentary_raw]
 
     def _contains(quote: str, src: str) -> bool:
         # Reverse containment (source inside quote) only counts for non-trivial
@@ -402,38 +589,49 @@ def _verify_citations(answer: str, results: list) -> str:
         if len(quoted_norm) < 15 or ar_chars < 12:
             return m.group(0)
 
-        q_words = set(quoted_norm.split())
-
-        # 1. Check in retrieved search results (strictly grounded in retrieved indexed context)
-        for src_raw, src_norm, s_words in source_entries:
-            # Exact or normalized substring match
+        # 1. Verbatim in retrieved search results (strictly grounded in retrieved indexed context)
+        for src_raw, src_norm in source_entries:
             if _contains(quoted_raw, src_raw) or _contains(quoted_norm, src_norm):
                 return m.group(0)
-            # Word overlap on normalized text (≥40% of quoted words found)
-            if q_words and len(q_words & s_words) / len(q_words) >= 0.4:
-                return m.group(0)
 
-        # Quote not found in retrieved results — LLM generated/recalled scripture from model weights
+        # 2. Near-copy of a retrieved verse / hadith → restore the canonical text
+        q_skel = _skeleton(quoted_raw)
+        best, best_cov = None, 0.0
+        for entry in scripture:
+            cov = _skeleton_coverage(q_skel, entry[3])
+            if cov > best_cov or (cov == best_cov and best and len(entry[0]) < len(best[0])):
+                best, best_cov = entry, cov
+        if best and best_cov >= _SCRIPTURE_MIN_COVERAGE:
+            logger.info("Restoring canonical text for corrupted scripture quote (coverage %.2f)", best_cov)
+            return f"{best[1]}{best[0]}{best[2]}"
+
+        # 3. Quoting a retrieved fatwa / tafsir / translation (≥40% of quoted words found)
+        q_words = set(quoted_norm.split())
+        if q_words and any(len(q_words & w) / len(q_words) >= 0.4 for w in commentary_words):
+            return m.group(0)
+
+        # Not grounded in retrieved results — LLM generated/recalled scripture from model weights
         logger.warning("Hallucination detected: quoted text not in retrieved results: %.80s...", quoted_norm)
+        return "❝ ⚠️ [تم حذف نص غير موثق — النص غير موجود في البيانات المسترجعة] ❞"
+
+    return _QUOTE_RE.sub(_check_quote, answer)
 
-        # Auto-correct to authentic indexed Quran verse if top retrieved source is a Quran verse
-        quran_sources = [r for r in results if r.get("type") == "quran" and r.get("arabic")]
-        if quran_sources and (len(quran_sources) == 1 or quran_sources[0].get("_score", 0) >= 2.0):
-            correct_ar = quran_sources[0]["arabic"]
-            logger.info("Auto-correcting hallucinated scripture quote to indexed Quran verse %s", quran_sources[0].get("id"))
-            return f"﴿{correct_ar}﴾"
 
-        hadith_sources = [r for r in results if r.get("type") == "hadith" and r.get("arabic")]
-        if hadith_sources and len(hadith_sources) == 1:
-            correct_ar = hadith_sources[0]["arabic"]
-            logger.info("Auto-correcting hallucinated quote to indexed Hadith %s", hadith_sources[0].get("id"))
-            return f"❝ {correct_ar} ❞"
+def _skeleton(text: str) -> str:
+    """Consonant skeleton: drops diacritics, long vowels, hamza seats and non-Arabic
+    characters, so Uthmani and plain spellings of the same verse compare equal."""
+    return re.sub(r"[^\u0621-\u064A]|[اويىء]", "", normalize_arabic(text))
 
-        return "❝ ⚠️ [تم حذف نص غير موثق — النص غير موجود في البيانات المسترجعة] ❞"
 
-    return _QUOTE_RE.sub(_check_quote, answer)
+def _skeleton_coverage(quote_skel: str, source_skel: str, min_run: int = 3) -> float:
+    """Fraction of the quote's skeleton found, in order, in the source (runs of ≥ min_run letters)."""
+    if not quote_skel or not source_skel:
+        return 0.0
+    blocks = SequenceMatcher(None, quote_skel, source_skel, autojunk=False).get_matching_blocks()
+    return sum(b.size for b in blocks if b.size >= min_run) / len(quote_skel)
 
 
+_SCRIPTURE_MIN_COVERAGE = 0.8
 _MIN_REVERSE_MATCH = 15
 
 
@@ -610,14 +808,13 @@ class AppState:
     llm:             Optional[LLMProvider]         = None
     bm25_index:      Optional[BM25InvertedIndex]   = None
     reranker:        Optional[Any]                 = None
+    type_indexes:    Dict[str, Tuple[Any, Any]]    = {}   # source -> (sub-index, positions)
     ready:           bool                          = False
 
     # O(1) lookup tables built at startup
     quran_verse_idx: Dict[Tuple[int, int], dict]   = {}   # (surah_num, verse_num) -> item
     hadith_ref_idx:  Dict[Tuple[str, int], dict]   = {}   # (collection, hadith_num) -> item
     surah_info_map:  Dict[str, dict]               = {}   # normalised name key -> info
-    # Pre-normalized Quran texts for fast citation verification: (norm_text, word_set)
-    quran_norm_texts: List[Tuple[str, FrozenSet[str]]] = []
 
 
 state = AppState()
@@ -675,17 +872,11 @@ def _build_lookup_tables(dataset: list) -> None:
     state.hadith_ref_idx  = hadith_ref_idx
     state.surah_info_map  = surah_info_map
 
-    quran_norm_texts: List[Tuple[str, FrozenSet[str]]] = []
-    for item in quran_verse_idx.values():
-        v_norm = normalize_arabic(item.get("arabic", "") or "")
-        if v_norm:
-            quran_norm_texts.append((v_norm, frozenset(v_norm.split())))
-    state.quran_norm_texts = quran_norm_texts
-
     logger.info("Building full-dataset inverted BM25 index...")
     bm25 = BM25InvertedIndex()
     bm25.build(dataset)
     state.bm25_index = bm25
+    _normalized_texts(dataset)  # warm text_search's normalized-text cache
 
     logger.info(
         "Lookup tables ready: %d Quran verses | %d hadith refs | %d surah keys | %d BM25 terms",
@@ -710,6 +901,7 @@ async def lifespan(app: FastAPI):
 
     logger.info("Building lookup tables...")
     _build_lookup_tables(state.dataset)
+    state.type_indexes = build_type_indexes(state.faiss_index, state.dataset)
 
     logger.info("Initializing LLM provider: %s", cfg.LLM_BACKEND)
     state.llm = get_llm_provider()
@@ -719,18 +911,18 @@ async def lifespan(app: FastAPI):
         try:
             logger.info("Loading cross-encoder reranker: %s", cfg.RERANK_MODEL)
             from sentence_transformers import CrossEncoder
-            state.reranker = CrossEncoder(cfg.RERANK_MODEL)
+            state.reranker = CrossEncoder(cfg.RERANK_MODEL, max_length=512)
             logger.info("Cross-encoder reranker ready")
         except Exception as exc:
             logger.warning("Could not load CrossEncoder (%s), continuing without reranker", exc)
 
     state.ready = True
     logger.info(
-        "QModel v8 ready | backend=%s | dataset=%d | faiss=%d | threshold=%.2f",
+        "QModel v8.5 ready | backend=%s | dataset=%d | faiss=%d | gate=%s",
         cfg.LLM_BACKEND,
         len(state.dataset) if state.dataset else 0,
         state.faiss_index.ntotal if state.faiss_index else 0,
-        cfg.CONFIDENCE_THRESHOLD,
+        f"reranker≥{cfg.RERANK_MIN_SCORE}" if state.reranker else f"dense≥{cfg.DENSE_MIN_SCORE}",
     )
     yield
     state.ready = False
@@ -749,6 +941,43 @@ def check_ready():
 # ═══════════════════════════════════════════════════════════════════════
 # SHARED PRE-PROCESSING (steps 1-7, used by both pipeline variants)
 # ═══════════════════════════════════════════════════════════════════════
+_SOURCE_QURAN_RE = re.compile(
+    # normalize_arabic maps قران → قرآن (_SPELLING_MAP), so accept both spellings.
+    r"(?:^|[\s:،])(?:ايه|الايه|ايات|الايات|قر[اآ]ن|القر[اآ]ن|سوره|السوره)(?:[\s:،]|$)|في القر[اآ]ن|ايات من"
+    r"|\b(?:qur'?an|koran|verses?|ayahs?|ayat|surahs?|suras?)\b",
+    re.I,
+)
+_SOURCE_HADITH_RE = re.compile(
+    r"(?:^|[\s:،])(?:حديث|الحديث|احاديث|الاحاديث)(?:[\s:،]|$)|في الحديث|حديث نبوي|قال رسول الله"
+    r"|\b(?:hadiths?|ahadith|sunnah|the prophet|messenger of allah)\b",
+    re.I,
+)
+_SOURCE_FATWA_RE = re.compile(
+    r"(?:^|[\s:،])(?:فتوي|الفتوي|فتاوي|الفتاوي)(?:[\s:،]|$)|حكم الشرع|هل يجوز|هل يحرم"
+    r"|\bfatwas?\b",
+    re.I,
+)
+
+
+def _detect_source_type(question: str) -> Optional[Literal["quran", "hadith", "fatwa"]]:
+    """Restrict retrieval to one source when the question names it (Arabic or English).
+
+    A question naming both the Qur'an and hadith ("Quran and Sunnah") stays unfiltered.
+    """
+    norm_q = normalize_arabic(question).lower()
+    quran = bool(_SOURCE_QURAN_RE.search(norm_q))
+    hadith = bool(_SOURCE_HADITH_RE.search(norm_q))
+    if quran and hadith:
+        return None
+    if quran:
+        return "quran"
+    if hadith:
+        return "hadith"
+    if _SOURCE_FATWA_RE.search(norm_q):
+        return "fatwa"
+    return None
+
+
 async def _rag_preprocess(
     question: str,
     top_k: int,
@@ -767,13 +996,7 @@ async def _rag_preprocess(
     
     # 1b. Override source_type if explicitly mentioned in query
     if not source_type:
-        norm_q = normalize_arabic(question).lower()
-        if re.search(r"(?:^|[\s:،])(?:ايه|الايه|ايات|الايات|قران|القران|سوره|السوره)(?:[\s:،]|$)|في القران|ايات من", norm_q):
-            source_type = "quran"
-        elif re.search(r"(?:^|[\s:،])(?:حديث|الحديث|احاديث|الاحاديث)(?:[\s:،]|$)|في الحديث|حديث نبوي|قال رسول الله", norm_q):
-            source_type = "hadith"
-        elif re.search(r"(?:^|[\s:،])(?:فتوي|الفتوي|فتاوي|الفتاوي)(?:[\s:،]|$)|حكم الشرع|هل يجوز|هل يحرم", norm_q):
-            source_type = "fatwa"
+        source_type = _detect_source_type(question)
 
     # 2. Concurrent: surah info + analysis intent + hybrid search (with multi-vector & reranker)
     surah_task  = detect_surah_info(question, rewrite)
@@ -784,8 +1007,9 @@ async def _rag_preprocess(
         top_k, source_type, grade_filter,
         bm25_index=state.bm25_index,
         reranker_model=state.reranker,
+        type_indexes=state.type_indexes,
     )
-    surah_det, analysis_kw, results = await asyncio.gather(
+    surah_det, analysis_kw, hybrid_results = await asyncio.gather(
         surah_task, kw_task, search_task,
     )
 
@@ -808,7 +1032,6 @@ async def _rag_preprocess(
                 lookup_hadith_references(query, state.dataset, limit=top_k,
                                          hadith_idx=state.hadith_ref_idx)
             )
-    results = merge_search_results(direct_results, results, limit=top_k)
 
     # 2c. Text search fallback (offloaded to thread worker)
     ar_q = rewrite.get("ar_query", "")
@@ -819,7 +1042,7 @@ async def _rag_preprocess(
         text_src = "hadith"
 
     text_limit = top_k * 2 if intent in ("auth", "hadith", "tafsir") else top_k
-    text_results = []
+    text_results: list = []
     for q in dict.fromkeys([ar_q, question]):
         if not q:
             continue
@@ -844,12 +1067,18 @@ async def _rag_preprocess(
             ))
             hits = hits[:20]
 
-        for hit in hits:
-            if intent == "auth" and hit.get("_score", 0) > 2.0:
-                hit = {**hit, "_score": hit["_score"] + 1.0}
-            text_results.append(hit)
-    results = merge_search_results(results, text_results, limit=max(top_k * 3, 20))
+        hits = [
+            {**hit, "_score": hit["_score"] + 1.0}
+            if intent == "auth" and hit.get("_score", 0) > 2.0 else hit
+            for hit in hits
+        ]
+        text_results.append((q, hits))
+    results = fuse_search_results(
+        direct_results, hybrid_results, text_results, limit=max(top_k * 3, 20),
+    )
     results = cluster_hadith_candidates(results)
+    if state.reranker is not None:
+        results = await _rerank_by_relevance(question, results)
     results = results[:top_k]
     # Apply Ayah context expansion windowing
     results = expand_quran_context(results, state.quran_verse_idx, window_size=cfg.AYAH_WINDOW_SIZE)
@@ -881,9 +1110,17 @@ async def _rag_preprocess(
     lang      = detect_language(question)
     top_score = results[0].get("_score", 0.0) if results else 0.0
 
+    # 5. Confidence: is any retrieved passage actually relevant?
+    if surah_info or (analysis and analysis.get("total_count")):
+        confident, confidence, conf_source = True, 1.0, "metadata"
+    else:
+        confident, confidence, conf_source = await _assess_confidence(
+            question, results, hybrid_results,
+        )
+
     logger.info(
-        "Search done | intent=%s | top_score=%.3f | threshold=%.2f",
-        intent, top_score, cfg.CONFIDENCE_THRESHOLD,
+        "Search done | intent=%s | top_score=%.3f | confidence=%.3f (%s) | confident=%s",
+        intent, top_score, confidence, conf_source, confident,
     )
 
     return {
@@ -894,12 +1131,99 @@ async def _rag_preprocess(
         "lang":       lang,
         "top_score":  top_score,
         "intent":     intent,
+        "confident":  confident,
+        "confidence": confidence,
+        "confidence_source": conf_source,
     }
 
 
+async def _rerank_by_relevance(question: str, results: list) -> list:
+    """Order the fused pool by cross-encoder relevance; pinned lexical hits stay first.
+
+    Scores are stored as ``_cross_score`` so the confidence gate reuses them.
+    """
+    try:
+        probs = await score_relevance(question, results, state.reranker)
+    except Exception as exc:
+        logger.warning("Final reranking failed (%s) — keeping fused order", exc)
+        return results
+    scored = [{**r, "_cross_score": p} for r, p in zip(results, probs)]
+    pinned = [r for r in scored if r.get("_pinned")]
+    rest = sorted((r for r in scored if not r.get("_pinned")),
+                  key=lambda r: r["_cross_score"], reverse=True)
+    return pinned + rest
+
+
+# Creative-writing requests are out of scope even when retrieval finds loosely related
+# verses (an ocean poem matches verses about the sea). Requires a writing verb plus an
+# indefinite creative form, so "the story of Prophet Yusuf" or "is listening to songs
+# haram?" still pass; Arabic needs قصيدة/أبيات rather than شعر, which also means hair.
+_CREATIVE_REQUEST_RE = re.compile(
+    r"\b(?:write|compose|create|generate|make up)\s+(?:me\s+)?(?:a|an|some)\s+(?:\w+\s+){0,2}?"
+    r"(?:poem|poetry|story|song|lyrics|rap|joke|limerick|haiku)s?\b"
+    r"|\btell me (?:a|some) jokes?\b"
+    r"|(?:اكتب|أكتب|اكتبي|ألف|الف|أنظم|انظم|اعطني|أعطني)\s+(?:لي\s+)?(?:\S+\s+){0,2}?"
+    r"(?:قصيدة|قصيده|أبيات|ابيات|أغنية|اغنية|نكتة|نكته)",
+    re.IGNORECASE,
+)
+
+
+async def _assess_confidence(
+    question: str, results: list, hybrid_results: list,
+) -> Tuple[bool, float, str]:
+    """Decide whether retrieval found something genuinely relevant.
+
+    Rank-fusion scores only say how candidates compare to each other — an
+    off-topic question still has a #1 result — so the gate needs an absolute
+    signal: lexical evidence first, then the cross-encoder, then dense cosine.
+    """
+    if _CREATIVE_REQUEST_RE.search(question):
+        return False, 0.0, "scope"
+    if any(r.get("_pinned") for r in results):
+        return True, 1.0, "lexical"
+    if state.reranker is not None and results:
+        try:
+            probs = await score_relevance(question, results, state.reranker)
+            best = max(probs)
+            return best >= cfg.RERANK_MIN_SCORE, best, "reranker"
+        except Exception as exc:
+            logger.warning("Relevance scoring failed (%s) — falling back to dense similarity", exc)
+    best = max((r.get("_dense", 0.0) for r in hybrid_results), default=0.0)
+    return best >= cfg.DENSE_MIN_SCORE, best, "dense"
+
+
 # ═══════════════════════════════════════════════════════════════════════
 # CORE RAG PIPELINE  (non-streaming, with hallucination checks)
 # ═══════════════════════════════════════════════════════════════════════
+# Chinese / Japanese / Korean script, incl. CJK punctuation (、。) and fullwidth forms (（）).
+# The Qwen backbone occasionally drifts into Chinese mid-answer despite the prompt rules.
+_CJK_RE = re.compile(r"[\u3000-\u303F\u3040-\u30FF\u3400-\u4DBF\u4E00-\u9FFF\uAC00-\uD7AF\uFF00-\uFFEF]+")
+_STREAM_MAX_CJK = 20
+
+
+def _split_thinking(raw: str) -> Tuple[str, Optional[str]]:
+    """Separate a Qwen-style <think> block from the answer."""
+    thinking = None
+    if getattr(cfg, "SHOW_THINKING", False):
+        think_match = re.search(r"<think>([\s\S]*?)</think>", raw, flags=re.IGNORECASE)
+        if think_match:
+            thinking = think_match.group(1).strip()
+    answer = re.sub(r"<think>[\s\S]*?</think>", "", raw, flags=re.IGNORECASE)
+    answer = re.sub(r"<think>[\s\S]*$", "", answer, flags=re.IGNORECASE)
+    return answer.strip(), thinking
+
+
+def _with_language_reminder(messages: List[dict], lang: str) -> List[dict]:
+    """Append a stricter same-language instruction to the last user message."""
+    reminder = (
+        "\n\nتنبيه: اكتب الإجابة كاملة باللغة العربية فقط، دون أي حرف صيني."
+        if lang == "arabic" else
+        "\n\nReminder: write the whole answer in English only, with no Chinese characters."
+    )
+    last = messages[-1]
+    return messages[:-1] + [{**last, "content": last["content"] + reminder}]
+
+
 async def run_rag_pipeline(
     question: str,
     top_k: int = cfg.TOP_K_RETURN,
@@ -926,13 +1250,11 @@ async def run_rag_pipeline(
     top_score = ctx["top_score"]
     intent    = ctx["intent"]
 
-    # 5. Confidence gate (skip for surah_info)
-    # Use dynamic threshold based on intent
-    threshold = cfg.INTENT_THRESHOLDS.get(intent, cfg.CONFIDENCE_THRESHOLD)
-    if not surah_info and top_score < threshold:
+    # 5. Confidence gate
+    if not ctx["confident"]:
         logger.warning(
-            "Low confidence (%.3f < %.2f for intent %s) — returning safe fallback",
-            top_score, threshold, intent,
+            "Low confidence (%s %.3f, intent %s) — returning safe fallback",
+            ctx["confidence_source"], ctx["confidence"], intent,
         )
         return {
             "answer":     not_found_answer(lang),
@@ -967,22 +1289,24 @@ async def run_rag_pipeline(
     messages = build_messages(context, question, lang, intent, analysis, surah_info)
 
     try:
-        answer = await state.llm.chat(
+        answer, thinking = _split_thinking(await state.llm.chat(
             messages,
             max_tokens=max_tokens or cfg.MAX_TOKENS,
             temperature=cfg.TEMPERATURE,
-        )
-        # Extract reasoning content (thinking block)
-        thinking = None
-        if getattr(cfg, "SHOW_THINKING", False):
-            think_match = re.search(r"<think>([\s\S]*?)</think>", answer, flags=re.IGNORECASE)
-            if think_match:
-                thinking = think_match.group(1).strip()
-            
-        # Strip residual <think> blocks that Qwen models may emit
-        answer = re.sub(r"<think>[\s\S]*?</think>", "", answer, flags=re.IGNORECASE)
-        answer = re.sub(r"<think>[\s\S]*$", "", answer, flags=re.IGNORECASE)
-        answer = answer.strip()
+        ))
+
+        # Language drift into Chinese: regenerate once, deterministically and with a
+        # stricter reminder; if it persists, fall back to listing the retrieved sources.
+        if _CJK_RE.search(answer):
+            logger.warning("CJK characters in answer — regenerating once")
+            answer, thinking = _split_thinking(await state.llm.chat(
+                _with_language_reminder(messages, lang),
+                max_tokens=max_tokens or cfg.MAX_TOKENS,
+                temperature=0.0,
+            ))
+            if _CJK_RE.search(answer):
+                logger.warning("CJK characters persist after retry — using results fallback")
+                answer = _results_fallback(results, lang) if results else not_found_answer(lang)
 
         if not answer:
             logger.warning("LLM returned empty answer — using results fallback")
@@ -1001,6 +1325,7 @@ async def run_rag_pipeline(
     answer = _verify_references(answer, results)
     if surah_info:
         answer = _verify_surah_info(answer, surah_info)
+    answer = _attach_source_refs(answer, results, lang)
 
     latency = int((time.perf_counter() - t0) * 1000)
     logger.info(
@@ -1025,6 +1350,16 @@ async def run_rag_pipeline(
 # ═══════════════════════════════════════════════════════════════════════
 # STREAMING RAG PIPELINE  (token streaming with sentence verification)
 # ═══════════════════════════════════════════════════════════════════════
+_STREAM_QUOTE_SPAN = re.compile(r"❝[^❞]*❞|«[^»]*»|﴿[^﴾]*﴾")
+# Long sentences (an isnad is one sentence of narrator names) are released at a comma
+# after _STREAM_SOFT_SPLIT chars, or at a space after _STREAM_HARD_SPLIT, so the reader
+# isn't left waiting. Never within two words after سورة / الآية / Surah / verse: the
+# verifier must see "سورة البقرة، الآية 255" whole.
+_STREAM_SOFT_SPLIT = 120
+_STREAM_HARD_SPLIT = 300
+_REF_OPENING = re.compile(r"(?:سورة|سوره|surah|sura|الآية|الاية|آية|ayah|verse)(?:\s+\S+){0,2}\s*$", re.I)
+
+
 class StreamingVerificationBuffer:
     """Sentence-buffered streaming citation & reference verification."""
 
@@ -1032,40 +1367,61 @@ class StreamingVerificationBuffer:
         self.results = results
         self.surah_info = surah_info
         self.buffer = ""
+        self.cjk_chars = 0  # CJK characters stripped so far (language drift)
+        self.emitted = ""   # verified text sent so far
+
+    def _clean(self, chunk: str) -> str:
+        chunk = _clean_links(_verify_citations(chunk, self.results))
+        chunk = _verify_references(chunk, self.results)
+        if self.surah_info:
+            chunk = _verify_surah_info(chunk, self.surah_info)
+        # Tokens already sent can't be regenerated, so drop CJK runs instead
+        self.cjk_chars += sum(len(m) for m in _CJK_RE.findall(chunk))
+        chunk = _CJK_RE.sub("", chunk)
+        self.emitted += chunk
+        return chunk
 
     def feed(self, token: str) -> List[str]:
         self.buffer += token
         emitted: List[str] = []
         while True:
             # If a sacred quote was opened but not yet closed, hold until closed (up to 400 chars)
-            if ("❝" in self.buffer and "❞" not in self.buffer and len(self.buffer) < 400) or \
-               ("«" in self.buffer and "»" not in self.buffer and len(self.buffer) < 400):
+            if len(self.buffer) < 400 and any(o in self.buffer and c not in self.buffer
+                                              for o, c in (("❝", "❞"), ("«", "»"), ("﴿", "﴾"))):
                 break
 
-            match = re.search(r"(\n|\. |\? |؟ |└─+┘)", self.buffer)
-            if not match:
-                break
-            idx = match.end()
+            # First sentence boundary outside any complete quote, so quotes reach the verifier whole
+            quotes = [q.span() for q in _STREAM_QUOTE_SPAN.finditer(self.buffer)]
+            match = next((m for m in re.finditer(r"(\n|\. |\? |؟ |└─+┘)", self.buffer)
+                          if not any(qs < m.start() < qe for qs, qe in quotes)), None)
+            if match:
+                idx = match.end()
+            else:
+                idx = self._soft_split(quotes)
+                if idx is None:
+                    break
             chunk = self.buffer[:idx]
             self.buffer = self.buffer[idx:]
-
-            chunk = _verify_citations(chunk, self.results)
-            chunk = _verify_references(chunk, self.results)
-            if self.surah_info:
-                chunk = _verify_surah_info(chunk, self.surah_info)
-            emitted.append(chunk)
+            emitted.append(self._clean(chunk))
         return emitted
 
+    def _soft_split(self, quotes: List[Tuple[int, int]]) -> Optional[int]:
+        """End of the last comma (else space) safe to split a long sentence at."""
+        for limit, pattern in ((_STREAM_SOFT_SPLIT, r"[،,] "), (_STREAM_HARD_SPLIT, r" ")):
+            if len(self.buffer) < limit:
+                return None
+            for m in reversed(list(re.finditer(pattern, self.buffer))):
+                if any(qs < m.start() < qe for qs, qe in quotes) or _REF_OPENING.search(self.buffer[:m.start()]):
+                    continue
+                return m.end()
+        return None
+
     def flush(self) -> str:
         if not self.buffer:
             return ""
         chunk = self.buffer
         self.buffer = ""
-        chunk = _verify_citations(chunk, self.results)
-        chunk = _verify_references(chunk, self.results)
-        if self.surah_info:
-            chunk = _verify_surah_info(chunk, self.surah_info)
-        return chunk
+        return self._clean(chunk)
 
 
 async def run_rag_pipeline_stream(
@@ -1104,7 +1460,7 @@ async def run_rag_pipeline_stream(
     }
 
     # ── Deterministic / fallback paths ────────────────────────────────
-    if not surah_info and top_score < cfg.CONFIDENCE_THRESHOLD:
+    if not ctx["confident"]:
         yield {"type": "token", "content": not_found_answer(lang)}
         yield {"type": "done",  "latency_ms": int((time.perf_counter() - t0) * 1000)}
         return
@@ -1151,10 +1507,22 @@ async def run_rag_pipeline_stream(
             else:
                 for verified_chunk in verifier.feed(token):
                     yield {"type": "token", "content": verified_chunk}
-
-        remaining = verifier.flush()
-        if remaining:
-            yield {"type": "token", "content": remaining}
+                if verifier.cjk_chars > _STREAM_MAX_CJK:
+                    break
+
+        if verifier.cjk_chars > _STREAM_MAX_CJK:
+            # The model switched language; stop rather than stream a gutted answer
+            logger.warning("CJK language drift in stream — generation stopped")
+            yield {"type": "token", "content": (
+                "\n\n⚠️ توقفت الإجابة بسبب خلل في لغة النموذج. يُرجى إعادة المحاولة." if lang == "arabic" else
+                "\n\n⚠️ The answer was cut short by a model language error. Please try again.")}
+        else:
+            remaining = verifier.flush()
+            if remaining:
+                yield {"type": "token", "content": remaining}
+            links = _source_refs(verifier.emitted, results, lang)
+            if links:
+                yield {"type": "token", "content": f"\n\n{links}"}
 
     except Exception as exc:
         logger.error("LLM stream failed: %s", exc)

app/synonyms.py

@@ -29,6 +29,14 @@ ISLAMIC_THESAURUS: Dict[str, List[str]] = {
     "mirath":      ["inheritance", "ميراث", "فرائض"],
     "fara'id":     ["inheritance", "فرائض", "laws of inheritance"],
     "will":        ["وصية", "wasiyyah"],
+    # The Qur'an says "wine" (khamr); questions say "alcohol"
+    "alcohol":     ["khamr", "wine", "intoxicants", "خمر", "مسكر"],
+    "khamr":       ["wine", "alcohol", "خمر", "مسكر"],
+    "wine":        ["khamr", "alcohol", "خمر"],
+    "خمر":         ["مسكر", "wine", "alcohol"],
+    # "مصارف" alone also means "banks"; pin the zakat sense when the phrase is used
+    "مصارف الزكاة":  ["مستحقي الزكاة", "أصناف الزكاة", "إنما الصدقات للفقراء", "zakat recipients"],
+    "zakat recipients": ["مصارف الزكاة", "أصناف الزكاة", "إنما الصدقات للفقراء"],
     "wasiyyah":    ["will", "وصية", "testament"],
     
     # Technical terms
@@ -52,8 +60,42 @@ ISLAMIC_THESAURUS: Dict[str, List[str]] = {
     "jihad":       ["striving", "جهاد", "struggle"],
     "intercession":["shafa'ah", "شفاعة"],
     "shafa'ah":    ["intercession", "شفاعة"],
+
+    # Arabic keys with the English used in Qur'an / hadith translations, so Arabic
+    # questions also reach English text
+    "سواك":   ["siwak", "miswak", "tooth-stick"],
+    "وضوء":   ["wudu", "ablution", "wash your faces", "فاغسلوا وجوهكم"],
+    # 5:6 never says "ablution": "when you rise to prayer, wash your faces and your forearms"
+    "ablution": ["wudu", "wash your faces", "الوضوء", "فاغسلوا وجوهكم"],
+    "wudu":     ["ablution", "wash your faces", "الوضوء", "فاغسلوا وجوهكم"],
+    "ربا":    ["riba", "usury", "interest"],
+    "حج":     ["hajj", "pilgrimage"],
+    "ميراث":  ["inheritance", "mirath", "فرائض"],
+    "زكاة":   ["zakat", "alms", "charity"],
+    "صيام":   ["fasting", "sawm"],
+    "صبر":    ["patience", "sabr"],
+    "debt":   ["دين", "تداينتم", "loan"],
 }
 
+def synonyms_in_text(text: str) -> List[str]:
+    """Synonyms of every thesaurus term that occurs in *text*.
+
+    Single words match by light stem (السواك ↔ سواك, debts ↔ debt), multi-word
+    terms by phrase. Used by the fast (no-LLM) query rewrite.
+    """
+    from app.arabic_nlp import light_stem, normalize_arabic
+
+    norm = normalize_arabic(text, aggressive=True).lower()
+    stems = {light_stem(w) for w in norm.split()} | {w.rstrip("s") for w in norm.split()}
+    found: List[str] = []
+    for key, syns in ISLAMIC_THESAURUS.items():
+        key_norm = normalize_arabic(key, aggressive=True).lower()
+        hit = key_norm in norm if " " in key_norm else (light_stem(key_norm) in stems or key_norm.rstrip("s") in stems)
+        if hit:
+            found.extend(x for x in syns if x not in found)
+    return found
+
+
 def expand_synonyms(keywords: List[str]) -> List[str]:
     """Expand a list of keywords with synonyms from the thesaurus."""
     expanded = set(keywords)

build_index.py

@@ -33,7 +33,7 @@ import zipfile
 import numpy as np
 from pathlib import Path
 from collections import defaultdict
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
 
 import faiss
 import requests
@@ -46,6 +46,13 @@ CACHE_DIR = BASE_DIR / "data" / "cache"
 METADATA_PATH = BASE_DIR / "metadata.json"
 INDEX_PATH = BASE_DIR / "QModel.index"
 
+# Fatwa sources, loaded in order (earlier copies win on duplicates):
+# hand-collected data/fatwas/, then data_pipeline/scrape_fatwas.py output.
+FATWA_DIRS = [BASE_DIR / "data" / "fatwas", BASE_DIR / "data" / "fatwas_scraped"]
+# Placeholder records whose text doesn't match the Islamweb pages their URLs point to;
+# indexing them would cite unrelated fatwas as sources.
+FATWA_EXCLUDE = [BASE_DIR / "data" / "fatwas" / "islamweb.json"]
+
 # ── Quran source URLs ─────────────────────────────────────────────────
 QURAN_JSON_URL = (
     "https://raw.githubusercontent.com/risan/quran-json/main/data/quran.json"
@@ -968,26 +975,67 @@ def build_hadith_entries(
 # STEP 5: GENERATE METADATA
 # ══════════════════════════════════════════════════════════════════════
 
-def load_fatwa_entries() -> List[Dict]:
-    """Load fatwa entries from data/fatwas/*.json if present."""
-    fatwas_dir = BASE_DIR / "data" / "fatwas"
+def _fatwa_content_key(record: Dict) -> Tuple:
+    # BinBaz ids redirect (legacy 4213 → canonical 1307), so the same fatwa can arrive
+    # under two ids from different scrapes; match on content as well.
+    return (
+        record.get("source"),
+        " ".join(record.get("title", "").split()),
+        " ".join(record.get("answer_ar", "").split())[:200],
+    )
+
+
+# Scraped pages that list audio lectures rather than answer a question
+_FATWA_JUNK_MARKERS = ("المواد الصوتية تحت هذا التصنيف", "إم بي ثري")
+
+
+def _fatwa_has_content(record: Dict) -> bool:
+    answer = (record.get("answer_ar") or record.get("answer_en") or record.get("english") or "").strip()
+    return bool(answer) and not any(m in answer for m in _FATWA_JUNK_MARKERS)
+
+
+def load_fatwa_entries(
+    dirs: Sequence[Path] = FATWA_DIRS,
+    exclude: Iterable[Path] = FATWA_EXCLUDE,
+) -> List[Dict]:
+    """Load fatwas from each directory's *.json in order, skipping excluded files.
+
+    Earlier copies win when two records share an id or content.
+    """
+    excluded = {Path(p).resolve() for p in exclude}
     entries: List[Dict] = []
-    if not fatwas_dir.exists():
-        return entries
+    seen_ids: set = set()
+    seen_content: set = set()
 
     print("\n" + "=" * 60)
     print("Step 4c: Loading Fatwa Entries")
     print("=" * 60)
 
-    for json_file in fatwas_dir.glob("*.json"):
-        try:
-            with open(json_file, "r", encoding="utf-8") as f:
-                data = json.load(f)
-                if isinstance(data, list):
-                    entries.extend(data)
-                    print(f"  ✓ Loaded {len(data):,} fatwas from {json_file.name}")
-        except Exception as exc:
-            print(f"  ✗ Error loading {json_file.name}: {exc}")
+    for fatwas_dir in dirs:
+        for json_file in sorted(Path(fatwas_dir).glob("*.json")):
+            if json_file.resolve() in excluded:
+                print(f"  - Skipping {json_file} (excluded)")
+                continue
+            try:
+                with open(json_file, "r", encoding="utf-8") as f:
+                    data = json.load(f)
+            except Exception as exc:
+                print(f"  ✗ Error loading {json_file}: {exc}")
+                continue
+            if not isinstance(data, list):
+                continue
+            kept = 0
+            for record in data:
+                key = _fatwa_content_key(record)
+                if record.get("type") != "fatwa" or record.get("id") in seen_ids or key in seen_content:
+                    continue
+                if not _fatwa_has_content(record):
+                    continue
+                seen_ids.add(record["id"])
+                seen_content.add(key)
+                entries.append(record)
+                kept += 1
+            print(f"  ✓ Loaded {kept:,} fatwas from {json_file} ({len(data) - kept} duplicates/empty skipped)")
 
     print(f"  Total Fatwa entries: {len(entries):,}")
     return entries
@@ -1030,6 +1078,32 @@ def generate_metadata(
 # STEP 6: BUILD FAISS INDEX
 # ══════════════════════════════════════════════════════════════════════
 
+def passage_text(doc: Dict) -> str:
+    """Text embedded for a document (e5 asymmetric models require the 'passage: ' prefix)."""
+    doc_type = doc.get("type")
+    if doc_type == "quran":
+        # Include both English and Arabic tafsir snippets for rich bilingual matching
+        tafsir_en_snip = doc.get("tafsir_en", "")[:350]
+        tafsir_ar_snip = doc.get("tafsir_ar", "")[:350]
+        text = (
+            f"passage: {doc.get('arabic', '')} {doc.get('english', '')} "
+            f"{tafsir_en_snip} {tafsir_ar_snip}"
+        )
+    elif doc_type == "fatwa":
+        text = (
+            f"passage: {doc.get('source', '')} {doc.get('scholar_or_council', '')} "
+            f"{doc.get('title', '')} {doc.get('question_ar', '')} "
+            f"{doc.get('answer_ar', '')[:500]}"
+        )
+    else:  # hadith
+        text = (
+            f"passage: {doc.get('collection', '')} {doc.get('chapter', '')} "
+            f"{doc.get('arabic', '')} "
+            f"{doc.get('english', '')}"
+        )
+    return text.strip()
+
+
 def build_faiss_index(
     documents: List[Dict],
     model_name: str = DEFAULT_EMBED_MODEL,
@@ -1046,31 +1120,7 @@ def build_faiss_index(
     embedding_dim = model.get_sentence_embedding_dimension()
     print(f"  Embedding dimension: {embedding_dim}")
 
-    # Build text for each document (e5 asymmetric models require 'passage: ' prefix for documents)
-    all_texts: List[str] = []
-    for doc in documents:
-        doc_type = doc.get("type")
-        if doc_type == "quran":
-            # Include both English and Arabic tafsir snippets for rich bilingual matching
-            tafsir_en_snip = doc.get("tafsir_en", "")[:350]
-            tafsir_ar_snip = doc.get("tafsir_ar", "")[:350]
-            text = (
-                f"passage: {doc.get('arabic', '')} {doc.get('english', '')} "
-                f"{tafsir_en_snip} {tafsir_ar_snip}"
-            )
-        elif doc_type == "fatwa":
-            text = (
-                f"passage: {doc.get('source', '')} {doc.get('scholar_or_council', '')} "
-                f"{doc.get('title', '')} {doc.get('question_ar', '')} "
-                f"{doc.get('answer_ar', '')[:500]}"
-            )
-        else:  # hadith
-            text = (
-                f"passage: {doc.get('collection', '')} {doc.get('chapter', '')} "
-                f"{doc.get('arabic', '')} "
-                f"{doc.get('english', '')}"
-            )
-        all_texts.append(text.strip())
+    all_texts: List[str] = [passage_text(doc) for doc in documents]
 
     print(f"\n  Generating embeddings for {len(all_texts):,} documents …")
     all_embeddings = []

data_pipeline/scrape_fatwas.py

@@ -7,7 +7,7 @@ Scrapes and normalizes fatwas from:
 2. BinBaz Portal (https://binbaz.org.sa/fatwas)
 3. Islamway (https://ar.islamway.net/fatawa)
 
-Outputs structured JSON files into data/fatwas/ directory.
+Outputs structured JSON files into data/fatwas_scraped/ (read by build_index.py).
 """
 
 import os
@@ -16,19 +16,17 @@ import time
 import re
 import argparse
 from pathlib import Path
-from typing import Dict, Any, List, Optional
+from typing import Dict, Any, List, Optional, Tuple
 import requests
 from bs4 import BeautifulSoup
 
 BASE_DIR = Path(__file__).resolve().parent.parent
-FATWAS_DATA_DIR = BASE_DIR / "data" / "fatwas"
+# Hand-collected fatwas live in data/fatwas/; build_index.FATWA_DIRS reads both.
+FATWAS_DATA_DIR = BASE_DIR / "data" / "fatwas_scraped"
 
+# Identify the crawler honestly; if a site refuses it, skip that site rather than disguise requests.
 HEADERS = {
-    "User-Agent": (
-        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
-        "AppleWebKit/537.36 (KHTML, like Gecko) "
-        "Chrome/120.0.0.0 Safari/537.36"
-    )
+    "User-Agent": "QModelFatwaIndexer/1.0 (+https://huggingface.co/aelgendy/QModel)",
 }
 
 try:
@@ -38,13 +36,16 @@ except ImportError:
     PARSER = "html.parser"
 
 
-def fetch_html(url: str, timeout: int = 10, retries: int = 3) -> Optional[str]:
-    """Fetch URL with retries and exponential backoff."""
+def fetch_page(url: str, timeout: int = 10, retries: int = 3) -> Optional[Tuple[str, str]]:
+    """Fetch URL with retries and exponential backoff; returns (final_url, html) after redirects."""
     for attempt in range(retries):
         try:
             resp = requests.get(url, headers=HEADERS, timeout=timeout)
             if resp.status_code == 200:
-                return resp.text
+                # These sites are UTF-8; requests falls back to ISO-8859-1 when the
+                # header omits a charset, which garbles Arabic.
+                resp.encoding = "utf-8"
+                return resp.url, resp.text
             if resp.status_code == 404:
                 return None
             if resp.status_code == 429:
@@ -57,6 +58,17 @@ def fetch_html(url: str, timeout: int = 10, retries: int = 3) -> Optional[str]:
     return None
 
 
+def fetch_html(url: str, timeout: int = 10, retries: int = 3) -> Optional[str]:
+    page = fetch_page(url, timeout=timeout, retries=retries)
+    return page[1] if page else None
+
+
+def canonical_id(final_url: str, requested_id: int) -> int:
+    """Old fatwa ids redirect to the canonical one; key records on that to avoid duplicates."""
+    m = re.search(r"/fatwas?/(\d+)", final_url)
+    return int(m.group(1)) if m else requested_id
+
+
 class IslamwebExtractor:
     SOURCE_NAME = "Islamweb"
     DEFAULT_SCHOLAR = "مركز الفتوى - إسلام ويب"
@@ -65,27 +77,27 @@ class IslamwebExtractor:
     def scrape_id(fatwa_id: int, timeout: int = 10) -> Optional[Dict[str, Any]]:
         url = f"https://www.islamweb.net/ar/fatwa/{fatwa_id}/"
         try:
-            html = fetch_html(url, timeout=timeout)
-            if not html:
+            page = fetch_page(url, timeout=timeout)
+            if not page:
                 return None
+            url, html = page
+            fatwa_id = canonical_id(url, fatwa_id)
 
             soup = BeautifulSoup(html, PARSER)
-            title_el = soup.select_one('.main-title, h1, title')
+            title_el = soup.select_one('.item-fatwa h2, h2')
             title = title_el.get_text(" ", strip=True) if title_el else ""
-            if "فتاوى إسلام ويب" in title:
-                title = title.replace("فتاوى إسلام ويب", "").strip()
-
-            # Question extraction
-            q_el = soup.select_one('.fatwa-details-text, .fatwa-details, .fatwa-question')
-            question = q_el.get_text("\n", strip=True) if q_el else ""
-            question = re.sub(r'^\s*السؤال\s*:?\s*', '', question).strip()
-
-            # Answer extraction
-            ans_el = soup.select_one('.fatwa-answer, .fatwa_answer')
-            answer = ans_el.get_text("\n", strip=True) if ans_el else ""
-            answer = re.sub(r'^\s*الإجابة\s*:?\s*', '', answer).strip()
-            
-            if not title and not answer:
+
+            # Layout: two .quest-fatwa blocks — "السؤال …" then "الإجابــة …"
+            blocks = soup.select('.quest-fatwa')
+            question, answer = "", ""
+            for block in blocks:
+                text = block.get_text("\n", strip=True)
+                if "السؤال" in text[:200] and not question:
+                    question = re.sub(r'^[\s\S]*?السؤال\s*:?\s*', '', text, count=1).strip()
+                elif re.search(r'الإجاب\S*', text[:40]) and not answer:
+                    answer = re.sub(r'^\s*الإجاب\S*\s*:?\s*', '', text, count=1).strip()
+
+            if not title or not answer:
                 return None
 
             topics = []
@@ -122,9 +134,11 @@ class BinBazExtractor:
     def scrape_id(fatwa_id: int, timeout: int = 10) -> Optional[Dict[str, Any]]:
         url = f"https://binbaz.org.sa/fatwas/{fatwa_id}"
         try:
-            html = fetch_html(url, timeout=timeout)
-            if not html:
+            page = fetch_page(url, timeout=timeout)
+            if not page:
                 return None
+            url, html = page
+            fatwa_id = canonical_id(url, fatwa_id)
 
             soup = BeautifulSoup(html, PARSER)
             title_el = soup.select_one('h1, .article-title')
@@ -307,9 +321,11 @@ def run_scraper(source: str, start_id: int, end_id: int, delay: float = 1.0):
             continue
 
         record = extractor.scrape_id(fid)
-        if record:
+        if record and record["id"] in existing_ids:
+            print(f"[{source}] ID {fid} redirects to already-scraped {record['id']}")
+        elif record:
             new_records.append(record)
-            existing_ids.add(target_id)
+            existing_ids.add(record["id"])
             print(f"[{source}] Scraped ID {fid}: {record['title'][:50]}")
             # Checkpoint every 5 records so progress is not lost
             if len(new_records) % 5 == 0:

docker-compose.yml

@@ -52,7 +52,7 @@ services:
       interval: 30s
       timeout: 10s
       retries: 3
-      start_period: 60s
+      start_period: 300s   # model + index loading takes ~100 s, more on first start
 
 networks:
   qmodel-network:

eval/golden.jsonl

@@ -0,0 +1,147 @@
+{"id": "q-topic-en-01", "category": "quran_topic", "lang": "en", "query": "Which verse says Allah does not burden a soul beyond what it can bear?", "relevant": ["2:286", "2:233", "6:152", "7:42", "23:62", "65:7"]}
+{"id": "q-topic-en-02", "category": "quran_topic", "lang": "en", "query": "Verse about fasting being prescribed for the believers", "relevant": ["2:183"]}
+{"id": "q-topic-en-03", "category": "quran_topic", "lang": "en", "query": "In which month was the Quran revealed?", "relevant": ["2:185", "97:1", "44:3"], "required": ["2:185"]}
+{"id": "q-topic-en-04", "category": "quran_topic", "lang": "en", "query": "Allah says He is near and answers the call of the one who supplicates", "relevant": ["2:186"]}
+{"id": "q-topic-en-05", "category": "quran_topic", "lang": "en", "query": "Quran verses forbidding interest and usury", "relevant": ["2:275", "2:276", "2:278", "2:279", "3:130", "4:161", "30:39"]}
+{"id": "q-topic-en-06", "category": "quran_topic", "lang": "en", "query": "Seek help through patience and prayer", "relevant": ["2:45", "2:153"]}
+{"id": "q-topic-en-07", "category": "quran_topic", "lang": "en", "query": "Verse that says with hardship comes ease", "relevant": ["94:5", "94:6"]}
+{"id": "q-topic-en-08", "category": "quran_topic", "lang": "en", "query": "The verse of light", "relevant": ["24:35"]}
+{"id": "q-topic-en-09", "category": "quran_topic", "lang": "en", "query": "Quran command for women to cover themselves with head coverings", "relevant": ["24:31", "33:59"]}
+{"id": "q-topic-en-10", "category": "quran_topic", "lang": "en", "query": "Treat your parents kindly and do not say uff to them", "relevant": ["17:23", "17:24"], "required": ["17:23"]}
+{"id": "q-topic-en-11", "category": "quran_topic", "lang": "en", "query": "Killing one soul is like killing all of mankind", "relevant": ["5:32"]}
+{"id": "q-topic-en-12", "category": "quran_topic", "lang": "en", "query": "How is inheritance divided between sons and daughters?", "relevant": ["4:11", "4:12", "4:176"], "required": ["4:11"]}
+{"id": "q-topic-en-13", "category": "quran_topic", "lang": "en", "query": "How to perform ablution before prayer according to the Quran", "relevant": ["5:6", "4:43"], "required": ["5:6"]}
+{"id": "q-topic-en-14", "category": "quran_topic", "lang": "en", "query": "Is alcohol forbidden in the Quran?", "relevant": ["5:90", "5:91", "2:219"], "required": ["5:90"]}
+{"id": "q-topic-en-15", "category": "quran_topic", "lang": "en", "query": "Why did Allah create jinn and humans?", "relevant": ["51:56"]}
+{"id": "q-topic-en-16", "category": "quran_topic", "lang": "en", "query": "Do not despair of the mercy of Allah", "relevant": ["39:53", "12:87", "15:56"]}
+{"id": "q-topic-en-17", "category": "quran_topic", "lang": "en", "query": "Hearts find rest in the remembrance of Allah", "relevant": ["13:28"]}
+{"id": "q-topic-en-18", "category": "quran_topic", "lang": "en", "query": "Who is eligible to receive zakat?", "relevant": ["9:60"]}
+{"id": "q-topic-en-19", "category": "quran_topic", "lang": "en", "query": "Quran verse about avoiding suspicion, spying and backbiting", "relevant": ["49:12"]}
+{"id": "q-topic-en-20", "category": "quran_topic", "lang": "en", "query": "We made you into nations and tribes so that you may know one another", "relevant": ["49:13"]}
+{"id": "q-topic-en-21", "category": "quran_topic", "lang": "en", "query": "Which women is a man forbidden to marry?", "relevant": ["4:22", "4:23", "4:24", "2:221"], "required": ["4:23"]}
+{"id": "q-topic-en-22", "category": "quran_topic", "lang": "en", "query": "Can a man marry up to four wives?", "relevant": ["4:3", "4:129"], "required": ["4:3"]}
+{"id": "q-topic-en-23", "category": "quran_topic", "lang": "en", "query": "What foods are forbidden in the Quran?", "relevant": ["2:173", "5:3", "6:145", "16:115"]}
+{"id": "q-topic-en-24", "category": "quran_topic", "lang": "en", "query": "Who is obliged to perform Hajj?", "relevant": ["3:97", "2:196", "22:27"], "required": ["3:97"]}
+{"id": "q-topic-en-25", "category": "quran_topic", "lang": "en", "query": "Was Jesus crucified according to the Quran?", "relevant": ["4:157", "4:158"], "required": ["4:157"]}
+{"id": "q-topic-en-26", "category": "quran_topic", "lang": "en", "query": "Writing down debts and loan contracts", "relevant": ["2:282"]}
+{"id": "q-topic-en-27", "category": "quran_topic", "lang": "en", "query": "The day Allah perfected the religion", "relevant": ["5:3"]}
+{"id": "q-topic-en-28", "category": "quran_topic", "lang": "en", "query": "Luqman's advice to his son", "relevant": ["31:13", "31:14", "31:15", "31:16", "31:17", "31:18", "31:19"]}
+{"id": "q-topic-en-29", "category": "quran_topic", "lang": "en", "query": "The first revelation: read in the name of your Lord", "relevant": ["96:1", "96:2", "96:3", "96:4", "96:5"], "required": ["96:1"]}
+{"id": "q-topic-en-30", "category": "quran_topic", "lang": "en", "query": "The Night of Decree is better than a thousand months", "relevant": ["97:1", "97:2", "97:3", "97:4", "97:5"], "required": ["97:3"]}
+{"id": "q-topic-ar-01", "category": "quran_topic", "lang": "ar", "query": "ما هي آية الكرسي", "relevant": ["2:255"]}
+{"id": "q-topic-ar-02", "category": "quran_topic", "lang": "ar", "query": "آيات عن الصبر على البلاء", "relevant": ["2:155", "2:156", "2:157", "2:153", "2:45", "3:200", "39:10", "103:3", "16:127"]}
+{"id": "q-topic-ar-03", "category": "quran_topic", "lang": "ar", "query": "بر الوالدين في القرآن", "relevant": ["17:23", "17:24", "31:14", "46:15", "29:8", "4:36", "2:83", "6:151"]}
+{"id": "q-topic-ar-04", "category": "quran_topic", "lang": "ar", "query": "تحريم الخمر والميسر", "relevant": ["5:90", "5:91", "2:219"], "required": ["5:90"]}
+{"id": "q-topic-ar-05", "category": "quran_topic", "lang": "ar", "query": "مصارف الزكاة", "relevant": ["9:60", "fatwa_islamweb_748", "fatwa_islamweb_2057"]}
+{"id": "q-topic-ar-06", "category": "quran_topic", "lang": "ar", "query": "الوضوء في القرآن", "relevant": ["5:6", "4:43"], "required": ["5:6"]}
+{"id": "q-topic-ar-07", "category": "quran_topic", "lang": "ar", "query": "الدعوة إلى الله بالحكمة والموعظة الحسنة", "relevant": ["16:125"]}
+{"id": "q-topic-ar-08", "category": "quran_topic", "lang": "ar", "query": "الاعتصام بحبل الله وعدم التفرق", "relevant": ["3:103"]}
+{"id": "q-topic-ar-09", "category": "quran_topic", "lang": "ar", "query": "القتال في سبيل الله وعدم الاعتداء", "relevant": ["2:190", "2:191", "2:192", "2:193"], "required": ["2:190"]}
+{"id": "q-topic-ar-10", "category": "quran_topic", "lang": "ar", "query": "الحجاب في القرآن", "relevant": ["24:31", "33:59", "33:53"]}
+{"id": "q-topic-ar-11", "category": "quran_topic", "lang": "ar", "query": "التوبة ومغفرة الذنوب جميعا", "relevant": ["39:53", "4:110", "66:8", "25:70"]}
+{"id": "q-topic-ar-12", "category": "quran_topic", "lang": "ar", "query": "خلق الإنسان في أحسن تقويم", "relevant": ["95:4"]}
+{"id": "q-topic-ar-13", "category": "quran_topic", "lang": "ar", "query": "الأمر بالعدل والإحسان", "relevant": ["16:90", "4:58", "4:135", "5:8"]}
+{"id": "q-topic-ar-14", "category": "quran_topic", "lang": "ar", "query": "ليلة القدر", "relevant": ["97:1", "97:2", "97:3", "97:4", "97:5", "44:3", "bukhari_1944", "bukhari_35", "fatwa_islamweb_2762", "fatwa_islamweb_6198"]}
+{"id": "q-topic-ar-15", "category": "quran_topic", "lang": "ar", "query": "تحريم الربا", "relevant": ["2:275", "2:276", "2:278", "2:279", "3:130", "4:161", "30:39"]}
+{"id": "q-text-ar-01", "category": "quran_text", "lang": "ar", "query": "آية: إن مع العسر يسرا", "relevant": ["94:5", "94:6"]}
+{"id": "q-text-ar-02", "category": "quran_text", "lang": "ar", "query": "لا إكراه في الدين قد تبين الرشد من الغي", "relevant": ["2:256"]}
+{"id": "q-text-ar-03", "category": "quran_text", "lang": "ar", "query": "فبأي آلاء ربكما تكذبان", "relevant": ["55:13", "55:16", "55:18", "55:21", "55:23", "55:25", "55:28", "55:30", "55:32", "55:34", "55:36", "55:38", "55:40", "55:42", "55:45", "55:47", "55:49", "55:51", "55:53", "55:55", "55:57", "55:59", "55:61", "55:63", "55:65", "55:67", "55:69", "55:71", "55:73", "55:75", "55:77"]}
+{"id": "q-text-ar-04", "category": "quran_text", "lang": "ar", "query": "ألا بذكر الله تطمئن القلوب", "relevant": ["13:28"]}
+{"id": "q-text-ar-05", "category": "quran_text", "lang": "ar", "query": "وما خلقت الجن والإنس إلا ليعبدون", "relevant": ["51:56"]}
+{"id": "q-text-ar-06", "category": "quran_text", "lang": "ar", "query": "قل هو الله أحد", "relevant": ["112:1"]}
+{"id": "q-text-ar-07", "category": "quran_text", "lang": "ar", "query": "الله نور السماوات والأرض", "relevant": ["24:35"]}
+{"id": "q-text-ar-08", "category": "quran_text", "lang": "ar", "query": "كل نفس ذائقة الموت", "relevant": ["3:185", "21:35", "29:57"]}
+{"id": "q-text-en-01", "category": "quran_text", "lang": "en", "query": "Every soul will taste death", "relevant": ["3:185", "21:35", "29:57"]}
+{"id": "q-text-en-02", "category": "quran_text", "lang": "en", "query": "Indeed, mankind is in loss", "relevant": ["103:2"]}
+{"id": "q-text-en-03", "category": "quran_text", "lang": "en", "query": "Recite in the name of your Lord who created", "relevant": ["96:1"]}
+{"id": "q-ref-01", "category": "quran_ref", "lang": "en", "query": "2:255", "relevant": ["2:255"]}
+{"id": "q-ref-02", "category": "quran_ref", "lang": "ar", "query": "سورة البقرة آية 286", "relevant": ["2:286"]}
+{"id": "q-ref-03", "category": "quran_ref", "lang": "en", "query": "Al-Baqarah 183", "relevant": ["2:183"]}
+{"id": "q-ref-04", "category": "quran_ref", "lang": "ar", "query": "تفسير الآية 13 من سورة الحجرات", "relevant": ["49:13"]}
+{"id": "q-ref-05", "category": "quran_ref", "lang": "en", "query": "Surah An-Nur verse 35", "relevant": ["24:35"]}
+{"id": "h-01", "category": "hadith_topic", "lang": "en", "query": "Hadith about actions being judged by intentions", "relevant": ["bukhari_1", "bukhari_54", "bukhari_2429", "bukhari_3735", "bukhari_6445", "bukhari_6696", "muslim_4791", "abudawud_2202", "ibnmajah_3965", "tirmidhi_1696", "nasai_75", "nasai_3443", "nasai_3803", "ahmad_162", "riyad_1218", "forty_nawawi_1"]}
+{"id": "h-02", "category": "hadith_text", "lang": "ar", "query": "حديث إنما الأعمال بالنيات", "relevant": ["bukhari_1", "bukhari_54", "bukhari_2429", "bukhari_3735", "bukhari_6445", "bukhari_6696", "muslim_4791", "abudawud_2202", "ibnmajah_3965", "tirmidhi_1696", "nasai_75", "nasai_3443", "nasai_3803", "ahmad_162", "riyad_1218", "forty_nawawi_1"]}
+{"id": "h-03", "category": "hadith_topic", "lang": "en", "query": "What are the five pillars of Islam according to the hadith?", "relevant": ["bukhari_8", "bukhari_4312", "muslim_19", "muslim_20", "muslim_21", "tirmidhi_2679", "nasai_5010", "riyad_396", "riyad_527", "riyad_592", "forty_nawawi_3"]}
+{"id": "h-04", "category": "hadith_text", "lang": "en", "query": "None of you truly believes until he loves for his brother what he loves for himself", "relevant": ["bukhari_13", "muslim_77", "tirmidhi_2585", "ibnmajah_4145", "nasai_5025", "nasai_5026", "nasai_5048", "darimi_2008", "riyad_1400", "riyad_1453", "forty_nawawi_13"]}
+{"id": "h-05", "category": "hadith_topic", "lang": "ar", "query": "حديث عن حب الخير للأخ المسلم كما يحبه لنفسه", "relevant": ["bukhari_13", "muslim_77", "tirmidhi_2585", "ibnmajah_4145", "nasai_5025", "nasai_5026", "nasai_5048", "darimi_2008", "riyad_1400", "riyad_1453", "forty_nawawi_13"]}
+{"id": "h-06", "category": "hadith_topic", "lang": "en", "query": "The Prophet's advice to a man who asked for counsel: do not get angry", "relevant": ["bukhari_5884", "tirmidhi_2088", "riyad_1265", "riyad_1855", "forty_nawawi_16"]}
+{"id": "h-07", "category": "hadith_text", "lang": "ar", "query": "حديث ليس الشديد بالصرعة", "relevant": ["bukhari_5882", "muslim_6480", "muslim_6481", "malik_1769", "riyad_1262", "riyad_1863", "bulugh_1679", "adab_1320"]}
+{"id": "h-08", "category": "hadith_text", "lang": "en", "query": "Hadith: the religion is sincere advice", "relevant": ["muslim_103", "abudawud_4946", "tirmidhi_1994", "nasai_4207", "nasai_4208", "nasai_4209", "nasai_4210", "darimi_2021", "riyad_1398", "bulugh_1730", "forty_nawawi_7"]}
+{"id": "h-09", "category": "hadith_topic", "lang": "ar", "query": "حديث التبسم في وجه أخيك صدقة", "relevant": ["tirmidhi_2024", "adab_893"]}
+{"id": "h-10", "category": "hadith_topic", "lang": "en", "query": "Is seeking knowledge obligatory on every Muslim?", "relevant": ["ibnmajah_4303"]}
+{"id": "h-11", "category": "hadith_topic", "lang": "en", "query": "Who deserves my good companionship the most? Your mother", "relevant": ["muslim_6341", "muslim_6342", "ibnmajah_2442", "abudawud_5141", "riyad_1533"]}
+{"id": "h-12", "category": "hadith_topic", "lang": "ar", "query": "فضل صيام رمضان إيمانا واحتسابا", "relevant": ["bukhari_38", "bukhari_1833", "bukhari_1941", "muslim_1673", "abudawud_1373", "ibnmajah_1375", "nasai_2208", "nasai_2209", "nasai_2210", "riyad_540"]}
+{"id": "h-13", "category": "hadith_text", "lang": "en", "query": "The lawful is clear and the unlawful is clear, and between them are doubtful matters", "relevant": ["bukhari_52", "bukhari_1977", "muslim_3965", "muslim_3968", "tirmidhi_1209", "ibnmajah_3721", "nasai_5406", "nasai_5407", "darimi_1802", "darimi_2926", "forty_nawawi_6"]}
+{"id": "h-14", "category": "hadith_text", "lang": "ar", "query": "الطهور شطر الإيمان", "relevant": ["muslim_440", "darimi_4", "riyad_352", "riyad_734", "riyad_1242", "forty_nawawi_23"]}
+{"id": "h-15", "category": "hadith_topic", "lang": "en", "query": "Reward for relieving a believer of hardship", "relevant": ["muslim_6687", "tirmidhi_1442", "ibnmajah_4304", "riyad_1462", "bulugh_1663", "forty_nawawi_36"]}
+{"id": "h-16", "category": "hadith_text", "lang": "ar", "query": "حديث دع ما يريبك إلى ما لا يريبك", "relevant": ["tirmidhi_2588", "nasai_5721", "darimi_2922", "riyad_1272", "riyad_1809", "forty_nawawi_11"]}
+{"id": "h-17", "category": "hadith_topic", "lang": "en", "query": "Leaving what does not concern you is part of a person's good Islam", "relevant": ["tirmidhi_2386", "tirmidhi_2387", "ibnmajah_3713", "malik_1760", "riyad_1284", "bulugh_1673", "forty_nawawi_12"]}
+{"id": "h-18", "category": "hadith_text", "lang": "ar", "query": "الراحمون يرحمهم الرحمن", "relevant": ["tirmidhi_1992", "abudawud_4943"]}
+{"id": "h-19", "category": "hadith_topic", "lang": "en", "query": "Seven people who will be shaded by Allah on the Day of Judgment", "relevant": ["bukhari_642", "bukhari_1375", "bukhari_6240", "bukhari_6560", "muslim_2264", "tirmidhi_2460", "nasai_5389", "malik_1870", "riyad_1593", "riyad_1666", "riyad_1875", "bulugh_739"]}
+{"id": "h-20", "category": "hadith_text", "lang": "ar", "query": "كلمتان خفيفتان على اللسان ثقيلتان في الميزان", "relevant": ["bukhari_6167", "bukhari_6438", "muslim_6680", "tirmidhi_3551", "ibnmajah_3542", "riyad_729", "bukhari_7277"]}
+{"id": "h-21", "category": "hadith_text", "lang": "en", "query": "The best of you are those who learn the Quran and teach it", "relevant": ["bukhari_4820", "abudawud_1453", "tirmidhi_2990", "tirmidhi_2992", "darimi_2593", "ahmad_390", "ahmad_469", "riyad_314"]}
+{"id": "h-22", "category": "hadith_topic", "lang": "ar", "query": "فضل السواك", "relevant": ["bukhari_866", "bukhari_6966", "muslim_495", "abudawud_47", "tirmidhi_22", "tirmidhi_23", "ibnmajah_21", "nasai_7", "malik_145", "darimi_809", "ahmad_572", "ahmad_920", "riyad_517", "bulugh_37", "nasai_5", "nasai_6", "ibnmajah_1411"]}
+{"id": "h-23", "category": "hadith_text", "lang": "en", "query": "Pray as you have seen me praying", "relevant": ["bukhari_616", "bukhari_5780", "bukhari_6972", "darimi_579", "riyad_33", "bulugh_236", "bulugh_397", "adab_213"]}
+{"id": "h-24", "category": "hadith_text", "lang": "ar", "query": "حديث قدسي أنا عند ظن عبدي بي", "relevant": ["bukhari_7123", "bukhari_7219", "muslim_6640", "muslim_6664", "muslim_6782", "tirmidhi_3687", "ibnmajah_3558", "darimi_2000", "riyad_756", "riyad_1657", "qudsi_15"]}
+{"id": "h-25", "category": "hadith_topic", "lang": "en", "query": "Who is a true Muslim? The one from whose tongue and hand others are safe", "relevant": ["bukhari_10", "bukhari_6245", "muslim_69", "abudawud_2482", "tirmidhi_2697", "nasai_5005", "darimi_1984", "riyad_886", "riyad_1428", "adab_1147"]}
+{"id": "h-26", "category": "hadith_topic", "lang": "ar", "query": "ما الذي ينفع الميت بعد موته من عمله", "relevant": ["muslim_4094", "abudawud_2881", "darimi_3317", "bulugh_1067", "adab_38"]}
+{"id": "h-27", "category": "hadith_text", "lang": "en", "query": "Be in this world as if you were a stranger or a traveler", "relevant": ["bukhari_6177", "tirmidhi_2402", "ibnmajah_3852", "riyad_1687", "riyad_1790", "bulugh_1668", "forty_nawawi_40"]}
+{"id": "h-28", "category": "hadith_text", "lang": "ar", "query": "لا ضرر ولا ضرار", "relevant": ["ibnmajah_2074", "ibnmajah_2075", "malik_1520", "bulugh_1060", "forty_nawawi_32"]}
+{"id": "h-29", "category": "hadith_text", "lang": "en", "query": "Allah is beautiful and loves beauty", "relevant": ["muslim_171", "riyad_896", "riyad_1828", "tirmidhi_2067"]}
+{"id": "h-30", "category": "hadith_text", "lang": "ar", "query": "إن الله لا ينظر إلى صوركم وأموالكم ولكن ينظر إلى قلوبكم", "relevant": ["muslim_6382", "ibnmajah_3881"]}
+{"id": "h-31", "category": "hadith_topic", "lang": "en", "query": "Reward of praying in congregation compared to praying alone", "relevant": ["bukhari_630", "bukhari_631", "bukhari_634", "muslim_1372", "tirmidhi_215", "ibnmajah_523", "nasai_839", "malik_289", "riyad_385", "bulugh_477", "bulugh_479"]}
+{"id": "h-32", "category": "hadith_topic", "lang": "ar", "query": "فضل بناء المساجد", "relevant": ["bukhari_443", "muslim_1091", "muslim_1092", "muslim_7285", "muslim_7286", "ibnmajah_469", "ibnmajah_472", "nasai_690", "ahmad_354", "ahmad_408", "ahmad_475"]}
+{"id": "h-33", "category": "hadith_topic", "lang": "en", "query": "The five daily prayers are like bathing in a river five times a day", "relevant": ["bukhari_518", "muslim_1417", "tirmidhi_2950", "nasai_464", "darimi_512", "riyad_363"]}
+{"id": "h-34", "category": "hadith_topic", "lang": "ar", "query": "حديث جبريل عن الإسلام والإيمان والإحسان", "relevant": ["bukhari_50", "muslim_5", "tirmidhi_2680", "ibnmajah_4142", "ibnmajah_4143", "nasai_5000"]}
+{"id": "h-35", "category": "hadith_text", "lang": "en", "query": "O My servants, I have forbidden oppression for Myself", "relevant": ["muslim_6410", "bulugh_1692", "forty_nawawi_24", "qudsi_17"]}
+{"id": "h-auth-01", "category": "hadith_auth", "lang": "ar", "query": "هل حديث طلب العلم فريضة على كل مسلم صحيح؟", "relevant": ["ibnmajah_4303"]}
+{"id": "h-auth-02", "category": "hadith_auth", "lang": "en", "query": "Is the hadith 'actions are by intentions' authentic?", "relevant": ["bukhari_1", "bukhari_54", "bukhari_2429", "bukhari_3735", "bukhari_6445", "bukhari_6696", "muslim_4791", "abudawud_2202", "ibnmajah_3965", "tirmidhi_1696", "nasai_75", "nasai_3443", "nasai_3803", "ahmad_162", "riyad_1218", "forty_nawawi_1"]}
+{"id": "h-auth-03", "category": "hadith_auth", "lang": "ar", "query": "ما صحة حديث الطهور شطر الإيمان", "relevant": ["muslim_440", "darimi_4", "riyad_352", "riyad_734", "riyad_1242", "forty_nawawi_23"]}
+{"id": "h-auth-04", "category": "hadith_auth", "lang": "en", "query": "Is the hadith 'the strong man is not the one who wrestles' authentic?", "relevant": ["bukhari_5882", "muslim_6480", "muslim_6481", "malik_1769", "riyad_1262", "riyad_1863", "bulugh_1679", "adab_1320"]}
+{"id": "h-auth-05", "category": "hadith_auth", "lang": "ar", "query": "هل صحيح أن النبي قال الدين النصيحة؟", "relevant": ["muslim_103", "abudawud_4946", "tirmidhi_1994", "nasai_4207", "nasai_4208", "nasai_4209", "nasai_4210", "darimi_2021", "riyad_1398", "bulugh_1730", "forty_nawawi_7"]}
+{"id": "h-auth-06", "category": "hadith_auth", "lang": "ar", "query": "ما درجة حديث لا ضرر ولا ضرار", "relevant": ["ibnmajah_2074", "ibnmajah_2075", "malik_1520", "bulugh_1060", "forty_nawawi_32"]}
+{"id": "h-auth-07", "category": "hadith_auth", "lang": "en", "query": "Is it true that the Prophet said religion is sincere advice?", "relevant": ["muslim_103", "abudawud_4946", "tirmidhi_1994", "nasai_4207", "nasai_4208", "nasai_4209", "nasai_4210", "darimi_2021", "riyad_1398", "bulugh_1730", "forty_nawawi_7"]}
+{"id": "h-auth-08", "category": "hadith_auth", "lang": "ar", "query": "هل حديث كلمتان خفيفتان على اللسان صحيح؟", "relevant": ["bukhari_6167", "bukhari_6438", "muslim_6680", "tirmidhi_3551", "ibnmajah_3542", "riyad_729"]}
+{"id": "f-01", "category": "fatwa", "lang": "ar", "query": "هل الحقنة في العضل تبطل الصيام؟", "relevant": ["fatwa_islamweb_857", "fatwa_islamweb_3337", "fatwa_islamweb_4008"]}
+{"id": "f-02", "category": "fatwa", "lang": "ar", "query": "حكم الطلاق في حالة الغضب الشديد", "relevant": ["fatwa_binbaz_4213", "fatwa_islamweb_1496", "fatwa_islamweb_3073", "fatwa_islamweb_2182"]}
+{"id": "f-03", "category": "fatwa", "lang": "ar", "query": "كم عدد الرضعات التي تحرم النكاح؟", "relevant": ["fatwa_binbaz_4215"]}
+{"id": "f-04", "category": "fatwa", "lang": "ar", "query": "ما نصاب زكاة عروض التجارة وكيف تخرج؟", "relevant": ["fatwa_islamweb_930", "fatwa_islamweb_1008", "fatwa_islamweb_2205", "fatwa_islamweb_3078", "fatwa_islamweb_5039", "fatwa_islamweb_7609"]}
+{"id": "f-05", "category": "fatwa", "lang": "ar", "query": "متى يجوز للمسافر الجمع والقصر في الصلاة؟", "relevant": ["fatwa_islamweb_507", "fatwa_islamweb_361", "fatwa_islamweb_178", "fatwa_islamweb_181", "fatwa_islamweb_1359", "fatwa_islamweb_6215", "fatwa_islamweb_3323", "fatwa_islamweb_6851", "fatwa_islamweb_5891", "fatwa_islamweb_4824", "fatwa_islamweb_6227"]}
+{"id": "f-06", "category": "fatwa", "lang": "ar", "query": "ما هي شروط صحة عقد النكاح؟", "relevant": ["fatwa_islamweb_1766", "fatwa_islamweb_741", "fatwa_islamweb_964", "fatwa_islamweb_591", "fatwa_islamweb_5483", "fatwa_islamweb_7704"]}
+{"id": "f-07", "category": "fatwa", "lang": "ar", "query": "حكم إسبال الثوب لغير الخيلاء", "relevant": ["fatwa_binbaz_4211", "fatwa_islamweb_5943", "fatwa_islamweb_3900"]}
+{"id": "f-08", "category": "fatwa", "lang": "ar", "query": "هل كتابة الوصية واجبة وما صيغتها؟", "relevant": ["fatwa_binbaz_4210"]}
+{"id": "f-09", "category": "fatwa", "lang": "ar", "query": "هل يجوز السلام على المرأة الأجنبية؟", "relevant": ["fatwa_binbaz_4219", "fatwa_islamweb_6158"]}
+{"id": "f-10", "category": "fatwa", "lang": "ar", "query": "هل حديث أبغض الحلال إلى الله الطلاق صحيح؟", "relevant": ["fatwa_binbaz_4214", "abudawud_2179", "ibnmajah_1752", "bulugh_1241"]}
+{"id": "f-11", "category": "fatwa", "lang": "en", "query": "Does an intramuscular injection break the fast?", "relevant": ["fatwa_islamweb_857", "fatwa_islamweb_3337", "fatwa_islamweb_4008"]}
+{"id": "f-12", "category": "fatwa", "lang": "en", "query": "Rules for combining and shortening prayers while traveling", "relevant": ["fatwa_islamweb_507", "fatwa_islamweb_361", "fatwa_islamweb_178", "fatwa_islamweb_181", "fatwa_islamweb_1359", "fatwa_islamweb_6215", "fatwa_islamweb_3323", "fatwa_islamweb_6851", "fatwa_islamweb_5891", "fatwa_islamweb_4824", "fatwa_islamweb_6227"]}
+{"id": "f-13", "category": "fatwa", "lang": "ar", "query": "هل استعمال العطر يفطر الصائم؟", "relevant": ["fatwa_islamweb_892", "fatwa_islamweb_6247"]}
+{"id": "f-14", "category": "fatwa", "lang": "en", "query": "Is it allowed to combine two prayers without an excuse?", "relevant": ["fatwa_islamweb_134", "fatwa_islamweb_5589", "fatwa_islamweb_4724"]}
+{"id": "f-15", "category": "fatwa", "lang": "ar", "query": "ما المسافة التي يجوز للمسافر أن يفطر فيها في رمضان؟", "relevant": ["fatwa_islamweb_233", "fatwa_islamweb_178", "fatwa_islamweb_1887", "fatwa_islamweb_4824"]}
+{"id": "f-16", "category": "fatwa", "lang": "en", "query": "Does pulling a tooth break the fast?", "relevant": ["fatwa_islamweb_1100", "fatwa_islamweb_6477", "fatwa_islamweb_6380"]}
+{"id": "f-17", "category": "fatwa", "lang": "ar", "query": "حكم إيداع المال في البنوك الربوية", "relevant": ["fatwa_islamweb_331", "fatwa_islamweb_518", "fatwa_islamweb_623", "fatwa_islamweb_5773", "fatwa_islamweb_1954"]}
+{"id": "f-18", "category": "fatwa", "lang": "en", "query": "A man with liver disease must keep taking medication. May he skip fasting in Ramadan?", "relevant": ["fatwa_islamweb_2277", "fatwa_islamweb_5978", "fatwa_islamweb_7751"]}
+{"id": "neg-01", "category": "negative", "lang": "en", "query": "What is the capital of France?", "relevant": []}
+{"id": "neg-02", "category": "negative", "lang": "en", "query": "How do I fix a Python ImportError?", "relevant": []}
+{"id": "neg-03", "category": "negative", "lang": "en", "query": "Give me a recipe for chocolate cake", "relevant": []}
+{"id": "neg-04", "category": "negative", "lang": "en", "query": "Who won the 2022 FIFA World Cup?", "relevant": []}
+{"id": "neg-05", "category": "negative", "lang": "ar", "query": "ما هي عاصمة اليابان؟", "relevant": []}
+{"id": "neg-06", "category": "negative", "lang": "ar", "query": "اشرح نظرية النسبية لأينشتاين", "relevant": []}
+{"id": "neg-07", "category": "negative", "lang": "en", "query": "What is the current price of Bitcoin?", "relevant": []}
+{"id": "neg-08", "category": "negative", "lang": "ar", "query": "كيف أغير زيت السيارة؟", "relevant": []}
+{"id": "neg-09", "category": "negative", "lang": "en", "query": "Write a poem about the ocean", "relevant": []}
+{"id": "neg-10", "category": "negative", "lang": "ar", "query": "ما هو أفضل هاتف ذكي هذا العام؟", "relevant": []}
+{"id": "neg-11", "category": "negative", "lang": "en", "query": "How do I reset my router password?", "relevant": []}
+{"id": "neg-12", "category": "negative", "lang": "en", "query": "Explain how photosynthesis works", "relevant": []}
+{"id": "neg-13", "category": "negative", "lang": "en", "query": "What's the weather like in London tomorrow?", "relevant": []}
+{"id": "neg-14", "category": "negative", "lang": "en", "query": "Recommend a good laptop for programming", "relevant": []}
+{"id": "neg-15", "category": "negative", "lang": "en", "query": "Translate 'good morning' into Spanish", "relevant": []}
+{"id": "neg-16", "category": "negative", "lang": "en", "query": "How many players are on a basketball team?", "relevant": []}
+{"id": "neg-17", "category": "negative", "lang": "en", "query": "Summarize the plot of the movie Inception", "relevant": []}
+{"id": "neg-18", "category": "negative", "lang": "en", "query": "What is the boiling point of water in Fahrenheit?", "relevant": []}
+{"id": "neg-19", "category": "negative", "lang": "ar", "query": "كيف أتعلم البرمجة بلغة بايثون؟", "relevant": []}
+{"id": "neg-20", "category": "negative", "lang": "ar", "query": "ما هي أعراض الإنفلونزا؟", "relevant": []}
+{"id": "neg-21", "category": "negative", "lang": "ar", "query": "من فاز بكأس العالم لكرة القدم 2018؟", "relevant": []}
+{"id": "neg-22", "category": "negative", "lang": "ar", "query": "اقترح لي وصفة لعمل الكبسة", "relevant": []}
+{"id": "neg-23", "category": "negative", "lang": "ar", "query": "كم سعر الذهب اليوم؟", "relevant": []}
+{"id": "neg-24", "category": "negative", "lang": "ar", "query": "ما هي أفضل طريقة لتعلم اللغة الإنجليزية؟", "relevant": []}
+{"id": "neg-25", "category": "negative", "lang": "ar", "query": "اكتب لي رسالة بريد إلكتروني لمديري أطلب إجازة", "relevant": []}

eval/run_answers.py

@@ -0,0 +1,306 @@
+#!/usr/bin/env python3
+"""Answer-level evaluation: run the full RAG pipeline (with the LLM) over the golden set.
+
+Where eval/run_eval.py measures retrieval, this checks what users actually read:
+
+  refusals   — out-of-scope questions get the "not found" answer; real ones don't
+  language   — answer language matches the question; no CJK leakage
+  citations  — every cited verse / hadith / fatwa is among the retrieved sources
+               (precision), and at least one cited source is a golden-relevant one
+  quotes     — scripture quoted by the LLM, before _verify_citations: how many were
+               verbatim from the retrieved context, how many the verifier removed or
+               auto-corrected, and how many auto-corrections swapped in a different
+               real verse than the one quoted
+
+Usage:
+    python -m eval.run_answers --limit 5                       # smoke test
+    python -m eval.run_answers --only fatwa,negative
+    python -m eval.run_answers --out eval/results/answers.json
+Per-case answers are saved in the report for manual review.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import re
+import sys
+import time
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional, Set
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+from app.config import cfg  # noqa: E402
+from eval.run_eval import DEFAULT_GOLDEN, RESULTS_DIR, _git_commit, init_state, load_golden, result_ids, validate_labels  # noqa: E402
+
+_QURAN_URL = re.compile(r"aya-(\d+)-sora-(\d+)|sura(\d+)-aya(\d+)|quran\.com/(\d+)/(\d+)")
+_URL = re.compile(r"https?://[^\s\)\]>\"']+")
+_QUOTE = re.compile(r"﴿([^﴾]+)﴾|❝([^❞]+)❞|«([^»]+)»")
+_REMOVED_MARK = "تم حذف نص غير موثق"
+_CJK = re.compile(r"[぀-ヿ㐀-䶿一-鿿가-힯]")
+_REFUSAL_HINTS = (
+    "does not contain sufficient information", "not found in the", "no relevant", "cannot find",
+    "لم أجد", "لا تتوفر", "لا يوجد في", "لم يرد في",
+)
+# Hadith references as written in answers: "<collection>[, ][No.|#|رقم] <number>".
+# Arabic names need a qualifier ("صحيح مسلم", "رواه مسلم") since مسلم / أحمد are also common words.
+_HADITH_COLLECTIONS = {
+    "sahih al-bukhari": "bukhari", "bukhari": "bukhari", "البخاري": "bukhari",
+    "sahih muslim": "muslim", "muslim": "muslim", "صحيح مسلم": "muslim", "رواه مسلم": "muslim", "أخرجه مسلم": "muslim",
+    "abu dawood": "abudawud", "abu dawud": "abudawud", "أبي داود": "abudawud", "أبو داود": "abudawud",
+    "tirmidhi": "tirmidhi", "الترمذي": "tirmidhi", "nasai": "nasai", "nasa'i": "nasai", "النسائي": "nasai",
+    "ibn majah": "ibnmajah", "ابن ماجه": "ibnmajah",
+    "musnad ahmad": "ahmad", "مسند أحمد": "ahmad", "رواه أحمد": "ahmad", "أخرجه أحمد": "ahmad",
+    "muwatta malik": "malik", "muwatta": "malik", "الموطأ": "malik", "موطأ مالك": "malik",
+    "darimi": "darimi", "الدارمي": "darimi",
+    "riyad al-salihin": "riyad", "رياض الصالحين": "riyad", "bulugh al-maram": "bulugh", "بلوغ المرام": "bulugh",
+    "adab al-mufrad": "adab", "الأدب المفرد": "adab",
+    "shama'il al-muhammadiyyah": "shamail", "الشمائل المحمدية": "shamail",
+    "nawawi's 40 hadith": "forty_nawawi", "الأربعون النووية": "forty_nawawi",
+    "hadith qudsi": "qudsi", "الأحاديث القدسية": "qudsi",
+}
+_HADITH_REF = re.compile(
+    r"(" + "|".join(re.escape(k) for k in sorted(_HADITH_COLLECTIONS, key=len, reverse=True)) + r")"
+    r"(?:[\s,،:#.()\-–*]|no\b|number|hadith|في صحيحه|(?:ال)?رقم|برقم|(?:ال)?حديث"
+    r"|(?:كتاب|book of) [^\d\n،,]{1,30}[،,])*?(\d{1,5})\b",
+    re.I,
+)
+
+
+def cited_ids(answer: str, url_to_id: Dict[str, str]) -> Set[str]:
+    ids: Set[str] = set()
+    for m in _QURAN_URL.finditer(answer):
+        if m.group(1):
+            verse, surah = m.group(1), m.group(2)
+        elif m.group(3):
+            surah, verse = m.group(3), m.group(4)
+        else:
+            surah, verse = m.group(5), m.group(6)
+        ids.add(f"{int(surah)}:{int(verse)}")
+    for url in _URL.findall(answer):
+        key = url.rstrip("/.,")
+        if key in url_to_id:
+            ids.add(url_to_id[key])
+    for m in _HADITH_REF.finditer(answer):
+        ids.add(f"{_HADITH_COLLECTIONS[m.group(1).lower()]}_{int(m.group(2))}")
+    return ids
+
+
+def lang_ok(answer: str, lang: str) -> bool:
+    text = _URL.sub(" ", _QUOTE.sub(" ", answer))
+    ar = sum(1 for c in text if "؀" <= c <= "ۿ")
+    la = sum(1 for c in text if c.isascii() and c.isalpha())
+    if ar + la == 0:
+        return True
+    return ar / (ar + la) >= 0.5 if lang == "ar" else la / (ar + la) >= 0.5
+
+
+def is_refusal(answer: str, lang: str, not_found: Dict[str, str]) -> bool:
+    if not_found["arabic" if lang == "ar" else "english"].split("\n")[0] in answer:
+        return True
+    head = answer[:300].lower()
+    return any(h in head for h in _REFUSAL_HINTS) and len(answer) < 600
+
+
+class QuoteRecorder:
+    """Wraps _verify_citations to keep the LLM's raw output for each call."""
+
+    def __init__(self, fn):
+        self.fn, self.calls = fn, []
+
+    def __call__(self, answer, results):
+        out = self.fn(answer, results)
+        self.calls.append((answer, out))
+        return out
+
+
+def quote_stats(raw: str, verified: str, retrieved_norm: List[str], verse_by_skel: Dict[str, str],
+                normalize, corpus_norm: List[str]) -> dict:
+    from app.state import _skeleton
+    stats = {"quotes": 0, "verbatim": 0, "variant": 0, "fabricated": 0,
+             "removed": 0, "corrected": 0, "wrong_verse": 0}
+    raw_quotes = [next(g for g in m.groups() if g) for m in _QUOTE.finditer(raw)]
+    out_quotes = [next(g for g in m.groups() if g) for m in _QUOTE.finditer(verified)]
+    for i, q in enumerate(raw_quotes):
+        qn = normalize(q).strip()
+        if sum(1 for c in qn if "؀" <= c <= "ۿ") < 12:
+            continue  # the verifier ignores short / non-Arabic quotes too
+        stats["quotes"] += 1
+        q_skel = _skeleton(q)
+        # Verses the quote is a copy of. A short verse *inside* the quote (68:7 is the tail
+        # of 16:125) marks it as real scripture but doesn't identify which verse it is.
+        real = {vid for vs, vid in verse_by_skel.items() if q_skel in vs}
+        contained = {vid for vs, vid in verse_by_skel.items() if len(vs) >= 12 and vs in q_skel}
+        if any(qn in src for src in retrieved_norm):
+            stats["verbatim"] += 1
+        elif real or contained:
+            stats["variant"] += 1  # same letters as a real verse, but altered spelling/marks/foreign chars
+        elif not any(normalize(q, aggressive=True).lower().strip() in doc for doc in corpus_norm):
+            stats["fabricated"] += 1  # not in any indexed verse, hadith or fatwa
+        out = out_quotes[i] if i < len(out_quotes) else ""
+        if _REMOVED_MARK in out:
+            stats["removed"] += 1
+        elif normalize(out).strip() != qn:
+            stats["corrected"] += 1
+            substituted = {verse_by_skel.get(_skeleton(out))} - {None}
+            if real and substituted and not (real & substituted):
+                stats["wrong_verse"] += 1
+    return stats
+
+
+async def main_async(args) -> None:
+    from app import state as state_mod
+    from app.arabic_nlp import normalize_arabic
+    from app.prompts import not_found_answer
+    from app.search import _normalized_texts
+    from app.state import run_rag_pipeline, state
+
+    only = set(args.only.split(",")) if args.only else None
+    cases = load_golden(Path(args.golden), only)[: args.limit or None]
+    init_state(use_llm=True)
+    validate_labels(cases, state.dataset)
+
+    ref_to_id = {d["reference"]: d["id"] for d in state.dataset if d.get("type") == "hadith" and d.get("reference")}
+    url_to_id = {d["source_url"].rstrip("/"): d["id"] for d in state.dataset if d.get("source_url")}
+    from app.state import _skeleton
+    verse_by_skel = {_skeleton(d["arabic"]): d["id"] for d in state.dataset
+                     if d.get("type") == "quran" and d.get("arabic")}
+    not_found = {"arabic": not_found_answer("arabic"), "english": not_found_answer("english")}
+    corpus_norm = [ar for ar, _ in _normalized_texts(state.dataset)]
+
+    recorder = QuoteRecorder(state_mod._verify_citations)
+    state_mod._verify_citations = recorder
+
+    rows, t0 = [], time.perf_counter()
+    for i, case in enumerate(cases, 1):
+        recorder.calls.clear()
+        t = time.perf_counter()
+        out = await run_rag_pipeline(case["query"], top_k=cfg.TOP_K_RETURN)
+        answer = out["answer"]
+        sources = out.get("sources") or []
+        retrieved: Set[str] = set()
+        retrieved_norm: List[str] = []
+        for s in sources:
+            retrieved |= result_ids(s, ref_to_id)
+            retrieved |= {f"{s.get('surah_number')}:{c.get('ayah')}" for c in s.get("context_ayat") or []}
+            for field in ("arabic", "answer_ar"):
+                if s.get(field):
+                    retrieved_norm.append(normalize_arabic(s[field]))
+
+        relevant = set(case["relevant"])
+        refused = is_refusal(answer, case.get("lang", "en"), not_found)
+        cites = cited_ids(answer, url_to_id)
+        raw = recorder.calls[0][0] if recorder.calls else answer
+        row = {
+            "id": case["id"], "category": case["category"], "lang": case.get("lang"), "query": case["query"],
+            "intent": out.get("intent"), "refused": refused, "lang_ok": lang_ok(answer, case.get("lang", "en")),
+            "cjk": bool(_CJK.search(answer)), "cited": sorted(cites),
+            "ungrounded_citations": sorted(cites - retrieved),
+            "cites_relevant": bool(cites & relevant) if relevant else None,
+            "cites_required": set(case["required"]) <= cites if case.get("required") else None,
+            "retrieved_relevant": bool(retrieved & relevant) if relevant else None,
+            "quotes": quote_stats(raw, answer, retrieved_norm, verse_by_skel, normalize_arabic, corpus_norm),
+            "latency_s": round(time.perf_counter() - t, 1),
+            "answer": answer, "raw_answer": raw if raw != answer else None,
+        }
+        rows.append(row)
+        print(f"  {i}/{len(cases)} {case['id']:<14} {row['latency_s']:>5}s "
+              f"{'REFUSED' if refused else 'answered'} cites={len(cites)} ungrounded={len(row['ungrounded_citations'])}",
+              flush=True)
+
+    summary = summarize(rows)
+    print_summary(summary, time.perf_counter() - t0, len(rows))
+
+    report = {
+        "timestamp": datetime.now().isoformat(timespec="seconds"), "git_commit": _git_commit(),
+        "config": {k: getattr(cfg, k) for k in ("LLM_BACKEND", "TOP_K_RETURN", "USE_RERANKER", "FAST_QUERY_REWRITE",
+                                                  "TEMPERATURE", "MAX_TOKENS", "ENABLE_CITATION_VERIFICATION")},
+        "summary": summary, "cases": rows,
+    }
+    out_path = Path(args.out) if args.out else RESULTS_DIR / f"{datetime.now():%Y%m%d-%H%M%S}-answers.json"
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"\nReport written to {out_path}")
+
+
+def _rate(flags: List[bool]) -> Optional[float]:
+    return round(sum(flags) / len(flags), 3) if flags else None
+
+
+def summarize(rows: List[dict]) -> dict:
+    pos = [r for r in rows if r["cites_relevant"] is not None]
+    neg = [r for r in rows if r["cites_relevant"] is None]
+    answered = [r for r in pos if not r["refused"]]
+    q = defaultdict(int)
+    for r in rows:
+        for k, v in r["quotes"].items():
+            q[k] += v
+    by_cat = defaultdict(list)
+    for r in answered:
+        by_cat[r["category"]].append(r)
+    return {
+        "positives": len(pos), "negatives": len(neg),
+        "refusal": {
+            "negatives_refused": _rate([r["refused"] for r in neg]),
+            "positives_refused": _rate([r["refused"] for r in pos]),
+            "positives_refused_despite_retrieval": _rate([r["refused"] for r in pos if r["retrieved_relevant"]]),
+        },
+        "answered": {
+            "n": len(answered),
+            "cites_relevant": _rate([r["cites_relevant"] for r in answered]),
+            "cites_relevant_when_retrieved": _rate([r["cites_relevant"] for r in answered if r["retrieved_relevant"]]),
+            "cites_required": _rate([r["cites_required"] for r in answered if r.get("cites_required") is not None]),
+            "has_ungrounded_citation": _rate([bool(r["ungrounded_citations"]) for r in answered]),
+            "no_citation": _rate([not r["cited"] for r in answered]),
+            "lang_ok": _rate([r["lang_ok"] for r in answered]),
+            "cjk_leak": _rate([r["cjk"] for r in answered]),
+        },
+        "cites_relevant_by_category": {c: _rate([r["cites_relevant"] for r in rs]) for c, rs in sorted(by_cat.items())},
+        "quotes": dict(q),
+        "median_latency_s": sorted(r["latency_s"] for r in rows)[len(rows) // 2] if rows else None,
+    }
+
+
+def print_summary(s: dict, elapsed: float, n: int) -> None:
+    print(f"\n{n} cases in {elapsed:.0f}s (median {s['median_latency_s']}s/answer)\n")
+    r, a, q = s["refusal"], s["answered"], s["quotes"]
+    print("Refusals")
+    print(f"  off-topic refused:                      {r['negatives_refused']}  (n={s['negatives']})")
+    print(f"  real questions refused:                 {r['positives_refused']}  (n={s['positives']})")
+    print(f"  ...of which retrieval had the answer:   {r['positives_refused_despite_retrieval']}")
+    print(f"\nAnswered real questions (n={a['n']})")
+    print(f"  cites a golden-relevant source:         {a['cites_relevant']}")
+    print(f"  ...when retrieval had it:               {a['cites_relevant_when_retrieved']}")
+    print(f"  cites all must-include sources:         {a.get('cites_required')}")
+    print(f"  has a citation NOT in retrieved sources:{a['has_ungrounded_citation']}")
+    print(f"  no parseable citation:                  {a['no_citation']}")
+    print(f"  language matches question:              {a['lang_ok']}")
+    print(f"  CJK leakage:                            {a['cjk_leak']}")
+    for cat, v in s["cites_relevant_by_category"].items():
+        print(f"    {cat:<16} cites relevant: {v}")
+    print(f"\nQuoted scripture (≥12 Arabic letters), before verification: {q.get('quotes', 0)}")
+    for k in ("verbatim", "variant", "fabricated", "removed", "corrected", "wrong_verse"):
+        print(f"  {k:<12} {q.get(k, 0)}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Answer-level evaluation of QModel over the golden set.")
+    parser.add_argument("--golden", default=str(DEFAULT_GOLDEN))
+    parser.add_argument("--only", help="Comma-separated categories")
+    parser.add_argument("--limit", type=int, help="Evaluate only the first N cases")
+    parser.add_argument("--out", help="Report path (default eval/results/<timestamp>-answers.json)")
+    parser.add_argument("--verbose", action="store_true")
+    args = parser.parse_args()
+    logging.basicConfig(level=logging.INFO if args.verbose else logging.WARNING,
+                        format="%(levelname)s %(name)s: %(message)s")
+    asyncio.run(main_async(args))
+
+
+if __name__ == "__main__":
+    main()

eval/run_eval.py

@@ -0,0 +1,450 @@
+#!/usr/bin/env python3
+"""Retrieval evaluation harness for QModel.
+
+Runs every case in a golden set through the retrieval stack (no LLM generation)
+and reports ranking metrics plus confidence-gate behaviour.
+
+Golden set format (JSONL, one case per line):
+    {"id": "...", "category": "...", "lang": "ar|en", "query": "...", "relevant": ["2:255", "bukhari_1", ...]}
+`relevant` holds dataset `id`s; any one of them counts as a hit. An empty list
+marks a negative (out-of-scope) case that the confidence gate should reject.
+Optional `required` (a subset of `relevant`) names sources a complete answer must
+include — e.g. 5:90 for "is alcohol forbidden?", where 2:219 alone is a hit but
+misleading. These are scored separately as complete@k, so hit@k stays comparable.
+
+Modes:
+    pipeline  — the full pre-generation pipeline (_rag_preprocess): rewrite, hybrid
+                search, direct lookups, text search, merging, dedup. This is exactly
+                what the LLM receives as context.
+    hybrid    — hybrid_search() alone (dense + BM25 + RRF [+ reranker]), useful for
+                isolating fusion / embedding changes.
+
+Usage:
+    python -m eval.run_eval                         # pipeline mode, k = TOP_K_RETURN
+    python -m eval.run_eval --mode hybrid --k 20
+    python -m eval.run_eval --only fatwa,negative --show-failures
+    python -m eval.run_eval --out eval/results/baseline.json
+    python -m eval.run_eval --compare eval/results/baseline.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import re
+import subprocess
+import sys
+import time
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+from app.config import cfg  # noqa: E402
+
+DEFAULT_GOLDEN = ROOT / "eval" / "golden.jsonl"
+RESULTS_DIR = ROOT / "eval" / "results"
+_GRADE_SUFFIX = re.compile(r"\s*\[[^\]]*\]\s*$")
+
+
+# ═══════════════════════════════════════════════════════════════════════
+# LOADING
+# ═══════════════════════════════════════════════════════════════════════
+def load_golden(path: Path, only: Optional[set]) -> List[dict]:
+    cases = []
+    with open(path, encoding="utf-8") as f:
+        for line_no, line in enumerate(f, 1):
+            line = line.strip()
+            if not line or line.startswith("//"):
+                continue
+            case = json.loads(line)
+            for key in ("id", "category", "query", "relevant"):
+                if key not in case:
+                    raise ValueError(f"{path}:{line_no}: missing '{key}'")
+            extra = set(case.get("required", [])) - set(case["relevant"])
+            if extra:
+                raise ValueError(f"{path}:{line_no}: required ids not in relevant: {sorted(extra)}")
+            if only and case["category"] not in only:
+                continue
+            cases.append(case)
+    ids = [c["id"] for c in cases]
+    dupes = {i for i in ids if ids.count(i) > 1}
+    if dupes:
+        raise ValueError(f"Duplicate case ids: {sorted(dupes)}")
+    return cases
+
+
+def init_state(use_llm: bool) -> None:
+    """Load models/index into app.state the same way the server lifespan does."""
+    import faiss
+    from sentence_transformers import SentenceTransformer
+
+    from app.search import build_type_indexes
+    from app.state import _build_lookup_tables, infer_hadith_grade, state
+
+    t0 = time.perf_counter()
+    print(f"Loading embed model {cfg.EMBED_MODEL} …", flush=True)
+    state.embed_model = SentenceTransformer(cfg.EMBED_MODEL)
+    print(f"Loading FAISS index {cfg.FAISS_INDEX} …", flush=True)
+    state.faiss_index = faiss.read_index(str(ROOT / cfg.FAISS_INDEX))
+    print(f"Loading metadata {cfg.METADATA_FILE} …", flush=True)
+    with open(ROOT / cfg.METADATA_FILE, encoding="utf-8") as f:
+        state.dataset = [infer_hadith_grade(item) for item in json.load(f)]
+    print("Building lookup tables + BM25 …", flush=True)
+    _build_lookup_tables(state.dataset)
+    state.type_indexes = build_type_indexes(state.faiss_index, state.dataset)
+
+    state.reranker = None
+    if cfg.USE_RERANKER:
+        from sentence_transformers import CrossEncoder
+        print(f"Loading reranker {cfg.RERANK_MODEL} …", flush=True)
+        state.reranker = CrossEncoder(cfg.RERANK_MODEL, max_length=512)
+
+    state.llm = None
+    if use_llm:
+        from app.llm import get_llm_provider
+        print(f"Initializing LLM provider {cfg.LLM_BACKEND} …", flush=True)
+        state.llm = get_llm_provider()
+    elif not cfg.FAST_QUERY_REWRITE:
+        print("WARNING: FAST_QUERY_REWRITE is off but --llm not given; "
+              "queries will use the regex fallback rewrite.", flush=True)
+
+    state.ready = True
+    print(f"Ready in {time.perf_counter() - t0:.1f}s | dataset={len(state.dataset):,} "
+          f"| faiss={state.faiss_index.ntotal:,} | reranker={'on' if state.reranker else 'off'}\n",
+          flush=True)
+
+
+def validate_labels(cases: List[dict], dataset: list) -> None:
+    known = {item.get("id") for item in dataset}
+    missing = [(c["id"], r) for c in cases for r in c["relevant"] if r not in known]
+    if missing:
+        lines = "\n".join(f"  {cid}: {rid}" for cid, rid in missing[:20])
+        raise SystemExit(f"Golden set references {len(missing)} unknown dataset ids:\n{lines}")
+
+
+# ═══════════════════════════════════════════════════════════════════════
+# RETRIEVAL
+# ═══════════════════════════════════════════════════════════════════════
+async def retrieve(case: dict, mode: str, k: int) -> dict:
+    from app.search import hybrid_search, rewrite_query
+    from app.state import _rag_preprocess, state
+
+    if mode == "pipeline":
+        ctx = await _rag_preprocess(case["query"], k, None, None)
+        return {
+            "results": ctx["results"],
+            "intent": ctx["intent"],
+            "top_score": ctx["top_score"],
+            "gated": not ctx["confident"],
+            "confidence": ctx["confidence"],
+            "confidence_source": ctx["confidence_source"],
+        }
+
+    rewrite = await rewrite_query(case["query"], state.llm)
+    results = await hybrid_search(
+        case["query"], rewrite, state.embed_model, state.faiss_index, state.dataset,
+        top_n=k, bm25_index=state.bm25_index, reranker_model=state.reranker,
+    )
+    # The gate runs in _rag_preprocess, so hybrid mode doesn't evaluate it.
+    return {
+        "results": results,
+        "intent": rewrite.get("intent", "general"),
+        "top_score": results[0].get("_score", 0.0) if results else 0.0,
+        "gated": None,
+        "confidence": None,
+        "confidence_source": None,
+    }
+
+
+def result_ids(item: dict, ref_to_id: Dict[str, str]) -> set:
+    """All dataset ids a result stands for, including narrations merged into it by dedup."""
+    ids = {item.get("id")}
+    for alt in item.get("other_collections") or []:
+        ref = _GRADE_SUFFIX.sub("", alt).strip()
+        if ref in ref_to_id:
+            ids.add(ref_to_id[ref])
+    return ids
+
+
+def score_case(case: dict, out: dict, k: int, ref_to_id: Dict[str, str]) -> dict:
+    relevant = set(case["relevant"])
+    results = out["results"][:k]
+
+    retrieved, hit_ranks, found = [], [], set()
+    for rank, item in enumerate(results, 1):
+        ids = result_ids(item, ref_to_id)
+        retrieved.append(item.get("id"))
+        matched = ids & relevant
+        if matched:
+            hit_ranks.append(rank)
+            found |= matched
+
+    first = hit_ranks[0] if hit_ranks else None
+    row = {
+        "id": case["id"],
+        "category": case["category"],
+        "lang": case.get("lang", "?"),
+        "query": case["query"],
+        "intent": out["intent"],
+        "top_score": round(float(out["top_score"]), 4),
+        "gated": out["gated"],
+        "confidence": None if out["confidence"] is None else round(float(out["confidence"]), 4),
+        "confidence_source": out["confidence_source"],
+        "retrieved": retrieved,
+        "first_hit_rank": first,
+    }
+    if relevant:
+        row.update({
+            "hit@1": int(first is not None and first <= 1),
+            "hit@3": int(first is not None and first <= 3),
+            "hit@k": int(first is not None),
+            "mrr": 1.0 / first if first else 0.0,
+            # Deduped hadith results also match the narrations merged into them, so
+            # found can exceed k; cap at 1.
+            "recall@k": min(1.0, len(found) / min(len(relevant), k)),
+        })
+    if case.get("required"):
+        missing = sorted(set(case["required"]) - found)
+        row["complete@k"] = int(not missing)
+        row["missing_required"] = missing
+    return row
+
+
+# ═══════════════════════════════════════════════════════════════════════
+# REPORTING
+# ═══════════════════════════════════════════════════════════════════════
+METRICS = ("hit@1", "hit@3", "hit@k", "mrr", "recall@k")
+
+
+def aggregate(rows: List[dict]) -> dict:
+    pos = [r for r in rows if "hit@k" in r]
+    neg = [r for r in rows if "hit@k" not in r]
+
+    def mean(rs, key):
+        return round(sum(r[key] for r in rs) / len(rs), 4) if rs else None
+
+    def summarize(rs):
+        return {"n": len(rs), **{m: mean(rs, m) for m in METRICS}}
+
+    by_cat, by_lang = defaultdict(list), defaultdict(list)
+    for r in pos:
+        by_cat[r["category"]].append(r)
+        by_lang[r["lang"]].append(r)
+
+    req = [r for r in pos if "complete@k" in r]
+    return {
+        "overall": summarize(pos),
+        "required": {"n": len(req), "complete@k": mean(req, "complete@k")},
+        "by_category": {c: summarize(rs) for c, rs in sorted(by_cat.items())},
+        "by_lang": {lang: summarize(rs) for lang, rs in sorted(by_lang.items())},
+        "gate": _gate_summary(pos, neg),
+    }
+
+
+def _gate_summary(pos: List[dict], neg: List[dict]) -> Optional[dict]:
+    if any(r["gated"] is None for r in pos + neg):
+        return None
+    rate = lambda flags: round(sum(flags) / len(flags), 4) if flags else None  # noqa: E731
+    return {
+        # Positives refused even though retrieval found the answer.
+        "false_reject_rate": rate([int(r["gated"]) for r in pos if r["hit@k"]]),
+        # Positives refused regardless of retrieval outcome (what the user experiences).
+        "pos_reject_rate": rate([int(r["gated"]) for r in pos]),
+        # Out-of-scope queries the gate lets through to the LLM.
+        "negatives": len(neg),
+        "false_accept_rate": rate([int(not r["gated"]) for r in neg]),
+        "pos_confidence_median": _median([r["confidence"] for r in pos]),
+        "neg_confidence_median": _median([r["confidence"] for r in neg]),
+    }
+
+
+def _median(xs: List[float]) -> Optional[float]:
+    if not xs:
+        return None
+    s = sorted(xs)
+    mid = len(s) // 2
+    return round(s[mid] if len(s) % 2 else (s[mid - 1] + s[mid]) / 2, 4)
+
+
+def _fmt(v) -> str:
+    return "   —  " if v is None else f"{v:6.3f}"
+
+
+def print_report(summary: dict, k: int) -> None:
+    header = f"{'':22}{'n':>4}  " + "  ".join(f"{m.replace('@k', f'@{k}'):>8}" for m in METRICS)
+    print(header)
+    print("─" * len(header))
+
+    def line(label, s):
+        print(f"{label:22}{s['n']:>4}  " + "  ".join(f"{_fmt(s[m]):>8}" for m in METRICS))
+
+    line("OVERALL", summary["overall"])
+    print()
+    for cat, s in summary["by_category"].items():
+        line(f"  {cat}", s)
+    print()
+    for lang, s in summary["by_lang"].items():
+        line(f"  lang={lang}", s)
+    req = summary.get("required") or {}
+    if req.get("n"):
+        print(f"\n  complete@{k} (all must-include sources retrieved): {_fmt(req['complete@k'])}  (n={req['n']})")
+
+    g = summary["gate"]
+    if g is None:
+        print("\nConfidence gate: not evaluated in hybrid mode")
+        return
+    print("\nConfidence gate")
+    print(f"  false-reject rate (answer retrieved but gated): {_fmt(g['false_reject_rate'])}")
+    print(f"  reject rate on all positives:                  {_fmt(g['pos_reject_rate'])}")
+    print(f"  false-accept rate on {g['negatives']} negatives:          {_fmt(g['false_accept_rate'])}")
+    print(f"  median confidence  positives={_fmt(g['pos_confidence_median'])}  "
+          f"negatives={_fmt(g['neg_confidence_median'])}")
+
+
+def print_failures(rows: List[dict], k: int) -> None:
+    misses = [r for r in rows if r.get("hit@k") == 0]
+    incomplete = [r for r in rows if r.get("hit@k") == 1 and r.get("complete@k") == 0]
+    leaks = [r for r in rows if "hit@k" not in r and r["gated"] is False]
+    refused = [r for r in rows if "hit@k" in r and r["gated"]]
+    if misses:
+        print(f"\nMisses (no relevant result in top {k}):")
+        for r in misses:
+            print(f"  {r['id']:<14} [{r['intent']}] {r['query'][:60]}")
+            print(f"  {'':14} got: {', '.join(map(str, r['retrieved'][:5]))}")
+    if incomplete:
+        print(f"\nHits missing a must-include source:")
+        for r in incomplete:
+            print(f"  {r['id']:<14} {r['query'][:50]}  missing: {', '.join(r['missing_required'])}")
+    for title, group in (("Negatives that pass the gate", leaks), ("Positives refused by the gate", refused)):
+        if group:
+            print(f"\n{title}:")
+            for r in group:
+                print(f"  {r['id']:<14} {r['confidence_source']}={r['confidence']:.3f}  "
+                      f"{r['query'][:50]}  → {', '.join(map(str, r['retrieved'][:3]))}")
+
+
+def print_comparison(current: dict, prev_path: Path) -> None:
+    prev = json.loads(prev_path.read_text(encoding="utf-8"))
+    print(f"\nΔ vs {prev_path.name} ({prev.get('timestamp', '?')}, commit {prev.get('git_commit', '?')})")
+
+    def delta_line(label, cur, old):
+        cells = []
+        for m in METRICS:
+            a, b = cur.get(m), (old or {}).get(m)
+            cells.append("    —   " if a is None or b is None else f"{a - b:+8.3f}")
+        print(f"{label:22}      " + "  ".join(cells))
+
+    delta_line("OVERALL", current["summary"]["overall"], prev["summary"]["overall"])
+    cur_req, old_req = current["summary"].get("required") or {}, prev["summary"].get("required") or {}
+    if cur_req.get("complete@k") is not None and old_req.get("complete@k") is not None:
+        print(f"{'  complete@k':22}      {cur_req['complete@k'] - old_req['complete@k']:+8.3f}")
+    for cat, s in current["summary"]["by_category"].items():
+        delta_line(f"  {cat}", s, prev["summary"]["by_category"].get(cat))
+
+    old_rows = {r["id"]: r for r in prev["cases"]}
+    fixed, broke = [], []
+    for r in current["cases"]:
+        o = old_rows.get(r["id"])
+        if not o or "hit@k" not in r or "hit@k" not in o:
+            continue
+        if r["hit@k"] > o["hit@k"]:
+            fixed.append(r["id"])
+        elif r["hit@k"] < o["hit@k"]:
+            broke.append(r["id"])
+    print(f"\n  newly hit:  {', '.join(fixed) or '—'}")
+    print(f"  newly lost: {', '.join(broke) or '—'}")
+
+
+def _git_commit() -> str:
+    try:
+        sha = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, text=True).strip()
+        dirty = subprocess.call(["git", "diff", "--quiet"], cwd=ROOT) != 0
+        return sha + ("-dirty" if dirty else "")
+    except Exception:
+        return "unknown"
+
+
+# ═══════════════════════════════════════════════════════════════════════
+# MAIN
+# ═══════════════════════════════════════════════════════════════════════
+async def main_async(args: argparse.Namespace) -> None:
+    from app.state import state
+
+    only = set(args.only.split(",")) if args.only else None
+    cases = load_golden(Path(args.golden), only)
+    init_state(args.llm)
+    validate_labels(cases, state.dataset)
+
+    ref_to_id = {
+        item["reference"]: item["id"]
+        for item in state.dataset
+        if item.get("type") == "hadith" and item.get("reference")
+    }
+
+    rows, t0 = [], time.perf_counter()
+    for i, case in enumerate(cases, 1):
+        out = await retrieve(case, args.mode, args.k)
+        rows.append(score_case(case, out, args.k, ref_to_id))
+        if sys.stdout.isatty():
+            print(f"\r  {i}/{len(cases)} evaluated", end="", flush=True)
+    elapsed = time.perf_counter() - t0
+    print(f"\r  {len(cases)} cases in {elapsed:.1f}s ({elapsed / max(len(cases), 1):.2f}s/query)\n")
+
+    summary = aggregate(rows)
+    print_report(summary, args.k)
+    if args.show_failures:
+        print_failures(rows, args.k)
+
+    report = {
+        "timestamp": datetime.now().isoformat(timespec="seconds"),
+        "git_commit": _git_commit(),
+        "mode": args.mode,
+        "k": args.k,
+        "golden": str(Path(args.golden).resolve().relative_to(ROOT)),
+        "config": {
+            key: getattr(cfg, key) for key in (
+                "EMBED_MODEL", "TOP_K_SEARCH", "TOP_K_RETURN", "USE_RERANKER", "RERANK_MODEL",
+                "HADITH_BOOST", "QURAN_BOOST", "FATWA_BOOST", "RERANK_MIN_SCORE", "DENSE_MIN_SCORE",
+                "FAST_QUERY_REWRITE", "ENABLE_HADITH_DEDUPLICATION",
+            )
+        },
+        "summary": summary,
+        "cases": rows,
+    }
+
+    if args.compare:
+        print_comparison(report, Path(args.compare))
+
+    out_path = Path(args.out) if args.out else RESULTS_DIR / f"{datetime.now():%Y%m%d-%H%M%S}-{args.mode}.json"
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"\nReport written to {out_path.relative_to(ROOT) if out_path.is_relative_to(ROOT) else out_path}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Evaluate QModel retrieval against a golden set.")
+    parser.add_argument("--golden", default=str(DEFAULT_GOLDEN), help="Golden set JSONL path")
+    parser.add_argument("--mode", choices=("pipeline", "hybrid"), default="pipeline")
+    parser.add_argument("--k", type=int, default=cfg.TOP_K_RETURN,
+                        help=f"Results to score (default TOP_K_RETURN={cfg.TOP_K_RETURN}, what the LLM sees)")
+    parser.add_argument("--only", help="Comma-separated categories to run (e.g. fatwa,negative)")
+    parser.add_argument("--llm", action="store_true", help="Load the LLM backend for query rewriting")
+    parser.add_argument("--show-failures", action="store_true", help="List misses and leaked negatives")
+    parser.add_argument("--compare", help="Previous report JSON to diff against")
+    parser.add_argument("--out", help="Where to write the JSON report (default eval/results/<timestamp>.json)")
+    parser.add_argument("--verbose", action="store_true", help="Show app logging")
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.INFO if args.verbose else logging.WARNING,
+                        format="%(levelname)s %(name)s: %(message)s")
+    asyncio.run(main_async(args))
+
+
+if __name__ == "__main__":
+    main()

main.py

@@ -1,5 +1,5 @@
 """
-QModel v7 — Islamic RAG API
+QModel v8.5 — Islamic RAG API
 ===========================
 Specialized Quran & Hadith system with dual LLM backend support.
 
@@ -39,7 +39,7 @@ from app.routers import chat, fatwa, hadith, ops, quran
 # FASTAPI APP
 # ═══════════════════════════════════════════════════════════════════════
 app = FastAPI(
-    title="QModel v8 — Islamic RAG API",
+    title="QModel v8.5 — Islamic RAG API",
     description=(
         "Specialized Quran, Hadith & Fatwa system with dual LLM backend.\n\n"
         "**Capabilities:**\n"
@@ -47,7 +47,7 @@ app = FastAPI(
         "- Streaming support\n"
         "- Islamic knowledge RAG pipeline (Quran, Hadith, Fatwa)"
     ),
-    version="8.0.0",
+    version="8.5.0",
     lifespan=lifespan,
 )
 

scripts/update_fatwa_index.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""
+Replace the fatwa block of QModel.index + metadata.json without re-embedding Quran/Hadith.
+
+Fatwas are the last block of both files (build_index.py writes quran + hadith + fatwa),
+so this keeps every earlier vector, drops the old fatwa rows, and appends freshly
+embedded fatwas. By default it loads the same fatwas as build_index.py (FATWA_DIRS
+minus FATWA_EXCLUDE), so a fatwa-only update matches what a full rebuild produces.
+
+Usage:
+    python scripts/update_fatwa_index.py                 # build_index.py's fatwa sources
+    python scripts/update_fatwa_index.py --in-place
+    python scripts/update_fatwa_index.py --fatwa-dir data/fatwas_scraped   # override sources
+
+Without --in-place, writes QModel.fatwa-update.index / metadata.fatwa-update.json next to
+the originals; with it, replaces them and keeps .bak copies.
+"""
+
+import argparse
+import json
+import shutil
+import sys
+from pathlib import Path
+
+import faiss
+import numpy as np
+from sentence_transformers import SentenceTransformer
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+from build_index import (  # noqa: E402
+    DEFAULT_EMBED_MODEL,
+    EMBED_BATCH_SIZE,
+    FATWA_DIRS,
+    FATWA_EXCLUDE,
+    INDEX_PATH,
+    METADATA_PATH,
+    load_fatwa_entries,
+    passage_text,
+)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--fatwa-dir", action="append", type=Path,
+                        help="Directory of fatwa JSON files (repeatable; default: build_index.FATWA_DIRS)")
+    parser.add_argument("--exclude-file", action="append", type=Path,
+                        help="JSON file to skip (repeatable; default: build_index.FATWA_EXCLUDE)")
+    parser.add_argument("--model", default=DEFAULT_EMBED_MODEL)
+    parser.add_argument("--in-place", action="store_true", help="Overwrite QModel.index / metadata.json (keeps .bak)")
+    args = parser.parse_args()
+
+    print(f"Loading {INDEX_PATH.name} and {METADATA_PATH.name} …")
+    index = faiss.read_index(str(INDEX_PATH))
+    metadata = json.loads(METADATA_PATH.read_text(encoding="utf-8"))
+
+    first_fatwa = next((i for i, d in enumerate(metadata) if d.get("type") == "fatwa"), len(metadata))
+    if any(d.get("type") == "fatwa" for d in metadata[:first_fatwa]) or \
+       any(d.get("type") != "fatwa" for d in metadata[first_fatwa:]):
+        sys.exit("Fatwas are not a contiguous tail block of metadata.json — run build_index.py instead.")
+    if index.ntotal < first_fatwa:
+        sys.exit(f"Index has {index.ntotal} vectors but {first_fatwa} non-fatwa records — run build_index.py instead.")
+    print(f"  keeping {first_fatwa:,} Quran/Hadith rows; dropping {len(metadata) - first_fatwa} fatwa records "
+          f"and {index.ntotal - first_fatwa} fatwa vectors")
+
+    fatwas = load_fatwa_entries(args.fatwa_dir or FATWA_DIRS, args.exclude_file or FATWA_EXCLUDE)
+    if not fatwas:
+        sys.exit("No fatwas found.")
+
+    import torch
+    device = "mps" if torch.backends.mps.is_available() else ("cuda" if torch.cuda.is_available() else "cpu")
+    print(f"Embedding {len(fatwas):,} fatwas with {args.model} on {device} …")
+    model = SentenceTransformer(args.model, device=device)
+    vectors = model.encode(
+        [passage_text(f) for f in fatwas], batch_size=EMBED_BATCH_SIZE,
+        convert_to_numpy=True, show_progress_bar=True,
+    ).astype(np.float32)
+    faiss.normalize_L2(vectors)
+    if vectors.shape[1] != index.d:
+        sys.exit(f"Embedding dim {vectors.shape[1]} != index dim {index.d} — use the model the index was built with.")
+
+    new_index = faiss.IndexFlatIP(index.d)
+    new_index.add(index.reconstruct_n(0, first_fatwa))
+    new_index.add(vectors)
+    new_metadata = metadata[:first_fatwa] + fatwas
+    assert new_index.ntotal == len(new_metadata)
+
+    if args.in_place:
+        out_index, out_meta = INDEX_PATH, METADATA_PATH
+        shutil.copy2(INDEX_PATH, INDEX_PATH.with_suffix(".index.bak"))
+        shutil.copy2(METADATA_PATH, METADATA_PATH.with_suffix(".json.bak"))
+    else:
+        out_index = INDEX_PATH.with_name("QModel.fatwa-update.index")
+        out_meta = METADATA_PATH.with_name("metadata.fatwa-update.json")
+
+    faiss.write_index(new_index, str(out_index))
+    out_meta.write_text(json.dumps(new_metadata, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"Wrote {out_index.name} ({new_index.ntotal:,} vectors) and {out_meta.name} "
+          f"({len(new_metadata):,} records, {len(fatwas):,} fatwas)")
+
+
+if __name__ == "__main__":
+    main()

tests/test_enhancements.py

@@ -240,10 +240,11 @@ class TestReviewFixes(unittest.IsolatedAsyncioTestCase):
         embed = MagicMock()
         embed.encode.return_value = np.zeros((2, 4), dtype="float32")
         index = MagicMock()
-        # First call returns doc 0, later calls doc 1
+        # One batched call per search (Arabic + English vectors → two rows):
+        # the first search returns doc 0, the second doc 1
         index.search.side_effect = [
-            (np.array([[0.9]]), np.array([[0]])), (np.array([[0.9]]), np.array([[0]])),
-            (np.array([[0.9]]), np.array([[1]])), (np.array([[0.9]]), np.array([[1]])),
+            (np.array([[0.9], [0.9]]), np.array([[0], [0]])),
+            (np.array([[0.9], [0.9]]), np.array([[1], [1]])),
         ]
         rw1 = {"ar_query": "تفسير آية الكرسي", "en_query": "kursi", "keywords": [], "intent": "tafsir"}
         rw2 = {"ar_query": "تفسير سورة الإخلاص", "en_query": "ikhlas", "keywords": [], "intent": "tafsir"}
@@ -272,25 +273,35 @@ class TestReviewFixes(unittest.IsolatedAsyncioTestCase):
         ctx = build_context([huge])
         self.assertLessEqual(len(ctx), cfg.CONTEXT_MAX_CHARS + 100)
 
-    # ── #5: citation verification uses precomputed Quran texts ───────
-    def test_verify_citations_uses_precomputed_quran(self):
-        from app import state as state_mod
+    # ── #5: citation verification is grounded in the retrieved results ──
+    def test_verify_citations_grounded_in_retrieved_results(self):
+        from app.state import _verify_citations
         verse = "اللَّهُ لَا إِلَٰهَ إِلَّا هُوَ الْحَيُّ الْقَيُّومُ"
-        saved = (state_mod.state.quran_norm_texts, state_mod.state.dataset, state_mod.state.bm25_index)
-        try:
-            norm = state_mod.normalize_arabic(verse)
-            state_mod.state.quran_norm_texts = [
-                (norm, frozenset(norm.split())),
-                ("الم", frozenset({"الم"})),  # short verse must not whitelist everything
-            ]
-            state_mod.state.dataset = None
-            state_mod.state.bm25_index = None
-            ok = state_mod._verify_citations(f"❝{verse}❞", results=[])
-            self.assertIn(verse, ok)
-            fake = state_mod._verify_citations("❝قال الملك العظيم كلاما مختلقا تماما هنا❞", results=[])
-            self.assertIn("⚠️", fake)
-        finally:
-            (state_mod.state.quran_norm_texts, state_mod.state.dataset, state_mod.state.bm25_index) = saved
+        retrieved = [{"type": "quran", "arabic": verse + " لَا تَأْخُذُهُ سِنَةٌ وَلَا نَوْمٌ"}]
+
+        # A quote found in the retrieved passages is kept
+        self.assertIn(verse, _verify_citations(f"❝{verse}❞", results=retrieved))
+        # A genuine verse that was not retrieved still fails (strict grounding)
+        self.assertIn("⚠️", _verify_citations(f"❝{verse}❞", results=[]))
+        # A short retrieved verse must not whitelist an unrelated long quote
+        short = [{"type": "hadith", "arabic": "الم"}, {"type": "hadith", "arabic": "x"}]
+        self.assertIn("⚠️", _verify_citations("❝قال الملك العظيم كلاما مختلقا تماما هنا❞", results=short))
+
+    def test_verify_citations_restores_corrupted_scripture(self):
+        from app.state import _verify_citations
+        v_2_286 = "لَا يُكَلِّفُ ٱللَّهُ نَفۡسًا إِلَّا وُسۡعَهَاۚ لَهَا مَا كَسَبَتۡ وَعَلَيۡهَا مَا ٱكۡتَسَبَتۡۗ"
+        v_2_45 = "وَٱسۡتَعِينُواْ بِٱلصَّبۡرِ وَٱلصَّلَوٰةِۚ وَإِنَّهَا لَكَبِيرَةٌ إِلَّا عَلَى ٱلۡخَٰشِعِينَ"
+        retrieved = [{"type": "quran", "arabic": v_2_45}, {"type": "quran", "arabic": v_2_286}]
+
+        # A foreign letter inside a word (Hebrew alef) → canonical verse restored
+        corrupted = v_2_286.replace("إِلَّا", "إِلَّא")
+        self.assertEqual(_verify_citations(f"﴿{corrupted}﴾", retrieved), f"﴿{v_2_286}﴾")
+        # Plain (non-Uthmani) spelling → matched to the retrieved verse it copies, not the top one
+        plain = "واستعينوا بالصبر والصلاة وإنها لكبيرة إلا على الخاشعين"
+        self.assertEqual(_verify_citations(f"﴿{plain}﴾", retrieved), f"﴿{v_2_45}﴾")
+        # A quote far from any retrieved verse is not swapped for one
+        other = "وَٱعۡتَصِمُواْ بِحَبۡلِ ٱللَّهِ جَمِيعٗا وَلَا تَفَرَّقُواْ"
+        self.assertIn("⚠️", _verify_citations(f"﴿{other}﴾", retrieved))
 
 
     # ── #6: arbitrary window size in expand_quran_context ───────────
@@ -315,8 +326,8 @@ class TestReviewFixes(unittest.IsolatedAsyncioTestCase):
         from app.search import rerank_with_cross_encoder
 
         mock_encoder = MagicMock()
-        # Returns raw unbounded logits (negative and positive)
-        mock_encoder.predict.return_value = [-3.5, 4.2]
+        # CrossEncoder.predict applies sigmoid for single-label rerankers → probabilities
+        mock_encoder.predict.return_value = [0.03, 0.98]
 
         candidates = [
             {"arabic": "ضعيف الصلة", "english": "less relevant", "_score": 0.5},
@@ -330,5 +341,430 @@ class TestReviewFixes(unittest.IsolatedAsyncioTestCase):
         self.assertGreater(reranked[1]["_score"], 0.0)
 
 
+class TestRetrievalRouting(unittest.TestCase):
+
+    def test_detect_source_type(self):
+        from app.state import _detect_source_type
+
+        self.assertEqual(_detect_source_type("Is alcohol forbidden in the Quran?"), "quran")
+        self.assertEqual(_detect_source_type("تحريم الخمر في القرآن"), "quran")
+        self.assertEqual(_detect_source_type("The Prophet's advice about anger"), "hadith")
+        self.assertEqual(_detect_source_type("حديث جبريل عن الإسلام"), "hadith")
+        self.assertEqual(_detect_source_type("هل يجوز السلام على المرأة الأجنبية؟"), "fatwa")
+        # Naming both sources keeps retrieval unfiltered
+        self.assertIsNone(_detect_source_type("What do the Quran and Sunnah say about fasting?"))
+        self.assertIsNone(_detect_source_type("Who is obliged to perform Hajj?"))
+
+    def test_bare_surah_verse_reference(self):
+        from app.search import lookup_quran_verses
+
+        verse = {
+            "id": "2:183", "type": "quran", "surah_number": 2, "verse_number": 183,
+            "surah_name_ar": "البقرة", "surah_name_en": "The Cow",
+            "surah_name_transliteration": "Al-Baqarah",
+        }
+        for query in ("Al-Baqarah 183", "البقرة 183", "Baqarah 183", "The Cow 183"):
+            self.assertEqual([r["id"] for r in lookup_quran_verses(query, [verse])], ["2:183"], query)
+        # A name that is not exactly a surah name must not be read as a reference
+        for query in ("iPhone 183", "Chapter 183", "Al-Baqarah tafsir 183"):
+            self.assertEqual(lookup_quran_verses(query, [verse]), [], query)
+
+    def test_fuse_ranks_semantic_hits_above_weak_text_overlap(self):
+        from app.search import fuse_search_results
+
+        semantic = [{"id": f"h{i}", "type": "fatwa", "source": f"s{i}", "arabic": f"a{i}", "_score": 0.8} for i in range(3)]
+        weak_text = [{"id": "noise", "type": "fatwa", "source": "n", "arabic": "n", "_score": 1.5, "_match": "partial"}]
+        exact = [{"id": "quote", "type": "fatwa", "source": "q", "arabic": "q", "_score": 3.0, "_match": "exact"}]
+        direct = [{"id": "ref", "type": "fatwa", "source": "r", "arabic": "r", "_score": 9.5}]
+
+        fused = fuse_search_results(
+            direct, semantic, [("three content words here", exact + weak_text)],
+        )
+        ids = [r["id"] for r in fused]
+        self.assertEqual(ids[:2], ["ref", "quote"])  # pinned tiers first
+        self.assertNotIn("noise", ids)               # word-overlap hits dropped
+        self.assertEqual(ids[2:], ["h0", "h1", "h2"])
+        self.assertTrue(fused[0]["_pinned"] and fused[1]["_pinned"])
+
+
+class TestAuthPhrasing(unittest.TestCase):
+
+    def test_strip_auth_phrasing(self):
+        from app.search import _detect_intent_regex, strip_auth_phrasing
+
+        cases = {
+            "هل حديث طلب العلم فريضة على كل مسلم صحيح؟": "طلب العلم فريضة على كل مسلم",
+            "ما درجة حديث: أبغض الحلال إلى الله الطلاق": "أبغض الحلال إلى الله الطلاق",
+            "هل صحيح أن النبي قال الجنة تحت أقدام الأمهات؟": "الجنة تحت أقدام الأمهات",
+            "Is the hadith 'actions are by intentions' authentic?": "actions are by intentions",
+            "Is it true that the Prophet said religion is sincere advice?": "religion is sincere advice",
+        }
+        for query, expected in cases.items():
+            self.assertEqual(_detect_intent_regex(query), "auth", query)
+            self.assertEqual(strip_auth_phrasing(query), expected, query)
+        # Apostrophes inside words are not quote marks
+        self.assertEqual(strip_auth_phrasing("Is the Prophet's saying about anger authentic?"),
+                         "Is the Prophet's saying about anger")
+        # Nothing left to extract → keep the original text
+        self.assertEqual(strip_auth_phrasing("صحيح؟"), "صحيح؟")
+
+
+class TestHadithVerifyRanking(unittest.IsolatedAsyncioTestCase):
+
+    async def test_verify_prefers_semantic_match_over_word_overlap(self):
+        from unittest.mock import patch
+        from app.routers import hadith as hadith_router
+        from app.state import state
+
+        weak = {"id": "noise", "type": "hadith", "collection": "Sunan an-Nasai", "hadith_number": 1,
+                "reference": "Sunan an-Nasai 1", "arabic": "x", "grade": "Da'if", "_score": 1.0, "_match": "partial"}
+        semantic = {"id": "bukhari_1", "type": "hadith", "collection": "Sahih al-Bukhari", "hadith_number": 1,
+                    "reference": "Sahih al-Bukhari 1", "arabic": "y", "grade": "Sahih", "_score": 0.7}
+
+        saved_ready = state.ready
+        state.ready = True
+        try:
+            with patch.object(hadith_router, "lookup_hadith_references", return_value=[]), \
+                 patch.object(hadith_router, "text_search", return_value=[weak]), \
+                 patch.object(hadith_router, "rewrite_query", AsyncMock(return_value={"intent": "general"})), \
+                 patch.object(hadith_router, "hybrid_search", AsyncMock(return_value=[semantic])):
+                resp = await hadith_router.verify_hadith(q="actions are by intentions", collection=None)
+        finally:
+            state.ready = saved_ready
+
+        self.assertTrue(resp.found)
+        self.assertEqual(resp.reference, "Sahih al-Bukhari 1")
+        self.assertEqual(resp.grade, "Sahih")
+
+
 if __name__ == "__main__":
     unittest.main()
+
+
+class TestLanguageDriftGuard(unittest.IsolatedAsyncioTestCase):
+    """Answers that drift into Chinese are regenerated (sync) or cut off (stream)."""
+
+    def _ctx(self):
+        return {"results": [{"type": "quran", "arabic": "x", "surah_number": 1, "verse_number": 1}],
+                "surah_info": None, "analysis": None, "lang": "english", "top_score": 1.0,
+                "intent": "tafsir", "confident": True, "confidence": 1.0, "confidence_source": "lexical"}
+
+    async def _run(self, replies):
+        from unittest.mock import MagicMock, patch
+        from app import state as state_mod
+        llm = MagicMock()
+        llm.chat = AsyncMock(side_effect=replies)
+        with patch.object(state_mod, "_rag_preprocess", AsyncMock(return_value=self._ctx())), \
+             patch.object(state_mod.state, "llm", llm), \
+             patch.object(state_mod.pipeline_cache, "get", AsyncMock(return_value=None)), \
+             patch.object(state_mod.pipeline_cache, "set", AsyncMock()):
+            out = await state_mod.run_rag_pipeline("q-drift-test")
+        return out["answer"], llm.chat
+
+    async def test_regenerates_once_on_cjk(self):
+        answer, chat = await self._run(["Fasting is 真主 obligatory.", "Fasting is obligatory."])
+        self.assertEqual(answer, "Fasting is obligatory.")
+        self.assertEqual(chat.call_count, 2)
+        self.assertEqual(chat.call_args.kwargs["temperature"], 0.0)
+        self.assertIn("no Chinese characters", chat.call_args.args[0][-1]["content"])
+
+    async def test_falls_back_when_cjk_persists(self):
+        answer, _ = await self._run(["答案 one", "答案 two"])
+        self.assertNotRegex(answer, r"[一-鿿]")
+
+    def test_stream_buffer_strips_cjk_and_holds_quran_quotes(self):
+        from app.state import StreamingVerificationBuffer
+        buf = StreamingVerificationBuffer(results=[{"type": "quran", "arabic": "الحمد لله رب العالمين"}])
+        self.assertEqual(buf.feed("Allah is 真主 merciful.\n"), ["Allah is  merciful.\n"])
+        self.assertEqual(buf.cjk_chars, 2)
+        # An open ﴿ quote is held across a line break until it closes
+        self.assertEqual(buf.feed("﴿الحمد لله\n"), [])
+        self.assertEqual(buf.feed("رب العالمين﴾\n"), ["﴿الحمد لله\nرب العالمين﴾\n"])
+
+
+class TestFatwaLinks(unittest.TestCase):
+    RESULTS = [
+        {"type": "fatwa", "source": "Islamweb", "title": "لا يقع طلاق الغضبان إذا غلب على عقله",
+         "source_url": "https://www.islamweb.net/ar/fatwa/101"},
+        {"type": "fatwa", "source": "Islamweb", "title": "حكم الحقنة في العضل للصائم",
+         "source_url": "https://www.islamweb.net/ar/fatwa/102"},
+        {"type": "fatwa", "source": "BinBaz", "title": "عدد الرضعات المحرمة",
+         "source_url": "https://binbaz.org.sa/fatwas/7"},
+    ]
+
+    def test_links_fatwa_whose_title_is_used(self):
+        from app.state import _attach_source_refs
+        answer = "الأرجح أنه لا يقع طلاق الغضبان إذا غلب الغضب على عقله.\n\nوالله أعلم."
+        out = _attach_source_refs(answer, self.RESULTS, "arabic")
+        self.assertIn("(https://www.islamweb.net/ar/fatwa/101)", out)
+        self.assertNotIn("fatwa/102", out)
+        self.assertTrue(out.endswith("والله أعلم."))
+
+    def test_source_credit_links_that_sources_top_fatwa(self):
+        from app.state import _attach_source_refs
+        out = _attach_source_refs("قال الإمام ابن باز: لا بد من خمس رضعات.", self.RESULTS, "arabic")
+        self.assertIn("binbaz.org.sa/fatwas/7", out)
+        self.assertNotIn("islamweb", out)
+
+    def test_no_link_when_already_linked_or_unrelated(self):
+        from app.state import _attach_source_refs
+        linked = "See [[1](https://www.islamweb.net/ar/fatwa/102)]. And Allah knows best."
+        self.assertEqual(_attach_source_refs(linked, self.RESULTS, "english"), linked)
+        plain = "Fasting is obligatory in Ramadan. And Allah knows best."
+        self.assertEqual(_attach_source_refs(plain, self.RESULTS, "english"), plain)
+
+    COPIED = ("المسافة التي يباح للمسافر الأخذ برخص السفر من قصر وفطر وغير ذلك هي ما يعادل "
+              "ثلاثة وثمانين كيلومترا عند جمهور أهل العلم ولا يجوز الفطر فيما دون ذلك")
+
+    def test_links_fatwa_whose_text_is_copied(self):
+        from app.state import _attach_source_refs
+        fatwa = {"type": "fatwa", "source": "Islamweb", "title": "حد السفر",
+                 "answer_ar": "الحمد لله، " + self.COPIED + ".", "source_url": "https://www.islamweb.net/ar/fatwa/233/"}
+        out = _attach_source_refs(self.COPIED + ".\n\nوالله أعلم.", self.RESULTS + [fatwa], "arabic")
+        self.assertIn("(https://www.islamweb.net/ar/fatwa/233/)", out)
+        self.assertNotIn("fatwa/101", out)
+
+    def test_shared_scripture_is_not_copying(self):
+        from app.state import _attach_source_refs
+        verse = {"type": "quran", "arabic": self.COPIED}
+        fatwa = {"type": "fatwa", "source": "Islamweb", "title": "حد السفر",
+                 "answer_ar": self.COPIED, "source_url": "https://www.islamweb.net/ar/fatwa/233/"}
+        answer = self.COPIED + "."
+        self.assertEqual(_attach_source_refs(answer, [verse, fatwa], "arabic"), answer)
+        self.assertEqual(_attach_source_refs(f"﴿{self.COPIED}﴾", [fatwa], "arabic"), f"﴿{self.COPIED}﴾")
+
+    def test_link_template_copied_literally(self):
+        from app.state import _clean_links
+        self.assertEqual(_clean_links("[[1]({https://www.islamweb.net/ar/fatwa/930/})]"),
+                         "[[1](https://www.islamweb.net/ar/fatwa/930/)]")
+        self.assertEqual(_clean_links("والله أعلم. [[1]({source_url})]"), "والله أعلم.")
+        self.assertEqual(_clean_links("[[1](!https://quran.com/4/161)]"), "[[1](https://quran.com/4/161)]")
+        self.assertEqual(_clean_links("![img](https://x.y/a.png)"), "![img](https://x.y/a.png)")
+
+
+class TestLightStem(unittest.TestCase):
+    def test_short_words_keep_root_letters(self):
+        from app.arabic_nlp import light_stem, normalize_arabic
+        stem = lambda w: light_stem(normalize_arabic(w, aggressive=True))
+        # A leading و/ف/ب/ل or trailing ا in a 3-letter word is a root letter, not an affix
+        for word, expected in [("فضل", "فضل"), ("فتح", "فتح"), ("لبس", "لبس"), ("الربا", "ربا")]:
+            self.assertEqual(stem(word), expected, word)
+        # Real affixes are still removed
+        for word, expected in [("كتابه", "كتاب"), ("وقالوا", "قال"), ("مسلمون", "مسلم"), ("الوضوء", "وضوء")]:
+            self.assertEqual(stem(word), expected, word)
+
+
+class TestSparseRerankSlots(unittest.TestCase):
+    def test_top_bm25_hits_enter_rerank_pool(self):
+        from app.search import _RERANK_POOL, _SPARSE_POOL_SLOTS, _reserve_sparse_slots
+        fused = [{"_dataset_idx": i} for i in range(40)]
+        # BM25 liked 2 (already in pool) and 30, 35 (outside it)
+        out = _reserve_sparse_slots(fused, [2, 30, 35])
+        pool = [c["_dataset_idx"] for c in out[:_RERANK_POOL]]
+        self.assertIn(30, pool)
+        self.assertIn(35, pool)
+        self.assertEqual(pool[: _RERANK_POOL - _SPARSE_POOL_SLOTS], list(range(_RERANK_POOL - _SPARSE_POOL_SLOTS)))
+        self.assertEqual(sorted(c["_dataset_idx"] for c in out), list(range(40)))  # nothing lost or duplicated
+
+
+class TestStreamingPipeline(unittest.IsolatedAsyncioTestCase):
+    """run_rag_pipeline_stream end to end with a scripted LLM token stream."""
+
+    VERSE = "لَا يُكَلِّفُ ٱللَّهُ نَفۡسًا إِلَّا وُسۡعَهَاۚ لَهَا مَا كَسَبَتۡ وَعَلَيۡهَا مَا ٱكۡتَسَبَتۡۗ"
+    FATWA = {"type": "fatwa", "source": "Islamweb", "title": "الحقنة في العضل لا تبطل الصوم إذا كانت للتداوي",
+             "source_url": "https://www.islamweb.net/ar/fatwa/857"}
+
+    async def _stream(self, tokens, results, lang="arabic"):
+        from unittest.mock import MagicMock, patch
+        from app import state as state_mod
+        ctx = {"results": results, "surah_info": None, "analysis": None, "lang": lang, "top_score": 1.0,
+               "intent": "general", "confident": True, "confidence": 1.0, "confidence_source": "lexical"}
+
+        async def chat_stream(*_a, **_k):
+            for t in tokens:
+                yield t
+
+        llm = MagicMock()
+        llm.chat_stream = chat_stream
+        with patch.object(state_mod, "_rag_preprocess", AsyncMock(return_value=ctx)), \
+             patch.object(state_mod.state, "llm", llm):
+            events = [e async for e in state_mod.run_rag_pipeline_stream("q")]
+        return "".join(e["content"] for e in events if e["type"] == "token")
+
+    async def test_corrupted_quote_split_across_tokens_is_restored(self):
+        corrupted = self.VERSE.replace("إِلَّا", "إِلَّא")  # Hebrew alef inside the word
+        half = len(corrupted) // 2
+        text = await self._stream(["قال تعالى:\n﴿", corrupted[:half], "\n", corrupted[half:], "﴾\n", "والله أعلم."],
+                                  [{"type": "quran", "arabic": self.VERSE}])
+        self.assertIn(f"﴿{self.VERSE}﴾", text)
+        self.assertNotIn("א", text)
+
+    async def test_fatwa_link_appended_after_stream(self):
+        text = await self._stream(["ذكر مركز الفتوى في إسلام ويب أن الحقنة لا تفطر.\n", "والله أعلم."], [self.FATWA])
+        self.assertTrue(text.rstrip().endswith("(https://www.islamweb.net/ar/fatwa/857)"))
+
+    async def test_language_drift_stops_generation(self):
+        drift = ["The ruling is clear.\n", "根据提供的教法裁定，对于因非骄傲或炫耀目的而发生的衣着过长。\n", "More text.\n"]
+        text = await self._stream(drift, [{"type": "quran", "arabic": self.VERSE}], lang="english")
+        self.assertIn("The ruling is clear.", text)
+        self.assertNotRegex(text, r"[一-鿿]")
+        self.assertNotIn("More text.", text)
+        self.assertIn("cut short", text)
+
+
+class TestSynonyms(unittest.TestCase):
+    def test_alcohol_reaches_quranic_vocabulary(self):
+        from app.synonyms import expand_synonyms
+        self.assertIn("خمر", expand_synonyms(["Alcohol"]))
+
+    def test_zakat_recipients_phrase_but_not_bare_masarif(self):
+        from app.synonyms import expand_synonyms
+        self.assertIn("أصناف الزكاة", expand_synonyms(["مصارف الزكاة"]))
+        self.assertEqual(expand_synonyms(["مصارف"]), ["مصارف"])  # alone it may mean banks
+
+
+class TestScopeGate(unittest.TestCase):
+    def test_creative_requests_detected(self):
+        from app.state import _CREATIVE_REQUEST_RE as rx
+        for q in ["Write a poem about the ocean", "compose me a short song about Ramadan",
+                  "Tell me a joke", "اكتب قصيدة عن البحر", "اكتب لي أبيات في الصبر"]:
+            self.assertRegex(q, rx)
+
+    def test_real_questions_not_caught(self):
+        from app.state import _CREATIVE_REQUEST_RE as rx
+        for q in ["Tell me the story of Prophet Yusuf", "How do I write a will?",
+                  "Is listening to songs haram?", "حكم صبغ الشعر", "هل كتابة الوصية واجبة؟",
+                  "ما حكم الشعر في الإسلام؟", "Write down debts and loan contracts"]:
+            self.assertNotRegex(q, rx)
+
+
+class TestFastRewrite(unittest.IsolatedAsyncioTestCase):
+    async def test_count_word_from_question_not_first_keyword(self):
+        from app import analysis
+        from app.search import rewrite_query
+        for q, word in [("كم مرة ذكرت كلمة الصبر في القرآن", "الصبر"),
+                        ("How many times is the word patience mentioned in the Quran?", "patience"),
+                        ("عدد مرات ذكر كلمة الجنة في القرآن", "الجنة"),
+                        ("كم مرة ورد لفظ الصلاة", "الصلاة"),
+                        ("كم مرة ذكر الصبر", "الصبر"),
+                        ("How many times does mercy appear in the Quran?", "mercy"),
+                        ("Count of the word taqwa in the Quran", "taqwa")]:
+            r = await rewrite_query(q, None)
+            self.assertEqual(await analysis.detect_analysis_intent(q, r), word)
+
+    async def test_keywords_are_stemmed_once(self):
+        from app.search import rewrite_query
+        from app.arabic_nlp import tokenize_bilingual
+        r = await rewrite_query("الوضوء في القرآن الكريم", None)
+        terms = [t for k in r["keywords"] for t in tokenize_bilingual(k)]
+        self.assertIn("وضوء", terms)
+        self.assertNotIn("ضوء", terms)  # stemming twice strips the root letter و
+
+    async def test_thesaurus_reaches_fast_rewrite(self):
+        from app.search import rewrite_query
+        r = await rewrite_query("فضل السواك للصائم", None)
+        self.assertIn("siwak", r["keywords"])
+        self.assertIn("siwak", r["en_query"])  # English vector gets English terms
+
+
+class TestHadithRefs(unittest.TestCase):
+    HADITH = {"type": "hadith", "collection": "Sahih al-Bukhari", "hadith_number": 631,
+              "reference": "Sahih al-Bukhari 631",
+              "arabic": "حَدَّثَنَا مُحَمَّدُ بْنُ الْمُثَنَّى قَالَ النَّبِيُّ صلى الله عليه وسلم وَصَلُّوا كَمَا رَأَيْتُمُونِي أُصَلِّي فَإِذَا حَضَرَتِ الصَّلاَةُ فَلْيُؤَذِّنْ لَكُمْ أَحَدُكُمْ",
+              "english": "Narrated Malik: The Prophet said, \"Pray as you have seen me praying and when the time for the prayer is due, one of you should pronounce the Adhan.\""}
+
+    def test_quoted_hadith_without_number_gets_reference(self):
+        from app.state import _attach_source_refs
+        answer = ('The Prophet said, "Pray as you have seen me praying and when the time for the prayer '
+                  'is due, one of you should pronounce the Adhan."\n\nAnd Allah knows best.')
+        out = _attach_source_refs(answer, [self.HADITH], "english")
+        self.assertIn("Reference: Sahih al-Bukhari 631", out)
+        self.assertTrue(out.endswith("And Allah knows best."))
+
+    def test_arabic_answer_gets_arabic_reference(self):
+        from app.state import _attach_source_refs
+        answer = "قال النبي: وَصَلُّوا كَمَا رَأَيْتُمُونِي أُصَلِّي فَإِذَا حَضَرَتِ الصَّلاَةُ فَلْيُؤَذِّنْ لَكُمْ أَحَدُكُمْ"
+        self.assertIn("المرجع: صحيح البخاري، رقم 631", _attach_source_refs(answer, [self.HADITH], "arabic"))
+
+    def test_cited_or_unquoted_hadith_left_alone(self):
+        from app.state import _attach_source_refs
+        cited = 'Bukhari 631: "Pray as you have seen me praying and when the time for the prayer is due, one of you should pronounce the Adhan."'
+        self.assertEqual(_attach_source_refs(cited, [self.HADITH], "english"), cited)
+        unrelated = "Prayer is obligatory five times a day. And Allah knows best."
+        self.assertEqual(_attach_source_refs(unrelated, [self.HADITH], "english"), unrelated)
+
+    def test_shared_isnad_alone_is_not_a_quote(self):
+        from app.state import _attach_source_refs
+        other = dict(self.HADITH, hadith_number=1, reference="Sahih al-Bukhari 1",
+                     arabic="حَدَّثَنَا مُحَمَّدُ بْنُ الْمُثَنَّى قَالَ حَدَّثَنَا عَبْدُ الْوَهَّابِ عَنْ أَيُّوبَ إِنَّمَا الأَعْمَالُ بِالنِّيَّاتِ وَإِنَّمَا لِكُلِّ امْرِئٍ مَا نَوَى",
+                     english="Actions are judged by intentions.")
+        answer = "حدثنا محمد بن المثنى قال حدثنا عبد الوهاب عن أيوب: صلوا كما رأيتموني أصلي."
+        self.assertNotIn("رقم 1", _attach_source_refs(answer, [other], "arabic"))
+
+
+class TestStreamSoftSplit(unittest.TestCase):
+    def _feed(self, text, step=7):
+        from app.state import StreamingVerificationBuffer
+        buf = StreamingVerificationBuffer(results=[])
+        out = []
+        for i in range(0, len(text), step):
+            out.extend(buf.feed(text[i:i + step]))
+        return out, buf
+
+    def test_long_isnad_released_at_commas(self):
+        isnad = ("حدثنا الحميدي عبد الله بن الزبير، قال حدثنا سفيان، قال حدثنا يحيى بن سعيد الأنصاري، "
+                 "قال أخبرني محمد بن إبراهيم التيمي، أنه سمع علقمة بن وقاص الليثي، يقول سمعت عمر بن الخطاب")
+        out, buf = self._feed(isnad)
+        self.assertTrue(out, "nothing released before the sentence ended")
+        self.assertTrue(all(c.endswith("، ") for c in out))
+        self.assertEqual("".join(out) + buf.buffer, isnad)
+
+    def test_surah_reference_not_split(self):
+        text = ("وقد ذكر الله تعالى هذا المعنى في مواضع كثيرة من كتابه الكريم وبينه أهل التفسير بيانا واضحا "
+                "كما في سورة البقرة، الآية 255 وهي آية الكرسي التي")
+        out, buf = self._feed(text)
+        self.assertFalse(any(c.rstrip().endswith("سورة البقرة،") for c in out))
+        self.assertIn("سورة البقرة، الآية 255", "".join(out[-1:]) + buf.buffer)
+
+    def test_space_fallback_keeps_references_whole(self):
+        words = "وهذا بيان طويل جدا لمعنى الآية الكريمة " * 9  # > 300 chars, no commas or full stops
+        text = words + "كما في سورة البقرة الآية 255 وهي آية الكرسي العظيمة " + words
+        out, buf = self._feed(text, step=5)
+        self.assertTrue(out)
+        for c in out:
+            self.assertNotRegex(c.rstrip(), r"(?:سورة|سورة البقرة|الآية)$")
+        self.assertIn("سورة البقرة الآية 255", "".join(out) + buf.buffer)
+
+
+class TestRerankerQuery(unittest.TestCase):
+    def test_source_phrase_dropped_and_thesaurus_added(self):
+        from app.search import _reranker_query
+        self.assertEqual(_reranker_query("Is alcohol forbidden in the Quran?"),
+                         "Is alcohol forbidden in the Quran? (khamr, wine, intoxicants)")  # English keeps it
+        self.assertEqual(_reranker_query("الوضوء في القرآن"), "الوضوء (فاغسلوا وجوهكم)")
+
+    def test_quran_as_subject_kept(self):
+        from app.search import _reranker_query
+        self.assertEqual(_reranker_query("In which month was the Quran revealed?"), "In which month was the Quran revealed?")
+        self.assertEqual(_reranker_query("قراءة القرآن"), "قراءة القرآن")
+
+
+class TestSurahUrls(unittest.TestCase):
+    VERSE = {"type": "quran", "surah_number": 2, "verse_number": 278, "surah_name_en": "Al-Baqarah",
+             "surah_name_ar": "البقرة", "arabic": "يا أيها الذين آمنوا اتقوا الله وذروا ما بقي من الربا"}
+
+    def test_url_format(self):
+        from app.search import surah_url
+        self.assertEqual(surah_url(2, 278), "https://quran.com/2/278")
+        self.assertEqual(surah_url("", 278), "")
+
+    def test_old_links_rewritten_even_without_retrieved_verses(self):
+        from app.state import _verify_references
+        old = "[[1](https://surahquran.com/aya-278-sora-2.html)]"
+        self.assertEqual(_verify_references(old, []), "[[1](https://quran.com/2/278)]")
+        self.assertEqual(_verify_references(old, [self.VERSE]), "[[1](https://quran.com/2/278)]")
+
+    def test_unretrieved_verse_link_corrected(self):
+        from app.state import _verify_references
+        self.assertEqual(_verify_references("(https://quran.com/2/278)", [self.VERSE]), "(https://quran.com/2/278)")
+        self.assertEqual(_verify_references("(https://quran.com/3/130)", [self.VERSE]), "(https://quran.com/2/278)")

tests/test_eval_parsing.py

@@ -0,0 +1,73 @@
+"""Tests for the answer-eval citation parser (eval/run_answers.py)."""
+
+import unittest
+
+from eval.run_answers import cited_ids, is_refusal, lang_ok
+
+
+class TestCitedIds(unittest.TestCase):
+    def test_quran_urls(self):
+        answer = ("[[1](https://surahquran.com/aya-255-sora-2.html)] ﴿...﴾ "
+                  "[[1](https://quran.ksu.edu.sa/tafseer/qortobi/sura24-aya35.html)]")
+        self.assertEqual(cited_ids(answer, {}), {"2:255", "24:35"})
+        self.assertEqual(cited_ids("[[1](https://quran.com/2/278)]", {}), {"2:278"})
+
+    def test_english_hadith_refs(self):
+        answer = ("Recorded in Sahih al-Bukhari 6116. Also Sunan Abu Dawood No. 4800 "
+                  "and Jami' at-Tirmidhi #2516, Sunan Ibn Majah, hadith 224.")
+        self.assertEqual(cited_ids(answer, {}),
+                         {"bukhari_6116", "abudawud_4800", "tirmidhi_2516", "ibnmajah_224"})
+
+    def test_arabic_hadith_refs(self):
+        answer = "رواه صحيح البخاري، رقم 13. وفي صحيح مسلم، رقم 45. وفي موطأ مالك، رقم 1."
+        self.assertEqual(cited_ids(answer, {}), {"bukhari_13", "muslim_45", "malik_1"})
+        answer = ("الترمذي، رقم الحديث 2024. وصحيح مسلم برقم 440. وسنن ابن ماجه، الرقم 4303. "
+                  "والبخاري، كتاب الإيمان، حديث رقم 1. ورواه مسلم في صحيحه (6382).")
+        self.assertEqual(cited_ids(answer, {}),
+                         {"tirmidhi_2024", "muslim_440", "ibnmajah_4303", "bukhari_1", "muslim_6382"})
+
+    def test_hadith_ref_needs_adjacent_number(self):
+        # The number belongs to the collection right before it, not one earlier in the sentence
+        self.assertEqual(cited_ids("Sahih al-Bukhari and Sahih Muslim 1907", {}), {"muslim_1907"})
+        # "مسلم" as an ordinary word ("every Muslim") is not a citation
+        self.assertEqual(cited_ids("طلب العلم فريضة على كل مسلم 3 مرات", {}), set())
+
+    def test_fatwa_urls(self):
+        url_to_id = {"https://www.islamweb.net/ar/fatwa/857": "islamweb_857"}
+        answer = "مركز الفتوى [[1](https://www.islamweb.net/ar/fatwa/857/)]."
+        self.assertEqual(cited_ids(answer, url_to_id), {"islamweb_857"})
+
+
+class TestAnswerChecks(unittest.TestCase):
+    def test_refusal_and_language(self):
+        not_found = {"arabic": "لم أجد", "english": "not found"}
+        self.assertTrue(is_refusal("The dataset does not contain sufficient information.", "en", not_found))
+        self.assertFalse(is_refusal("Yes, it is obligatory. And Allah knows best.", "en", not_found))
+        self.assertTrue(lang_ok("نعم، هذا واجب. والله أعلم.", "ar"))
+        self.assertFalse(lang_ok("Yes, it is obligatory.", "ar"))
+
+
+if __name__ == "__main__":
+    unittest.main()
+
+
+class TestRequiredLabels(unittest.TestCase):
+    def test_hit_without_required_source_is_incomplete(self):
+        from eval.run_eval import score_case
+        case = {"id": "x", "category": "quran_topic", "query": "Is alcohol forbidden?",
+                "relevant": ["5:90", "2:219"], "required": ["5:90"]}
+        out = {"results": [{"id": "29:49"}, {"id": "2:219"}], "intent": "general", "top_score": 1.0,
+               "gated": False, "confidence": 0.9, "confidence_source": "reranker"}
+        row = score_case(case, out, 8, {})
+        self.assertEqual(row["hit@k"], 1)          # still a hit, as before
+        self.assertEqual(row["complete@k"], 0)     # but the decisive verse is missing
+        self.assertEqual(row["missing_required"], ["5:90"])
+
+    def test_required_must_be_subset_of_relevant(self):
+        import tempfile
+        from pathlib import Path
+        from eval.run_eval import load_golden
+        with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False, encoding="utf-8") as f:
+            f.write('{"id": "x", "category": "c", "query": "q", "relevant": ["2:219"], "required": ["5:90"]}\n')
+        with self.assertRaises(ValueError):
+            load_golden(Path(f.name), None)