Update app.py

app.py

@@ -1,5 +1,6 @@
 # app.py - Hybrid RAG (FAISS semantic + BM25, fused with RRF) for multi-format documents.
-# Handles: PDFs (text + tables), DOCX (with tables), PPTX, XLSX/CSV, TXT/MD. Arabic + English.
+# Handles: PDFs (text + tables, OCR for scanned pages), DOCX (with tables), PPTX, XLSX/CSV, TXT/MD. Arabic + English.
+import glob
 import io
 import os
 import re
@@ -24,6 +25,8 @@ CHUNK_WORDS = 80          # stays under the embedding model's 128-token limit
 CHUNK_OVERLAP = 15
 TABLE_CHUNK_WORDS = 60    # table/spreadsheet rows are token-heavy, so smaller groups
 RRF_K = 60
+OCR_DPI = 300             # lower (e.g. 200) for faster OCR at some cost in accuracy
+OCR_LANGS = "ara+eng"
 
 # ========================= TEXT NORMALIZATION =========================
 DIACRITICS_RE = re.compile(r"[\u0610-\u061A\u064B-\u065F\u0670\u06D6-\u06ED]")
@@ -63,11 +66,29 @@ def pdf_tables(page):
         pass
     return out
 
+def _set_tessdata():
+    """Point PyMuPDF at Tesseract's language data (path differs by distro/version)."""
+    if os.environ.get("TESSDATA_PREFIX"):
+        return
+    for p in glob.glob("/usr/share/tesseract-ocr/*/tessdata") + ["/usr/share/tessdata"]:
+        if os.path.isdir(p):
+            os.environ["TESSDATA_PREFIX"] = p
+            return
+
+def ocr_page(page):
+    """OCR a scanned page. Returns "" if Tesseract or the language data is missing."""
+    try:
+        _set_tessdata()
+        tp = page.get_textpage_ocr(language=OCR_LANGS, dpi=OCR_DPI, full=True)
+        return unicodedata.normalize("NFKC", page.get_text("text", textpage=tp) or "").strip()
+    except Exception:
+        return ""
+
 def extract_pdf(name, data):
     blocks = []
     stats = {"pages": 0, "tables": 0, "notes": []}
     doc = fitz.open(stream=data, filetype="pdf")
-    empty_pages = []
+    empty_pages, ocr_pages = [], []
     for i, page in enumerate(doc, 1):
         stats["pages"] += 1
         loc = f"p.{i}"
@@ -75,13 +96,21 @@ def extract_pdf(name, data):
         for tbl in pdf_tables(page):
             blocks.append(make_block(tbl, name, loc, "table"))
             stats["tables"] += 1
+        if len(text) < 50:
+            ocr_text = ocr_page(page)
+            if len(ocr_text) > len(text):
+                text = ocr_text
+                ocr_pages.append(i)
         if len(text) < 50:
             empty_pages.append(i)
         if text:
             blocks.append(make_block(text, name, loc, "text"))
+    if ocr_pages:
+        shown = ", ".join(map(str, ocr_pages[:15])) + ("..." if len(ocr_pages) > 15 else "")
+        stats["notes"].append(f"OCR used on page(s) {shown}. Text may contain recognition errors.")
     if empty_pages:
         shown = ", ".join(map(str, empty_pages[:15])) + ("..." if len(empty_pages) > 15 else "")
-        stats["notes"].append(f"Little or no text on page(s) {shown}. Scanned pages cannot be read by this app.")
+        stats["notes"].append(f"No readable text found on page(s) {shown}, even after OCR.")
     return blocks, stats
 
 def extract_docx(name, data):
@@ -301,7 +330,7 @@ if uploaded_files:
     signature = tuple(sorted((f.name, f.size) for f in uploaded_files))
     if st.session_state.get("signature") != signature:
         all_blocks, report = [], {}
-        with st.spinner("Reading documents..."):
+        with st.spinner("Reading documents (scanned pages are OCR'd and may take a while)..."):
             for f in uploaded_files:
                 try:
                     blocks, stats = extract_file(f.name, f.getvalue())