Update app.py

app.py

@@ -25,17 +25,20 @@ CHUNK_WORDS = 80          # stays under the embedding model's 128-token limit
 CHUNK_OVERLAP = 15
 TABLE_CHUNK_WORDS = 60    # table/spreadsheet rows are token-heavy, so smaller groups
 RRF_K = 60
-OCR_DPI = 300             # lower (e.g. 200) for faster OCR at some cost in accuracy
+OCR_DPI = 400             # higher than before: small typewriter/fax text needs it. Lower (e.g. 300) for speed.
 OCR_LANGS = "ara+eng"
+OCR_FALLBACK_LANGS = "eng"  # used if Arabic language data is missing on the server
+OCR_CONTENT_THRESHOLD = 200  # gray level below which a pixel counts as "content" (0-255)
+OCR_CLIP_PAD = 8             # points of padding around detected content
 
 # ========================= TEXT NORMALIZATION =========================
-DIACRITICS_RE = re.compile(r"[\u0610-\u061A\u064B-\u065F\u0670\u06D6-\u06ED]")
+DIACRITICS_RE = re.compile(r"[ؐ-ًؚ-ٰٟۖ-ۭ]")
 DIGIT_MAP = str.maketrans("٠١٢٣٤٥٦٧٨٩۰۱۲۳۴۵۶۷۸۹", "01234567890123456789")
 
 def light_clean(text):
     text = unicodedata.normalize("NFKC", text)
     text = DIACRITICS_RE.sub("", text)
-    return text.replace("\u0640", "")
+    return text.replace("ـ", "")
 
 def normalize(text):
     text = light_clean(text).translate(DIGIT_MAP)
@@ -75,20 +78,55 @@ def _set_tessdata():
             os.environ["TESSDATA_PREFIX"] = p
             return
 
-def ocr_page(page):
-    """OCR a scanned page. Returns "" if Tesseract or the language data is missing."""
+def _content_clip(page):
+    """Bounding box of non-white content, so large blank margins don't shrink the text during OCR.
+    Returns None if the page is blank, or the full page rect if the content already fills it."""
     try:
-        _set_tessdata()
-        tp = page.get_textpage_ocr(language=OCR_LANGS, dpi=OCR_DPI, full=True)
-        return unicodedata.normalize("NFKC", page.get_text("text", textpage=tp) or "").strip()
+        pix = page.get_pixmap(dpi=72, colorspace=fitz.csGRAY)  # at 72 dpi, 1 pixel = 1 point
+        arr = np.frombuffer(pix.samples, dtype=np.uint8).reshape(pix.height, pix.width)
+        ys, xs = np.where(arr < OCR_CONTENT_THRESHOLD)
+        if xs.size == 0:
+            return None
+        r = page.rect
+        clip = fitz.Rect(
+            r.x0 + xs.min() - OCR_CLIP_PAD, r.y0 + ys.min() - OCR_CLIP_PAD,
+            r.x0 + xs.max() + 1 + OCR_CLIP_PAD, r.y0 + ys.max() + 1 + OCR_CLIP_PAD,
+        ) & r
+        if clip.is_empty or clip.width < 20 or clip.height < 20:
+            return r
+        return clip
     except Exception:
-        return ""
+        return page.rect
+
+def _ocr_with_lang(page, clip, lang):
+    pix = page.get_pixmap(dpi=OCR_DPI, clip=clip, colorspace=fitz.csGRAY)
+    ocr_doc = fitz.open("pdf", pix.pdfocr_tobytes(language=lang))
+    try:
+        return unicodedata.normalize("NFKC", ocr_doc[0].get_text("text") or "").strip()
+    finally:
+        ocr_doc.close()
+
+def ocr_page(page):
+    """OCR a scanned page. Returns (text, error).
+    error is None unless every language attempt raised, so 'OCR ran and found nothing'
+    can be told apart from 'OCR could not run' (e.g. Tesseract or language data missing)."""
+    _set_tessdata()
+    clip = _content_clip(page)
+    if clip is None:  # truly blank page: nothing to OCR, and not an error
+        return "", None
+    errors = []
+    for lang in dict.fromkeys((OCR_LANGS, OCR_FALLBACK_LANGS)):  # de-duplicated, order kept
+        try:
+            return _ocr_with_lang(page, clip, lang), None
+        except Exception as e:
+            errors.append(f"[{lang}] {e}")
+    return "", "; ".join(errors)
 
 def extract_pdf(name, data):
     blocks = []
     stats = {"pages": 0, "tables": 0, "notes": []}
     doc = fitz.open(stream=data, filetype="pdf")
-    empty_pages, ocr_pages = [], []
+    empty_pages, ocr_pages, ocr_errors = [], [], []
     for i, page in enumerate(doc, 1):
         stats["pages"] += 1
         loc = f"p.{i}"
@@ -97,7 +135,9 @@ def extract_pdf(name, data):
             blocks.append(make_block(tbl, name, loc, "table"))
             stats["tables"] += 1
         if len(text) < 50:
-            ocr_text = ocr_page(page)
+            ocr_text, ocr_err = ocr_page(page)
+            if ocr_err:
+                ocr_errors.append((i, ocr_err))
             if len(ocr_text) > len(text):
                 text = ocr_text
                 ocr_pages.append(i)
@@ -108,9 +148,19 @@ def extract_pdf(name, data):
     if ocr_pages:
         shown = ", ".join(map(str, ocr_pages[:15])) + ("..." if len(ocr_pages) > 15 else "")
         stats["notes"].append(f"OCR used on page(s) {shown}. Text may contain recognition errors.")
+    if ocr_errors:
+        # Report the first error once; it usually repeats for every page.
+        first_pages = ", ".join(str(p) for p, _ in ocr_errors[:15]) + ("..." if len(ocr_errors) > 15 else "")
+        stats["notes"].append(
+            f"OCR could not run on page(s) {first_pages}: {ocr_errors[0][1]}. "
+            "Check that Tesseract and its language data (eng, ara) are installed."
+        )
     if empty_pages:
         shown = ", ".join(map(str, empty_pages[:15])) + ("..." if len(empty_pages) > 15 else "")
-        stats["notes"].append(f"No readable text found on page(s) {shown}, even after OCR.")
+        stats["notes"].append(
+            f"No readable text found on page(s) {shown}"
+            + ("." if ocr_errors else ", even after OCR.")
+        )
     return blocks, stats
 
 def extract_docx(name, data):