| from typing import List |
| from io import BytesIO |
| import re |
|
|
| from PyPDF2 import PdfReader |
| from PyPDF2.errors import DependencyError |
|
|
|
|
| def extract_text_pypdf2(pdf_bytes: bytes) -> List[str]: |
| pages: List[str] = [] |
|
|
| try: |
| reader = PdfReader(BytesIO(pdf_bytes)) |
| except DependencyError: |
| |
| return [] |
|
|
| |
| if reader.is_encrypted: |
| try: |
| reader.decrypt("") |
| except Exception: |
| return [] |
|
|
| for page in reader.pages: |
| try: |
| text = page.extract_text() or "" |
| except Exception: |
| text = "" |
|
|
| text = re.sub(r"\s+\n", "\n", text) |
| text = re.sub(r"\n\s+", "\n", text) |
| text = re.sub(r"[ \t]+", " ", text) |
| pages.append(text.strip()) |
|
|
| return pages |
|
|
|
|
| ARABIC_RANGE = re.compile(r"[\u0600-\u06FF]") |
| LATIN_RANGE = re.compile(r"[A-Za-z]") |
|
|
|
|
| def is_text_usable(text: str, min_len: int = 300) -> bool: |
| if not text or len(text) < min_len: |
| return False |
|
|
| total_len = len(text) |
|
|
| arabic_chars = len(ARABIC_RANGE.findall(text)) |
| latin_chars = len(LATIN_RANGE.findall(text)) |
| spaces = text.count(" ") |
|
|
| |
| readable_ratio = (arabic_chars + latin_chars) / total_len |
|
|
| |
| |
| |
| space_ratio = spaces / total_len |
| if space_ratio < 0.02: |
| return False |
|
|
| |
| |
| |
| words = [w for w in text.split() if len(w) > 1] |
| if len(words) < 20: |
| return False |
|
|
| avg_word_len = sum(len(w) for w in words) / len(words) |
|
|
| |
| if avg_word_len > 12: |
| return False |
|
|
| |
| |
| |
| long_words = [w for w in words if len(w) > 15] |
| long_ratio = len(long_words) / len(words) |
|
|
| if long_ratio > 0.25: |
| return False |
|
|
| |
| |
| |
| if arabic_chars > 50: |
| return True |
|
|
| if latin_chars > 100: |
| return True |
|
|
| if readable_ratio < 0.2: |
| return False |
|
|
| return True |
|
|
