Fix OOM crashes on Render proxy by parsing PDFs using mmap from disk instead of massive byte duplication in memory
e73778d Download pdf_engine/pdf_loader.py from Basel0/qshub: direct link, hf CLI and curl.
- Browser
- Download file 4.49 kB
-
https://huggingface.co/spaces/Basel0/qshub/resolve/main/pdf_engine/pdf_loader.py
- Command line
-
hf download hf://spaces/Basel0/qshub/pdf_engine/pdf_loader.py
-
curl -L -o pdf_loader.py https://huggingface.co/spaces/Basel0/qshub/resolve/main/pdf_engine/pdf_loader.py
4.49 kB
| """ | |
| تحميل ومعالجة عدة ملفات PDF | |
| """ | |
| import fitz | |
| import numpy as np | |
| from PIL import Image | |
| import io | |
| from typing import List | |
| def load_pdf_pages(pdf_bytes: bytes, dpi: int = 150) -> List[np.ndarray]: | |
| doc = fitz.open(stream=pdf_bytes, filetype="pdf") | |
| pages = [] | |
| # Dynamic DPI reduction to prevent Out-Of-Memory (OOM) on 512MB free servers | |
| page_count = len(doc) | |
| if page_count > 10: | |
| dpi = min(dpi, 50) | |
| elif page_count > 3: | |
| dpi = min(dpi, 72) | |
| else: | |
| dpi = min(dpi, 100) | |
| for page_num in range(page_count): | |
| page = doc[page_num] | |
| mat = fitz.Matrix(dpi / 72, dpi / 72) | |
| pix = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB, alpha=True) | |
| img = Image.open(io.BytesIO(pix.tobytes("png"))).convert("RGBA") | |
| bg = Image.new("RGBA", img.size, (255, 255, 255, 255)) | |
| bg.paste(img, (0, 0), img) | |
| arr = np.array(bg.convert("RGB"))[:, :, ::-1].copy() | |
| pages.append(arr) | |
| doc.close() | |
| return pages | |
| def fast_save_pdf_pages(pdf_source: bytes | str, project_id: int, prefix: str, start_idx: int) -> int: | |
| """ | |
| High-speed PDF image extractor using pure C++ PyMuPDF rendering. | |
| Bypasses PIL/Numpy entirely to avoid memory leaks and strided byte exceptions. | |
| Renders directly to compressed JPEG (quality=95, max 1600px). | |
| Returns the number of pages processed. | |
| """ | |
| from utils.storage import save_raw_image_to_cache | |
| import gc | |
| if isinstance(pdf_source, str): | |
| doc = fitz.open(pdf_source) | |
| else: | |
| doc = fitz.open(stream=pdf_source, filetype="pdf") | |
| page_count = len(doc) | |
| for page_num in range(page_count): | |
| try: | |
| page = doc[page_num] | |
| rect = page.rect | |
| w, h = rect.width, rect.height | |
| # Calculate C++ rendering scale matrix to cap max dimension at 1200px (fast, crystal clear) | |
| max_dim = max(w, h) | |
| if max_dim > 0: | |
| scale = min(1200.0 / max_dim, 1.5) | |
| else: | |
| scale = 1.0 | |
| mat = fitz.Matrix(scale, scale) | |
| try: | |
| pix = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB, alpha=False) | |
| except Exception: | |
| pix = page.get_pixmap(colorspace=fitz.csRGB) | |
| # High-speed native C++ JPEG encoder (quality=90, ~150KB per page) | |
| jpg_bytes = pix.tobytes("jpg", jpg_quality=90) | |
| pix = None # Free PyMuPDF C memory immediately | |
| page = None | |
| global_idx = start_idx + page_num | |
| filename = f"{prefix}_page_{global_idx}.jpg" | |
| save_raw_image_to_cache(project_id, filename, jpg_bytes, format="JPEG") | |
| except Exception as page_err: | |
| print(f"[fast_save_pdf_pages] Warning: failed page {page_num}: {page_err}") | |
| # Periodic memory cleanup to prevent cloud container OOM SIGKILL (502 Gateway error) | |
| if page_num % 5 == 0: | |
| gc.collect() | |
| doc.close() | |
| doc = None | |
| gc.collect() | |
| return page_count | |
| def extract_page_text(pdf_source: bytes | str) -> List[str]: | |
| """يستخرج النص المباشر من كل صفحة PDF بدون OCR — أسرع للتصنيف الأولي""" | |
| if isinstance(pdf_source, str): | |
| doc = fitz.open(pdf_source) | |
| else: | |
| doc = fitz.open(stream=pdf_source, filetype="pdf") | |
| texts = [] | |
| for page in doc: | |
| try: | |
| texts.append(page.get_text().strip()) | |
| except Exception: | |
| texts.append("") | |
| doc.close() | |
| return texts | |
| def classify_page_by_text(page_text: str, page_image: np.ndarray = None) -> str: | |
| """يصنف الصفحة بناءً على النص — يرجع مفتاح PAGE_ITEMS_MAP""" | |
| from pdf_engine.smart_classifier import PAGE_ITEMS_MAP | |
| text_lower = page_text.lower() | |
| scores = {} | |
| for page_type, config in PAGE_ITEMS_MAP.items(): | |
| score = sum(1 for kw in config["drawing_keywords"] if kw.lower() in text_lower) | |
| if page_type == "schedules" and score > 0: | |
| score += 5 | |
| scores[page_type] = score | |
| best = max(scores, key=scores.get) | |
| return best if scores[best] > 0 else "unknown" | |
| def page_to_pil(page_array: np.ndarray) -> Image.Image: | |
| return Image.fromarray(page_array[:, :, ::-1].copy()) | |
| def get_page_count(pdf_bytes: bytes) -> int: | |
| doc = fitz.open(stream=pdf_bytes, filetype="pdf") | |
| count = len(doc) | |
| doc.close() | |
| return count | |