qshub / pdf_engine /pdf_loader.py
basel00omar's picture
Fix OOM crashes on Render proxy by parsing PDFs using mmap from disk instead of massive byte duplication in memory
e73778d
Raw History Blame Contribute Delete
4.49 kB
"""
تحميل ومعالجة عدة ملفات PDF
"""
import fitz
import numpy as np
from PIL import Image
import io
from typing import List
def load_pdf_pages(pdf_bytes: bytes, dpi: int = 150) -> List[np.ndarray]:
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
pages = []
# Dynamic DPI reduction to prevent Out-Of-Memory (OOM) on 512MB free servers
page_count = len(doc)
if page_count > 10:
dpi = min(dpi, 50)
elif page_count > 3:
dpi = min(dpi, 72)
else:
dpi = min(dpi, 100)
for page_num in range(page_count):
page = doc[page_num]
mat = fitz.Matrix(dpi / 72, dpi / 72)
pix = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB, alpha=True)
img = Image.open(io.BytesIO(pix.tobytes("png"))).convert("RGBA")
bg = Image.new("RGBA", img.size, (255, 255, 255, 255))
bg.paste(img, (0, 0), img)
arr = np.array(bg.convert("RGB"))[:, :, ::-1].copy()
pages.append(arr)
doc.close()
return pages
def fast_save_pdf_pages(pdf_source: bytes | str, project_id: int, prefix: str, start_idx: int) -> int:
"""
High-speed PDF image extractor using pure C++ PyMuPDF rendering.
Bypasses PIL/Numpy entirely to avoid memory leaks and strided byte exceptions.
Renders directly to compressed JPEG (quality=95, max 1600px).
Returns the number of pages processed.
"""
from utils.storage import save_raw_image_to_cache
import gc
if isinstance(pdf_source, str):
doc = fitz.open(pdf_source)
else:
doc = fitz.open(stream=pdf_source, filetype="pdf")
page_count = len(doc)
for page_num in range(page_count):
try:
page = doc[page_num]
rect = page.rect
w, h = rect.width, rect.height
# Calculate C++ rendering scale matrix to cap max dimension at 1200px (fast, crystal clear)
max_dim = max(w, h)
if max_dim > 0:
scale = min(1200.0 / max_dim, 1.5)
else:
scale = 1.0
mat = fitz.Matrix(scale, scale)
try:
pix = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB, alpha=False)
except Exception:
pix = page.get_pixmap(colorspace=fitz.csRGB)
# High-speed native C++ JPEG encoder (quality=90, ~150KB per page)
jpg_bytes = pix.tobytes("jpg", jpg_quality=90)
pix = None # Free PyMuPDF C memory immediately
page = None
global_idx = start_idx + page_num
filename = f"{prefix}_page_{global_idx}.jpg"
save_raw_image_to_cache(project_id, filename, jpg_bytes, format="JPEG")
except Exception as page_err:
print(f"[fast_save_pdf_pages] Warning: failed page {page_num}: {page_err}")
# Periodic memory cleanup to prevent cloud container OOM SIGKILL (502 Gateway error)
if page_num % 5 == 0:
gc.collect()
doc.close()
doc = None
gc.collect()
return page_count
def extract_page_text(pdf_source: bytes | str) -> List[str]:
"""يستخرج النص المباشر من كل صفحة PDF بدون OCR — أسرع للتصنيف الأولي"""
if isinstance(pdf_source, str):
doc = fitz.open(pdf_source)
else:
doc = fitz.open(stream=pdf_source, filetype="pdf")
texts = []
for page in doc:
try:
texts.append(page.get_text().strip())
except Exception:
texts.append("")
doc.close()
return texts
def classify_page_by_text(page_text: str, page_image: np.ndarray = None) -> str:
"""يصنف الصفحة بناءً على النص — يرجع مفتاح PAGE_ITEMS_MAP"""
from pdf_engine.smart_classifier import PAGE_ITEMS_MAP
text_lower = page_text.lower()
scores = {}
for page_type, config in PAGE_ITEMS_MAP.items():
score = sum(1 for kw in config["drawing_keywords"] if kw.lower() in text_lower)
if page_type == "schedules" and score > 0:
score += 5
scores[page_type] = score
best = max(scores, key=scores.get)
return best if scores[best] > 0 else "unknown"
def page_to_pil(page_array: np.ndarray) -> Image.Image:
return Image.fromarray(page_array[:, :, ::-1].copy())
def get_page_count(pdf_bytes: bytes) -> int:
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
count = len(doc)
doc.close()
return count