File size: 4,494 Bytes
1b4b6d2 ad8953c 1b4b6d2 e73778d 9394ed3 2580d29 9394ed3 0f3ab13 c9f5391 e73778d 9394ed3 0f3ab13 987a547 9b910ca c9f5391 0f3ab13 9394ed3 c9f5391 9394ed3 e73778d 1b4b6d2 e73778d 1b4b6d2 9b910ca 1b4b6d2 3f2a193 1b4b6d2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 | """
تحميل ومعالجة عدة ملفات PDF
"""
import fitz
import numpy as np
from PIL import Image
import io
from typing import List
def load_pdf_pages(pdf_bytes: bytes, dpi: int = 150) -> List[np.ndarray]:
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
pages = []
# Dynamic DPI reduction to prevent Out-Of-Memory (OOM) on 512MB free servers
page_count = len(doc)
if page_count > 10:
dpi = min(dpi, 50)
elif page_count > 3:
dpi = min(dpi, 72)
else:
dpi = min(dpi, 100)
for page_num in range(page_count):
page = doc[page_num]
mat = fitz.Matrix(dpi / 72, dpi / 72)
pix = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB, alpha=True)
img = Image.open(io.BytesIO(pix.tobytes("png"))).convert("RGBA")
bg = Image.new("RGBA", img.size, (255, 255, 255, 255))
bg.paste(img, (0, 0), img)
arr = np.array(bg.convert("RGB"))[:, :, ::-1].copy()
pages.append(arr)
doc.close()
return pages
def fast_save_pdf_pages(pdf_source: bytes | str, project_id: int, prefix: str, start_idx: int) -> int:
"""
High-speed PDF image extractor using pure C++ PyMuPDF rendering.
Bypasses PIL/Numpy entirely to avoid memory leaks and strided byte exceptions.
Renders directly to compressed JPEG (quality=95, max 1600px).
Returns the number of pages processed.
"""
from utils.storage import save_raw_image_to_cache
import gc
if isinstance(pdf_source, str):
doc = fitz.open(pdf_source)
else:
doc = fitz.open(stream=pdf_source, filetype="pdf")
page_count = len(doc)
for page_num in range(page_count):
try:
page = doc[page_num]
rect = page.rect
w, h = rect.width, rect.height
# Calculate C++ rendering scale matrix to cap max dimension at 1200px (fast, crystal clear)
max_dim = max(w, h)
if max_dim > 0:
scale = min(1200.0 / max_dim, 1.5)
else:
scale = 1.0
mat = fitz.Matrix(scale, scale)
try:
pix = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB, alpha=False)
except Exception:
pix = page.get_pixmap(colorspace=fitz.csRGB)
# High-speed native C++ JPEG encoder (quality=90, ~150KB per page)
jpg_bytes = pix.tobytes("jpg", jpg_quality=90)
pix = None # Free PyMuPDF C memory immediately
page = None
global_idx = start_idx + page_num
filename = f"{prefix}_page_{global_idx}.jpg"
save_raw_image_to_cache(project_id, filename, jpg_bytes, format="JPEG")
except Exception as page_err:
print(f"[fast_save_pdf_pages] Warning: failed page {page_num}: {page_err}")
# Periodic memory cleanup to prevent cloud container OOM SIGKILL (502 Gateway error)
if page_num % 5 == 0:
gc.collect()
doc.close()
doc = None
gc.collect()
return page_count
def extract_page_text(pdf_source: bytes | str) -> List[str]:
"""يستخرج النص المباشر من كل صفحة PDF بدون OCR — أسرع للتصنيف الأولي"""
if isinstance(pdf_source, str):
doc = fitz.open(pdf_source)
else:
doc = fitz.open(stream=pdf_source, filetype="pdf")
texts = []
for page in doc:
try:
texts.append(page.get_text().strip())
except Exception:
texts.append("")
doc.close()
return texts
def classify_page_by_text(page_text: str, page_image: np.ndarray = None) -> str:
"""يصنف الصفحة بناءً على النص — يرجع مفتاح PAGE_ITEMS_MAP"""
from pdf_engine.smart_classifier import PAGE_ITEMS_MAP
text_lower = page_text.lower()
scores = {}
for page_type, config in PAGE_ITEMS_MAP.items():
score = sum(1 for kw in config["drawing_keywords"] if kw.lower() in text_lower)
if page_type == "schedules" and score > 0:
score += 5
scores[page_type] = score
best = max(scores, key=scores.get)
return best if scores[best] > 0 else "unknown"
def page_to_pil(page_array: np.ndarray) -> Image.Image:
return Image.fromarray(page_array[:, :, ::-1].copy())
def get_page_count(pdf_bytes: bytes) -> int:
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
count = len(doc)
doc.close()
return count
|