File size: 4,494 Bytes
1b4b6d2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ad8953c
 
 
 
 
1b4b6d2
 
 
 
 
e73778d
9394ed3
2580d29
 
 
9394ed3
 
 
0f3ab13
c9f5391
e73778d
 
 
 
9394ed3
0f3ab13
987a547
9b910ca
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c9f5391
 
 
0f3ab13
9394ed3
c9f5391
 
9394ed3
 
 
e73778d
1b4b6d2
e73778d
 
 
 
1b4b6d2
 
9b910ca
 
 
 
1b4b6d2
 
 
 
 
 
 
 
 
 
 
 
3f2a193
 
1b4b6d2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
"""
تحميل ومعالجة عدة ملفات PDF
"""
import fitz
import numpy as np
from PIL import Image
import io
from typing import List


def load_pdf_pages(pdf_bytes: bytes, dpi: int = 150) -> List[np.ndarray]:
    doc = fitz.open(stream=pdf_bytes, filetype="pdf")
    pages = []
    
    # Dynamic DPI reduction to prevent Out-Of-Memory (OOM) on 512MB free servers
    page_count = len(doc)
    if page_count > 10:
        dpi = min(dpi, 50)
    elif page_count > 3:
        dpi = min(dpi, 72)
    else:
        dpi = min(dpi, 100)
        
    for page_num in range(page_count):
        page = doc[page_num]
        mat  = fitz.Matrix(dpi / 72, dpi / 72)
        pix  = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB, alpha=True)
        img  = Image.open(io.BytesIO(pix.tobytes("png"))).convert("RGBA")
        bg = Image.new("RGBA", img.size, (255, 255, 255, 255))
        bg.paste(img, (0, 0), img)
        arr  = np.array(bg.convert("RGB"))[:, :, ::-1].copy()
        pages.append(arr)
    doc.close()
    return pages


def fast_save_pdf_pages(pdf_source: bytes | str, project_id: int, prefix: str, start_idx: int) -> int:
    """
    High-speed PDF image extractor using pure C++ PyMuPDF rendering.
    Bypasses PIL/Numpy entirely to avoid memory leaks and strided byte exceptions.
    Renders directly to compressed JPEG (quality=95, max 1600px).
    Returns the number of pages processed.
    """
    from utils.storage import save_raw_image_to_cache

    import gc
    if isinstance(pdf_source, str):
        doc = fitz.open(pdf_source)
    else:
        doc = fitz.open(stream=pdf_source, filetype="pdf")
    page_count = len(doc)

    for page_num in range(page_count):
        try:
            page = doc[page_num]
            rect = page.rect
            w, h = rect.width, rect.height

            # Calculate C++ rendering scale matrix to cap max dimension at 1200px (fast, crystal clear)
            max_dim = max(w, h)
            if max_dim > 0:
                scale = min(1200.0 / max_dim, 1.5)
            else:
                scale = 1.0

            mat = fitz.Matrix(scale, scale)
            try:
                pix = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB, alpha=False)
            except Exception:
                pix = page.get_pixmap(colorspace=fitz.csRGB)

            # High-speed native C++ JPEG encoder (quality=90, ~150KB per page)
            jpg_bytes = pix.tobytes("jpg", jpg_quality=90)
            pix = None  # Free PyMuPDF C memory immediately
            page = None

            global_idx = start_idx + page_num
            filename = f"{prefix}_page_{global_idx}.jpg"
            save_raw_image_to_cache(project_id, filename, jpg_bytes, format="JPEG")
        except Exception as page_err:
            print(f"[fast_save_pdf_pages] Warning: failed page {page_num}: {page_err}")

        # Periodic memory cleanup to prevent cloud container OOM SIGKILL (502 Gateway error)
        if page_num % 5 == 0:
            gc.collect()

    doc.close()
    doc = None
    gc.collect()
    return page_count


def extract_page_text(pdf_source: bytes | str) -> List[str]:
    """يستخرج النص المباشر من كل صفحة PDF بدون OCR — أسرع للتصنيف الأولي"""
    if isinstance(pdf_source, str):
        doc = fitz.open(pdf_source)
    else:
        doc = fitz.open(stream=pdf_source, filetype="pdf")
    texts = []
    for page in doc:
        try:
            texts.append(page.get_text().strip())
        except Exception:
            texts.append("")
    doc.close()
    return texts


def classify_page_by_text(page_text: str, page_image: np.ndarray = None) -> str:
    """يصنف الصفحة بناءً على النص — يرجع مفتاح PAGE_ITEMS_MAP"""
    from pdf_engine.smart_classifier import PAGE_ITEMS_MAP

    text_lower = page_text.lower()
    scores     = {}
    for page_type, config in PAGE_ITEMS_MAP.items():
        score = sum(1 for kw in config["drawing_keywords"] if kw.lower() in text_lower)
        if page_type == "schedules" and score > 0:
            score += 5
        scores[page_type] = score

    best = max(scores, key=scores.get)
    return best if scores[best] > 0 else "unknown"


def page_to_pil(page_array: np.ndarray) -> Image.Image:
    return Image.fromarray(page_array[:, :, ::-1].copy())


def get_page_count(pdf_bytes: bytes) -> int:
    doc   = fitz.open(stream=pdf_bytes, filetype="pdf")
    count = len(doc)
    doc.close()
    return count