File size: 4,612 Bytes
f837c69
 
bd2a75a
6005177
f837c69
6005177
f837c69
760e17f
bd2a75a
760e17f
6005177
f837c69
 
bd2a75a
 
760e17f
57cfb76
c0b7721
 
 
 
 
 
 
 
 
760e17f
 
c0b7721
760e17f
 
 
 
 
c0b7721
760e17f
 
c0b7721
760e17f
57cfb76
c0b7721
760e17f
 
 
c0b7721
760e17f
bd2a75a
 
760e17f
72a6115
57cfb76
 
 
 
 
 
 
 
760e17f
 
72a6115
760e17f
 
72a6115
 
 
 
 
 
 
 
 
 
 
 
 
760e17f
 
 
 
 
57cfb76
760e17f
 
57cfb76
760e17f
 
 
 
 
 
 
 
 
 
57cfb76
 
 
 
 
760e17f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c0b7721
760e17f
 
c0b7721
72a6115
f837c69
 
57cfb76
 
 
 
 
 
f837c69
 
 
 
 
 
 
 
 
57cfb76
f837c69
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
"""Module handling PDF document processing and content extraction."""

import os
import re
from pathlib import Path

import fitz
from langfuse.decorators import langfuse_context, observe

from ..telemetry import Logger
from .images_processing import extract_images, extract_vector

LOGGER = Logger.get_logger()


@observe(name="โš™๏ธ extract_pdf_content")
def extract_pdf_content(pdf_path: str) -> tuple[str, list[dict], dict]:
    """Extract text and images from a PDF file.
    Args:
        pdf_path (str): Path to the PDF file.
    Returns:
        tuple: A 3-element tuple:
            - str: Extracted text from the PDF.
            - list: List of dictionaries containing image metadata.
            - dict: Metadata about the PDF, including the number of images and the PDF path.
    """
    doc = fitz.open(pdf_path)
    pdf = os.path.splitext(os.path.basename(pdf_path))[0]

    text = ""
    figs = []
    for page_num, page in enumerate(doc):
        # Parse the text
        text = " ".join([text, page.get_text().strip()])

        # Extract images
        figs += extract_images(pdf, doc, page, page_num)

        # Extract vector graphics
        figs += extract_vector(pdf, page, page_num)

    # Format the metadata
    metas = {"num_images": len(figs), "pdf_path": pdf_path}
    langfuse_context.update_current_observation(output={"figs": figs, "metas": metas})

    return text, figs, metas


@observe(name="๐Ÿ“Š process_documents")
def process_documents(pdf_files) -> tuple[list[str], list[dict], list[str]]:
    """Process a list of PDF files to extract text and images metadata.
    Args:
        pdf_files (list): List of paths to PDF files.
    Returns:
        tuple: A 2-element tuple:
            - list[str]: List of extracted text from each PDF.
            - list[dict]: List of metadata dictionaries for each PDF.
    """
    documents = []
    metadatas = []
    failed = []

    for pdf_path in pdf_files:
        try:
            text, imgs, metas = extract_pdf_content(pdf_path)
        except Exception as e:
            LOGGER.error("โŒ Error processing %s", pdf_path, exc_info=e)
            langfuse_context.update_current_observation(
                output={
                    "output.images_passage": None,
                    "output.num_images": None,
                    "output.pdf_path": pdf_path,
                }
            )
            failed.append(pdf_path)
            continue

        documents.append(text)

        # Format images
        images_info = []
        for _, img in enumerate(imgs, start=1):
            caption = img.get("caption")
            caption_str = str(caption) if caption is not None else ""
            full_path = f"gfx/{img['name']}"

            cleaned_caption = re.sub(
                r"^(fig(?:ure)?\.?\s*\d+\.\s*)",
                "",
                caption_str,
                flags=re.IGNORECASE,
            ).strip()
            caption = cleaned_caption if cleaned_caption else "None"

            images_info.append(
                (
                    f'{{"path": "{full_path}", '
                    f'"caption": "{caption}", '
                    f'"orientation": "{img["ratio"]}"}}'
                )
            )

        images_passage = "\n".join(images_info)
        # span.set_attribute("output.images_passage", images_passage)

        # Format metadata
        fixed_metadata = {
            "num_images": metas.get("num_images"),
            "pdf_path": metas.get("pdf_path"),
            "images_passage": images_passage,
        }

        langfuse_context.update_current_observation(
            output={
                "output.images_passage": images_passage,
                "output.num_images": metas.get("num_images"),
                "output.pdf_path": metas.get("pdf_path"),
            }
        )
        metadatas.append(fixed_metadata)

    return documents, metadatas, failed


@observe(name="๐Ÿ—‘๏ธ delete_uploaded_files")
def delete_uploaded_files(uploaded_files) -> None:
    """Delete uploaded files after processing.
    Args:
        uploaded_files (list): List of file paths to be deleted.
    """
    for file in uploaded_files:
        try:
            path = Path(file)
            if path.exists():
                path.unlink()  # Delete the file
                LOGGER.info(f"Deleted file: {file}")
            else:
                LOGGER.warning(f"File not found for deletion: {file}")
            LOGGER.info("๐Ÿงน Cleaned up uploaded files.")
        except (OSError, PermissionError) as e:
            LOGGER.warning(f"Failed to delete {file}", exc_info=e)