File size: 4,612 Bytes
f837c69 bd2a75a 6005177 f837c69 6005177 f837c69 760e17f bd2a75a 760e17f 6005177 f837c69 bd2a75a 760e17f 57cfb76 c0b7721 760e17f c0b7721 760e17f c0b7721 760e17f c0b7721 760e17f 57cfb76 c0b7721 760e17f c0b7721 760e17f bd2a75a 760e17f 72a6115 57cfb76 760e17f 72a6115 760e17f 72a6115 760e17f 57cfb76 760e17f 57cfb76 760e17f 57cfb76 760e17f c0b7721 760e17f c0b7721 72a6115 f837c69 57cfb76 f837c69 57cfb76 f837c69 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | """Module handling PDF document processing and content extraction."""
import os
import re
from pathlib import Path
import fitz
from langfuse.decorators import langfuse_context, observe
from ..telemetry import Logger
from .images_processing import extract_images, extract_vector
LOGGER = Logger.get_logger()
@observe(name="โ๏ธ extract_pdf_content")
def extract_pdf_content(pdf_path: str) -> tuple[str, list[dict], dict]:
"""Extract text and images from a PDF file.
Args:
pdf_path (str): Path to the PDF file.
Returns:
tuple: A 3-element tuple:
- str: Extracted text from the PDF.
- list: List of dictionaries containing image metadata.
- dict: Metadata about the PDF, including the number of images and the PDF path.
"""
doc = fitz.open(pdf_path)
pdf = os.path.splitext(os.path.basename(pdf_path))[0]
text = ""
figs = []
for page_num, page in enumerate(doc):
# Parse the text
text = " ".join([text, page.get_text().strip()])
# Extract images
figs += extract_images(pdf, doc, page, page_num)
# Extract vector graphics
figs += extract_vector(pdf, page, page_num)
# Format the metadata
metas = {"num_images": len(figs), "pdf_path": pdf_path}
langfuse_context.update_current_observation(output={"figs": figs, "metas": metas})
return text, figs, metas
@observe(name="๐ process_documents")
def process_documents(pdf_files) -> tuple[list[str], list[dict], list[str]]:
"""Process a list of PDF files to extract text and images metadata.
Args:
pdf_files (list): List of paths to PDF files.
Returns:
tuple: A 2-element tuple:
- list[str]: List of extracted text from each PDF.
- list[dict]: List of metadata dictionaries for each PDF.
"""
documents = []
metadatas = []
failed = []
for pdf_path in pdf_files:
try:
text, imgs, metas = extract_pdf_content(pdf_path)
except Exception as e:
LOGGER.error("โ Error processing %s", pdf_path, exc_info=e)
langfuse_context.update_current_observation(
output={
"output.images_passage": None,
"output.num_images": None,
"output.pdf_path": pdf_path,
}
)
failed.append(pdf_path)
continue
documents.append(text)
# Format images
images_info = []
for _, img in enumerate(imgs, start=1):
caption = img.get("caption")
caption_str = str(caption) if caption is not None else ""
full_path = f"gfx/{img['name']}"
cleaned_caption = re.sub(
r"^(fig(?:ure)?\.?\s*\d+\.\s*)",
"",
caption_str,
flags=re.IGNORECASE,
).strip()
caption = cleaned_caption if cleaned_caption else "None"
images_info.append(
(
f'{{"path": "{full_path}", '
f'"caption": "{caption}", '
f'"orientation": "{img["ratio"]}"}}'
)
)
images_passage = "\n".join(images_info)
# span.set_attribute("output.images_passage", images_passage)
# Format metadata
fixed_metadata = {
"num_images": metas.get("num_images"),
"pdf_path": metas.get("pdf_path"),
"images_passage": images_passage,
}
langfuse_context.update_current_observation(
output={
"output.images_passage": images_passage,
"output.num_images": metas.get("num_images"),
"output.pdf_path": metas.get("pdf_path"),
}
)
metadatas.append(fixed_metadata)
return documents, metadatas, failed
@observe(name="๐๏ธ delete_uploaded_files")
def delete_uploaded_files(uploaded_files) -> None:
"""Delete uploaded files after processing.
Args:
uploaded_files (list): List of file paths to be deleted.
"""
for file in uploaded_files:
try:
path = Path(file)
if path.exists():
path.unlink() # Delete the file
LOGGER.info(f"Deleted file: {file}")
else:
LOGGER.warning(f"File not found for deletion: {file}")
LOGGER.info("๐งน Cleaned up uploaded files.")
except (OSError, PermissionError) as e:
LOGGER.warning(f"Failed to delete {file}", exc_info=e)
|