ragntex / src /processing /document_processing.py
eranmi's picture
Merge branch 'dev' of https://github.com/AnnaErsh/RAGnTeX into dev
e1f4206
Raw History Blame Contribute Delete
4.61 kB
"""Module handling PDF document processing and content extraction."""
import os
import re
from pathlib import Path
import fitz
from langfuse.decorators import langfuse_context, observe
from ..telemetry import Logger
from .images_processing import extract_images, extract_vector
LOGGER = Logger.get_logger()
@observe(name="โš™๏ธ extract_pdf_content")
def extract_pdf_content(pdf_path: str) -> tuple[str, list[dict], dict]:
"""Extract text and images from a PDF file.
Args:
pdf_path (str): Path to the PDF file.
Returns:
tuple: A 3-element tuple:
- str: Extracted text from the PDF.
- list: List of dictionaries containing image metadata.
- dict: Metadata about the PDF, including the number of images and the PDF path.
"""
doc = fitz.open(pdf_path)
pdf = os.path.splitext(os.path.basename(pdf_path))[0]
text = ""
figs = []
for page_num, page in enumerate(doc):
# Parse the text
text = " ".join([text, page.get_text().strip()])
# Extract images
figs += extract_images(pdf, doc, page, page_num)
# Extract vector graphics
figs += extract_vector(pdf, page, page_num)
# Format the metadata
metas = {"num_images": len(figs), "pdf_path": pdf_path}
langfuse_context.update_current_observation(output={"figs": figs, "metas": metas})
return text, figs, metas
@observe(name="๐Ÿ“Š process_documents")
def process_documents(pdf_files) -> tuple[list[str], list[dict], list[str]]:
"""Process a list of PDF files to extract text and images metadata.
Args:
pdf_files (list): List of paths to PDF files.
Returns:
tuple: A 2-element tuple:
- list[str]: List of extracted text from each PDF.
- list[dict]: List of metadata dictionaries for each PDF.
"""
documents = []
metadatas = []
failed = []
for pdf_path in pdf_files:
try:
text, imgs, metas = extract_pdf_content(pdf_path)
except Exception as e:
LOGGER.error("โŒ Error processing %s", pdf_path, exc_info=e)
langfuse_context.update_current_observation(
output={
"output.images_passage": None,
"output.num_images": None,
"output.pdf_path": pdf_path,
}
)
failed.append(pdf_path)
continue
documents.append(text)
# Format images
images_info = []
for _, img in enumerate(imgs, start=1):
caption = img.get("caption")
caption_str = str(caption) if caption is not None else ""
full_path = f"gfx/{img['name']}"
cleaned_caption = re.sub(
r"^(fig(?:ure)?\.?\s*\d+\.\s*)",
"",
caption_str,
flags=re.IGNORECASE,
).strip()
caption = cleaned_caption if cleaned_caption else "None"
images_info.append(
(
f'{{"path": "{full_path}", '
f'"caption": "{caption}", '
f'"orientation": "{img["ratio"]}"}}'
)
)
images_passage = "\n".join(images_info)
# span.set_attribute("output.images_passage", images_passage)
# Format metadata
fixed_metadata = {
"num_images": metas.get("num_images"),
"pdf_path": metas.get("pdf_path"),
"images_passage": images_passage,
}
langfuse_context.update_current_observation(
output={
"output.images_passage": images_passage,
"output.num_images": metas.get("num_images"),
"output.pdf_path": metas.get("pdf_path"),
}
)
metadatas.append(fixed_metadata)
return documents, metadatas, failed
@observe(name="๐Ÿ—‘๏ธ delete_uploaded_files")
def delete_uploaded_files(uploaded_files) -> None:
"""Delete uploaded files after processing.
Args:
uploaded_files (list): List of file paths to be deleted.
"""
for file in uploaded_files:
try:
path = Path(file)
if path.exists():
path.unlink() # Delete the file
LOGGER.info(f"Deleted file: {file}")
else:
LOGGER.warning(f"File not found for deletion: {file}")
LOGGER.info("๐Ÿงน Cleaned up uploaded files.")
except (OSError, PermissionError) as e:
LOGGER.warning(f"Failed to delete {file}", exc_info=e)