pdf-knowledge-assistant / app /services /pdf_processor.py
abdullahhshafique's picture
feat: PDF Knowledge Assistant RAG backend
62e4c13
Raw History Blame Contribute Delete
2.73 kB
import uuid
import logging
from datetime import datetime, timezone
from typing import BinaryIO
import pdfplumber
from app.config import get_settings
logger = logging.getLogger(__name__)
settings = get_settings()
def split_text(text: str, chunk_size: int, chunk_overlap: int) -> list[str]:
"""Simple recursive text splitter — no langchain needed."""
if len(text) <= chunk_size:
return [text]
chunks = []
start = 0
while start < len(text):
end = start + chunk_size
# Try to break at a sentence or newline boundary
if end < len(text):
for sep in ['\n\n', '\n', '. ', ' ']:
idx = text.rfind(sep, start, end)
if idx != -1:
end = idx + len(sep)
break
chunks.append(text[start:end].strip())
start = end - chunk_overlap
return [c for c in chunks if c]
class PDFProcessor:
def extract_text(self, file: BinaryIO) -> tuple[list[dict], int]:
pages = []
with pdfplumber.open(file) as pdf:
page_count = len(pdf.pages)
for i, page in enumerate(pdf.pages, start=1):
text = (page.extract_text() or "").strip()
if text:
pages.append({"page": i, "text": text})
return pages, page_count
def create_chunks(self, pages: list[dict], doc_id: str, filename: str) -> list[dict]:
chunks = []
chunk_index = 0
for page_data in pages:
for chunk_text in split_text(page_data["text"], settings.chunk_size, settings.chunk_overlap):
if len(chunk_text.strip()) < 30:
continue
chunks.append({
"id": f"{doc_id}__p{page_data['page']}__c{chunk_index}",
"text": chunk_text,
"metadata": {
"doc_id": doc_id,
"filename": filename,
"page": page_data["page"],
"chunk_index": chunk_index,
"uploaded_at": datetime.now(timezone.utc).isoformat(),
},
})
chunk_index += 1
return chunks
def process(self, file: BinaryIO, filename: str) -> tuple[list[dict], str, int, int]:
doc_id = str(uuid.uuid4())
pages, page_count = self.extract_text(file)
if not pages:
raise ValueError("No extractable text found in the PDF.")
chunks = self.create_chunks(pages, doc_id, filename)
if not chunks:
raise ValueError("PDF text could not be split into usable chunks.")
return chunks, doc_id, page_count, len(chunks)