File size: 2,727 Bytes
62e4c13 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | import uuid
import logging
from datetime import datetime, timezone
from typing import BinaryIO
import pdfplumber
from app.config import get_settings
logger = logging.getLogger(__name__)
settings = get_settings()
def split_text(text: str, chunk_size: int, chunk_overlap: int) -> list[str]:
"""Simple recursive text splitter — no langchain needed."""
if len(text) <= chunk_size:
return [text]
chunks = []
start = 0
while start < len(text):
end = start + chunk_size
# Try to break at a sentence or newline boundary
if end < len(text):
for sep in ['\n\n', '\n', '. ', ' ']:
idx = text.rfind(sep, start, end)
if idx != -1:
end = idx + len(sep)
break
chunks.append(text[start:end].strip())
start = end - chunk_overlap
return [c for c in chunks if c]
class PDFProcessor:
def extract_text(self, file: BinaryIO) -> tuple[list[dict], int]:
pages = []
with pdfplumber.open(file) as pdf:
page_count = len(pdf.pages)
for i, page in enumerate(pdf.pages, start=1):
text = (page.extract_text() or "").strip()
if text:
pages.append({"page": i, "text": text})
return pages, page_count
def create_chunks(self, pages: list[dict], doc_id: str, filename: str) -> list[dict]:
chunks = []
chunk_index = 0
for page_data in pages:
for chunk_text in split_text(page_data["text"], settings.chunk_size, settings.chunk_overlap):
if len(chunk_text.strip()) < 30:
continue
chunks.append({
"id": f"{doc_id}__p{page_data['page']}__c{chunk_index}",
"text": chunk_text,
"metadata": {
"doc_id": doc_id,
"filename": filename,
"page": page_data["page"],
"chunk_index": chunk_index,
"uploaded_at": datetime.now(timezone.utc).isoformat(),
},
})
chunk_index += 1
return chunks
def process(self, file: BinaryIO, filename: str) -> tuple[list[dict], str, int, int]:
doc_id = str(uuid.uuid4())
pages, page_count = self.extract_text(file)
if not pages:
raise ValueError("No extractable text found in the PDF.")
chunks = self.create_chunks(pages, doc_id, filename)
if not chunks:
raise ValueError("PDF text could not be split into usable chunks.")
return chunks, doc_id, page_count, len(chunks)
|