Download app/services/pdf_processor.py from abdullahhshafique/pdf-knowledge-assistant: direct link, hf CLI and curl.
- Browser
- Download file 2.73 kB
-
https://huggingface.co/spaces/abdullahhshafique/pdf-knowledge-assistant/resolve/main/app/services/pdf_processor.py
- Command line
-
hf download hf://spaces/abdullahhshafique/pdf-knowledge-assistant/app/services/pdf_processor.py
-
curl -L -o pdf_processor.py https://huggingface.co/spaces/abdullahhshafique/pdf-knowledge-assistant/resolve/main/app/services/pdf_processor.py
2.73 kB
| import uuid | |
| import logging | |
| from datetime import datetime, timezone | |
| from typing import BinaryIO | |
| import pdfplumber | |
| from app.config import get_settings | |
| logger = logging.getLogger(__name__) | |
| settings = get_settings() | |
| def split_text(text: str, chunk_size: int, chunk_overlap: int) -> list[str]: | |
| """Simple recursive text splitter — no langchain needed.""" | |
| if len(text) <= chunk_size: | |
| return [text] | |
| chunks = [] | |
| start = 0 | |
| while start < len(text): | |
| end = start + chunk_size | |
| # Try to break at a sentence or newline boundary | |
| if end < len(text): | |
| for sep in ['\n\n', '\n', '. ', ' ']: | |
| idx = text.rfind(sep, start, end) | |
| if idx != -1: | |
| end = idx + len(sep) | |
| break | |
| chunks.append(text[start:end].strip()) | |
| start = end - chunk_overlap | |
| return [c for c in chunks if c] | |
| class PDFProcessor: | |
| def extract_text(self, file: BinaryIO) -> tuple[list[dict], int]: | |
| pages = [] | |
| with pdfplumber.open(file) as pdf: | |
| page_count = len(pdf.pages) | |
| for i, page in enumerate(pdf.pages, start=1): | |
| text = (page.extract_text() or "").strip() | |
| if text: | |
| pages.append({"page": i, "text": text}) | |
| return pages, page_count | |
| def create_chunks(self, pages: list[dict], doc_id: str, filename: str) -> list[dict]: | |
| chunks = [] | |
| chunk_index = 0 | |
| for page_data in pages: | |
| for chunk_text in split_text(page_data["text"], settings.chunk_size, settings.chunk_overlap): | |
| if len(chunk_text.strip()) < 30: | |
| continue | |
| chunks.append({ | |
| "id": f"{doc_id}__p{page_data['page']}__c{chunk_index}", | |
| "text": chunk_text, | |
| "metadata": { | |
| "doc_id": doc_id, | |
| "filename": filename, | |
| "page": page_data["page"], | |
| "chunk_index": chunk_index, | |
| "uploaded_at": datetime.now(timezone.utc).isoformat(), | |
| }, | |
| }) | |
| chunk_index += 1 | |
| return chunks | |
| def process(self, file: BinaryIO, filename: str) -> tuple[list[dict], str, int, int]: | |
| doc_id = str(uuid.uuid4()) | |
| pages, page_count = self.extract_text(file) | |
| if not pages: | |
| raise ValueError("No extractable text found in the PDF.") | |
| chunks = self.create_chunks(pages, doc_id, filename) | |
| if not chunks: | |
| raise ValueError("PDF text could not be split into usable chunks.") | |
| return chunks, doc_id, page_count, len(chunks) | |