| from langchain_community.document_loaders import PyPDFLoader | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from src.config import KNOWLEDGE_BASE_DIR, CHUNK_SIZE, CHUNK_OVERLAP | |
| import os | |
| def load_pdfs(): | |
| documents = [] | |
| pdf_files = [f for f in os.listdir(KNOWLEDGE_BASE_DIR) if f.endswith(".pdf")] | |
| if not pdf_files: | |
| print("No PDFs found in knowledge_base/ folder") | |
| return [] | |
| for pdf_file in pdf_files: | |
| path = os.path.join(KNOWLEDGE_BASE_DIR, pdf_file) | |
| print(f"Loading: {pdf_file}") | |
| loader = PyPDFLoader(path) | |
| documents.extend(loader.load()) | |
| print(f"\nTotal pages loaded: {len(documents)}") | |
| return documents | |
| def chunk_documents(documents): | |
| splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=CHUNK_SIZE, | |
| chunk_overlap=CHUNK_OVERLAP, | |
| separators=["\n\n", "\n", ".", " "] | |
| ) | |
| chunks = splitter.split_documents(documents) | |
| print(f"Total chunks created: {len(chunks)}") | |
| return chunks | |
| def ingest(): | |
| documents = load_pdfs() | |
| chunks = chunk_documents(documents) | |
| return chunks |