""" Document ingestion pipeline — migrated from server_acpc.py. Chunks documents and stores them in the Qdrant collection. """ import os from pathlib import Path from langchain_community.document_loaders import PyPDFLoader, DirectoryLoader from langchain_text_splitters import RecursiveCharacterTextSplitter from langchain_core.documents import Document from app.retrieval.qdrant_store import get_qdrant_client, get_vector_store, get_embeddings from app.config import settings def _get_metadata(file_path: str) -> dict: """Generate metadata matching the existing Qdrant collection schema.""" base_name = Path(file_path).stem url = f"https://acpc.gujarat.gov.in/assets/uploads/media-uploader/{base_name}.pdf" return { "file_name": Path(file_path).name, "url": url, "page_id": url, } def _load_file(file_path: str) -> list[Document]: """Load a single file (PDF or text).""" path = Path(file_path) if path.suffix.lower() == ".pdf": loader = PyPDFLoader(str(path)) docs = loader.load() else: from langchain_community.document_loaders import TextLoader loader = TextLoader(str(path), encoding="utf-8") docs = loader.load() # Inject file-level metadata meta = _get_metadata(file_path) for doc in docs: doc.metadata.update(meta) return docs def ingest_file(file_path: str) -> int: """Ingest a single file into Qdrant. Returns number of chunks created.""" docs = _load_file(file_path) splitter = RecursiveCharacterTextSplitter( chunk_size=settings.chunk_size, chunk_overlap=settings.chunk_overlap, ) chunks = splitter.split_documents(docs) vector_store = get_vector_store() vector_store.add_documents(chunks) return len(chunks) def ingest_directory(directory_path: str, recursive: bool = True) -> dict: """Ingest all PDFs from a directory. Returns summary.""" directory = Path(directory_path) pattern = "**/*.pdf" if recursive else "*.pdf" pdf_files = list(directory.glob(pattern)) total_chunks = 0 for pdf_file in pdf_files: total_chunks += ingest_file(str(pdf_file)) return {"total_documents": len(pdf_files), "total_chunks": total_chunks} def reset_collection() -> None: """Delete and recreate the Qdrant collection.""" client = get_qdrant_client() client.delete_collection(settings.collection_name) # Re-initialize the vector store (it recreates the collection on first use) global _vector_store_ref import app.retrieval.qdrant_store as qs qs._vector_store = None # Force re-init on next access