acpcchatbotadmin's picture
Upload 40 files
481dfe6 verified
Raw History Blame Contribute Delete
2.72 kB
"""
Document ingestion pipeline — migrated from server_acpc.py.
Chunks documents and stores them in the Qdrant collection.
"""
import os
from pathlib import Path
from langchain_community.document_loaders import PyPDFLoader, DirectoryLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_core.documents import Document
from app.retrieval.qdrant_store import get_qdrant_client, get_vector_store, get_embeddings
from app.config import settings
def _get_metadata(file_path: str) -> dict:
"""Generate metadata matching the existing Qdrant collection schema."""
base_name = Path(file_path).stem
url = f"https://acpc.gujarat.gov.in/assets/uploads/media-uploader/{base_name}.pdf"
return {
"file_name": Path(file_path).name,
"url": url,
"page_id": url,
}
def _load_file(file_path: str) -> list[Document]:
"""Load a single file (PDF or text)."""
path = Path(file_path)
if path.suffix.lower() == ".pdf":
loader = PyPDFLoader(str(path))
docs = loader.load()
else:
from langchain_community.document_loaders import TextLoader
loader = TextLoader(str(path), encoding="utf-8")
docs = loader.load()
# Inject file-level metadata
meta = _get_metadata(file_path)
for doc in docs:
doc.metadata.update(meta)
return docs
def ingest_file(file_path: str) -> int:
"""Ingest a single file into Qdrant. Returns number of chunks created."""
docs = _load_file(file_path)
splitter = RecursiveCharacterTextSplitter(
chunk_size=settings.chunk_size,
chunk_overlap=settings.chunk_overlap,
)
chunks = splitter.split_documents(docs)
vector_store = get_vector_store()
vector_store.add_documents(chunks)
return len(chunks)
def ingest_directory(directory_path: str, recursive: bool = True) -> dict:
"""Ingest all PDFs from a directory. Returns summary."""
directory = Path(directory_path)
pattern = "**/*.pdf" if recursive else "*.pdf"
pdf_files = list(directory.glob(pattern))
total_chunks = 0
for pdf_file in pdf_files:
total_chunks += ingest_file(str(pdf_file))
return {"total_documents": len(pdf_files), "total_chunks": total_chunks}
def reset_collection() -> None:
"""Delete and recreate the Qdrant collection."""
client = get_qdrant_client()
client.delete_collection(settings.collection_name)
# Re-initialize the vector store (it recreates the collection on first use)
global _vector_store_ref
import app.retrieval.qdrant_store as qs
qs._vector_store = None # Force re-init on next access