Download app/retrieval/ingestion.py from acpcchatbotadmin/acpcchatbot_testing_first: direct link, hf CLI and curl.
- Browser
- Download file 2.72 kB
-
https://huggingface.co/spaces/acpcchatbotadmin/acpcchatbot_testing_first/resolve/main/app/retrieval/ingestion.py
- Command line
-
hf download hf://spaces/acpcchatbotadmin/acpcchatbot_testing_first/app/retrieval/ingestion.py
-
curl -L -o ingestion.py https://huggingface.co/spaces/acpcchatbotadmin/acpcchatbot_testing_first/resolve/main/app/retrieval/ingestion.py
2.72 kB
| """ | |
| Document ingestion pipeline — migrated from server_acpc.py. | |
| Chunks documents and stores them in the Qdrant collection. | |
| """ | |
| import os | |
| from pathlib import Path | |
| from langchain_community.document_loaders import PyPDFLoader, DirectoryLoader | |
| from langchain_text_splitters import RecursiveCharacterTextSplitter | |
| from langchain_core.documents import Document | |
| from app.retrieval.qdrant_store import get_qdrant_client, get_vector_store, get_embeddings | |
| from app.config import settings | |
| def _get_metadata(file_path: str) -> dict: | |
| """Generate metadata matching the existing Qdrant collection schema.""" | |
| base_name = Path(file_path).stem | |
| url = f"https://acpc.gujarat.gov.in/assets/uploads/media-uploader/{base_name}.pdf" | |
| return { | |
| "file_name": Path(file_path).name, | |
| "url": url, | |
| "page_id": url, | |
| } | |
| def _load_file(file_path: str) -> list[Document]: | |
| """Load a single file (PDF or text).""" | |
| path = Path(file_path) | |
| if path.suffix.lower() == ".pdf": | |
| loader = PyPDFLoader(str(path)) | |
| docs = loader.load() | |
| else: | |
| from langchain_community.document_loaders import TextLoader | |
| loader = TextLoader(str(path), encoding="utf-8") | |
| docs = loader.load() | |
| # Inject file-level metadata | |
| meta = _get_metadata(file_path) | |
| for doc in docs: | |
| doc.metadata.update(meta) | |
| return docs | |
| def ingest_file(file_path: str) -> int: | |
| """Ingest a single file into Qdrant. Returns number of chunks created.""" | |
| docs = _load_file(file_path) | |
| splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=settings.chunk_size, | |
| chunk_overlap=settings.chunk_overlap, | |
| ) | |
| chunks = splitter.split_documents(docs) | |
| vector_store = get_vector_store() | |
| vector_store.add_documents(chunks) | |
| return len(chunks) | |
| def ingest_directory(directory_path: str, recursive: bool = True) -> dict: | |
| """Ingest all PDFs from a directory. Returns summary.""" | |
| directory = Path(directory_path) | |
| pattern = "**/*.pdf" if recursive else "*.pdf" | |
| pdf_files = list(directory.glob(pattern)) | |
| total_chunks = 0 | |
| for pdf_file in pdf_files: | |
| total_chunks += ingest_file(str(pdf_file)) | |
| return {"total_documents": len(pdf_files), "total_chunks": total_chunks} | |
| def reset_collection() -> None: | |
| """Delete and recreate the Qdrant collection.""" | |
| client = get_qdrant_client() | |
| client.delete_collection(settings.collection_name) | |
| # Re-initialize the vector store (it recreates the collection on first use) | |
| global _vector_store_ref | |
| import app.retrieval.qdrant_store as qs | |
| qs._vector_store = None # Force re-init on next access | |