Download src/database/db_manipulation.py from RAGnTeX/ragntex: direct link, hf CLI and curl.
- Browser
- Download file 3.98 kB
-
https://huggingface.co/spaces/RAGnTeX/ragntex/resolve/main/src/database/db_manipulation.py
- Command line
-
hf download hf://spaces/RAGnTeX/ragntex/src/database/db_manipulation.py
-
curl -L -o db_manipulation.py https://huggingface.co/spaces/RAGnTeX/ragntex/resolve/main/src/database/db_manipulation.py
3.98 kB
| """Module for adding and getting PDF files to/from the database.""" | |
| from typing import List, Mapping, Union | |
| from langfuse.decorators import langfuse_context, observe | |
| from ..processing import process_documents | |
| from ..telemetry import Logger | |
| from . import database | |
| LOGGER = Logger.get_logger() | |
| def ingest_files_to_db(pdf_files, session_id) -> list[str]: | |
| """Add new files to the database if they are not yet there. | |
| Args: | |
| pdf_files (list): List of PDF files to be ingested. | |
| session_id (str): Unique identifier for the current session. | |
| """ | |
| # Get or create the database collection | |
| db = database.chroma_client.get_or_create_collection( | |
| name=session_id, embedding_function=database.embed_fn | |
| ) | |
| # Get all the existing entries and extract the known filenames | |
| all_entries = db.get(include=["metadatas"]) or [] | |
| if not all_entries: | |
| LOGGER.info("π No existing entries in the database. Ingesting all PDFs.") | |
| existing_pdfs = set( | |
| meta.get("pdf_path") | |
| for meta in all_entries["metadatas"] # type: ignore[union-attr] | |
| if meta.get("pdf_path") | |
| ) | |
| # Filter out already ingested PDFs | |
| new_pdfs = [file for file in pdf_files if file not in existing_pdfs] | |
| if not new_pdfs: | |
| LOGGER.info("π No new PDFs to ingest. Skipping ingestion.") | |
| return [] | |
| # Process new files only | |
| documents, metadatas, failed = process_documents(new_pdfs) | |
| LOGGER.info("β Adding %d new PDFs to the database...", len(new_pdfs)) | |
| if documents: | |
| db.add( | |
| documents=documents, | |
| ids=[str(i) for i in range(len(documents))], | |
| metadatas=metadatas, | |
| ) | |
| LOGGER.info("β Documents ingested successfully.") | |
| return failed | |
| Metadata = Mapping[str, Union[str, int, float, bool]] | |
| DocumentsBatch = List[List[str]] | |
| MetadataBatch = List[List[Metadata]] | |
| def retrive_files_from_db( | |
| topic: str, session_id: str | |
| ) -> tuple[DocumentsBatch, MetadataBatch]: | |
| """Retrieve relevant documents from the database based on the provided topic. | |
| Args: | |
| topic (str): The topic for which to retrieve documents. | |
| session_id (str): Unique identifier for the current session. | |
| Returns: | |
| tuple: A 2-element tuple: | |
| - list[str]: List of retrieved documents. | |
| - list[dict]: List of metadata associated with the documents. | |
| """ | |
| # Get or create the database collection | |
| db = database.chroma_client.get_or_create_collection( | |
| name=session_id, embedding_function=database.embed_fn | |
| ) | |
| if not db: | |
| LOGGER.error("β Database collection not found or could not be created.") | |
| return [], [] | |
| LOGGER.info("π Retrieving relevant documents...") | |
| database.embed_fn.document_mode = False | |
| query_oneline = topic.replace("\n", " ") | |
| n_yelded_docs = 2 | |
| result = db.query(query_texts=[query_oneline], n_results=n_yelded_docs) | |
| langfuse_context.update_current_observation( | |
| output={ | |
| "input.query_oneline": query_oneline, | |
| "output.n_results": n_yelded_docs, | |
| "output.metadatas": result["metadatas"], | |
| } | |
| ) | |
| [documents] = result.get("documents") or [[]] | |
| [metadatas] = result.get("metadatas") or [[]] | |
| return [documents], [metadatas] | |
| def clean_db(session_id: str) -> None: | |
| """Clean the database collection for the given session ID. | |
| Args: | |
| session_id (str): Unique identifier for the current session. | |
| """ | |
| collections = database.chroma_client.list_collections() | |
| if any(col.name == session_id for col in collections): | |
| database.chroma_client.delete_collection(name=session_id) | |
| LOGGER.info("ποΈ Cleaning database for session ID: %s", session_id) | |
| else: | |
| LOGGER.warning( | |
| "β Database collection not found for session ID: %s", session_id | |
| ) | |