File size: 3,981 Bytes
57cfb76 6005177 57cfb76 6005177 d34dafd 57cfb76 2c834ce 57cfb76 2c834ce 57cfb76 2c834ce d34dafd e16c6ed 2c834ce 57cfb76 9f7e558 57cfb76 72a6115 57cfb76 72a6115 57cfb76 e16c6ed 57cfb76 2c834ce 57cfb76 2c834ce d34dafd e16c6ed 2c834ce 57cfb76 d34dafd 57cfb76 2c834ce d34dafd d458e3c d34dafd 2c834ce e16c6ed | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 | """Module for adding and getting PDF files to/from the database."""
from typing import List, Mapping, Union
from langfuse.decorators import langfuse_context, observe
from ..processing import process_documents
from ..telemetry import Logger
from . import database
LOGGER = Logger.get_logger()
@observe(name="π ingest_files_to_db")
def ingest_files_to_db(pdf_files, session_id) -> list[str]:
"""Add new files to the database if they are not yet there.
Args:
pdf_files (list): List of PDF files to be ingested.
session_id (str): Unique identifier for the current session.
"""
# Get or create the database collection
db = database.chroma_client.get_or_create_collection(
name=session_id, embedding_function=database.embed_fn
)
# Get all the existing entries and extract the known filenames
all_entries = db.get(include=["metadatas"]) or []
if not all_entries:
LOGGER.info("π No existing entries in the database. Ingesting all PDFs.")
existing_pdfs = set(
meta.get("pdf_path")
for meta in all_entries["metadatas"] # type: ignore[union-attr]
if meta.get("pdf_path")
)
# Filter out already ingested PDFs
new_pdfs = [file for file in pdf_files if file not in existing_pdfs]
if not new_pdfs:
LOGGER.info("π No new PDFs to ingest. Skipping ingestion.")
return []
# Process new files only
documents, metadatas, failed = process_documents(new_pdfs)
LOGGER.info("β Adding %d new PDFs to the database...", len(new_pdfs))
if documents:
db.add(
documents=documents,
ids=[str(i) for i in range(len(documents))],
metadatas=metadatas,
)
LOGGER.info("β
Documents ingested successfully.")
return failed
Metadata = Mapping[str, Union[str, int, float, bool]]
DocumentsBatch = List[List[str]]
MetadataBatch = List[List[Metadata]]
@observe(name="π retrieve_files_from_db")
def retrive_files_from_db(
topic: str, session_id: str
) -> tuple[DocumentsBatch, MetadataBatch]:
"""Retrieve relevant documents from the database based on the provided topic.
Args:
topic (str): The topic for which to retrieve documents.
session_id (str): Unique identifier for the current session.
Returns:
tuple: A 2-element tuple:
- list[str]: List of retrieved documents.
- list[dict]: List of metadata associated with the documents.
"""
# Get or create the database collection
db = database.chroma_client.get_or_create_collection(
name=session_id, embedding_function=database.embed_fn
)
if not db:
LOGGER.error("β Database collection not found or could not be created.")
return [], []
LOGGER.info("π Retrieving relevant documents...")
database.embed_fn.document_mode = False
query_oneline = topic.replace("\n", " ")
n_yelded_docs = 2
result = db.query(query_texts=[query_oneline], n_results=n_yelded_docs)
langfuse_context.update_current_observation(
output={
"input.query_oneline": query_oneline,
"output.n_results": n_yelded_docs,
"output.metadatas": result["metadatas"],
}
)
[documents] = result.get("documents") or [[]]
[metadatas] = result.get("metadatas") or [[]]
return [documents], [metadatas]
def clean_db(session_id: str) -> None:
"""Clean the database collection for the given session ID.
Args:
session_id (str): Unique identifier for the current session.
"""
collections = database.chroma_client.list_collections()
if any(col.name == session_id for col in collections):
database.chroma_client.delete_collection(name=session_id)
LOGGER.info("ποΈ Cleaning database for session ID: %s", session_id)
else:
LOGGER.warning(
"β Database collection not found for session ID: %s", session_id
)
|