File size: 3,981 Bytes
57cfb76
 
6005177
 
57cfb76
 
 
6005177
d34dafd
57cfb76
 
 
 
 
2c834ce
57cfb76
 
 
2c834ce
57cfb76
2c834ce
d34dafd
 
e16c6ed
2c834ce
57cfb76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9f7e558
57cfb76
 
72a6115
57cfb76
 
 
 
 
 
 
 
 
 
72a6115
 
57cfb76
 
 
 
 
 
 
e16c6ed
 
 
57cfb76
 
 
2c834ce
57cfb76
 
 
 
 
2c834ce
d34dafd
 
e16c6ed
2c834ce
 
 
 
57cfb76
d34dafd
57cfb76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2c834ce
 
 
 
 
 
 
d34dafd
d458e3c
d34dafd
2c834ce
 
e16c6ed
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
"""Module for adding and getting PDF files to/from the database."""

from typing import List, Mapping, Union

from langfuse.decorators import langfuse_context, observe

from ..processing import process_documents
from ..telemetry import Logger
from . import database

LOGGER = Logger.get_logger()


@observe(name="𝌊 ingest_files_to_db")
def ingest_files_to_db(pdf_files, session_id) -> list[str]:
    """Add new files to the database if they are not yet there.
    Args:
        pdf_files (list): List of PDF files to be ingested.
        session_id (str): Unique identifier for the current session.
    """
    # Get or create the database collection
    db = database.chroma_client.get_or_create_collection(
        name=session_id, embedding_function=database.embed_fn
    )

    # Get all the existing entries and extract the known filenames
    all_entries = db.get(include=["metadatas"]) or []

    if not all_entries:
        LOGGER.info("πŸ“‚ No existing entries in the database. Ingesting all PDFs.")

    existing_pdfs = set(
        meta.get("pdf_path")
        for meta in all_entries["metadatas"]  # type: ignore[union-attr]
        if meta.get("pdf_path")
    )

    # Filter out already ingested PDFs
    new_pdfs = [file for file in pdf_files if file not in existing_pdfs]

    if not new_pdfs:
        LOGGER.info("πŸ“‚ No new PDFs to ingest. Skipping ingestion.")
        return []

    # Process new files only
    documents, metadatas, failed = process_documents(new_pdfs)

    LOGGER.info("βž• Adding %d new PDFs to the database...", len(new_pdfs))
    if documents:
        db.add(
            documents=documents,
            ids=[str(i) for i in range(len(documents))],
            metadatas=metadatas,
        )
    LOGGER.info("βœ… Documents ingested successfully.")

    return failed


Metadata = Mapping[str, Union[str, int, float, bool]]
DocumentsBatch = List[List[str]]
MetadataBatch = List[List[Metadata]]


@observe(name="πŸ” retrieve_files_from_db")
def retrive_files_from_db(
    topic: str, session_id: str
) -> tuple[DocumentsBatch, MetadataBatch]:
    """Retrieve relevant documents from the database based on the provided topic.
    Args:
        topic (str): The topic for which to retrieve documents.
        session_id (str): Unique identifier for the current session.
    Returns:
        tuple: A 2-element tuple:
            - list[str]: List of retrieved documents.
            - list[dict]: List of metadata associated with the documents.
    """
    # Get or create the database collection
    db = database.chroma_client.get_or_create_collection(
        name=session_id, embedding_function=database.embed_fn
    )
    if not db:
        LOGGER.error("❌ Database collection not found or could not be created.")
        return [], []

    LOGGER.info("πŸ” Retrieving relevant documents...")
    database.embed_fn.document_mode = False
    query_oneline = topic.replace("\n", " ")

    n_yelded_docs = 2
    result = db.query(query_texts=[query_oneline], n_results=n_yelded_docs)
    langfuse_context.update_current_observation(
        output={
            "input.query_oneline": query_oneline,
            "output.n_results": n_yelded_docs,
            "output.metadatas": result["metadatas"],
        }
    )
    [documents] = result.get("documents") or [[]]
    [metadatas] = result.get("metadatas") or [[]]

    return [documents], [metadatas]


def clean_db(session_id: str) -> None:
    """Clean the database collection for the given session ID.
    Args:
        session_id (str): Unique identifier for the current session.
    """
    collections = database.chroma_client.list_collections()
    if any(col.name == session_id for col in collections):
        database.chroma_client.delete_collection(name=session_id)
        LOGGER.info("πŸ—‘οΈ Cleaning database for session ID: %s", session_id)
    else:
        LOGGER.warning(
            "❌ Database collection not found for session ID: %s", session_id
        )