KajetanNiewczas commited on
Commit Β·
d34dafd
1
Parent(s): 7929974
Trying to fix the new chromadb client approach
Browse files- ragntex.py +3 -1
- src/__init__.py +1 -1
- src/database/__init__.py +3 -2
- src/database/database.py +7 -3
- src/database/db_manipulation.py +8 -8
ragntex.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
|
| 3 |
import os
|
| 4 |
|
| 5 |
-
from src import demo, init_telemetry
|
| 6 |
|
| 7 |
# Setup OpenTelemetry
|
| 8 |
_ = init_telemetry()
|
|
@@ -10,9 +10,11 @@ _ = init_telemetry()
|
|
| 10 |
# Run Gradio
|
| 11 |
if os.getenv("IN_DOCKER") == "true":
|
| 12 |
os.chdir("/")
|
|
|
|
| 13 |
demo.launch(
|
| 14 |
server_name="0.0.0.0", # nosec: intentional bind all interfaces for HuggingFace Spaces
|
| 15 |
server_port=7860,
|
| 16 |
)
|
| 17 |
else:
|
|
|
|
| 18 |
demo.launch(share=True)
|
|
|
|
| 2 |
|
| 3 |
import os
|
| 4 |
|
| 5 |
+
from src import demo, init_telemetry, database
|
| 6 |
|
| 7 |
# Setup OpenTelemetry
|
| 8 |
_ = init_telemetry()
|
|
|
|
| 10 |
# Run Gradio
|
| 11 |
if os.getenv("IN_DOCKER") == "true":
|
| 12 |
os.chdir("/")
|
| 13 |
+
database.init_chroma_client()
|
| 14 |
demo.launch(
|
| 15 |
server_name="0.0.0.0", # nosec: intentional bind all interfaces for HuggingFace Spaces
|
| 16 |
server_port=7860,
|
| 17 |
)
|
| 18 |
else:
|
| 19 |
+
database.init_chroma_client()
|
| 20 |
demo.launch(share=True)
|
src/__init__.py
CHANGED
|
@@ -7,5 +7,5 @@ from .telemetry import init_telemetry
|
|
| 7 |
__all__ = [
|
| 8 |
"demo",
|
| 9 |
"init_telemetry",
|
| 10 |
-
"generate_presentation"
|
| 11 |
]
|
|
|
|
| 7 |
__all__ = [
|
| 8 |
"demo",
|
| 9 |
"init_telemetry",
|
| 10 |
+
"generate_presentation"
|
| 11 |
]
|
src/database/__init__.py
CHANGED
|
@@ -1,7 +1,8 @@
|
|
| 1 |
"""This module initializes the database package."""
|
| 2 |
|
| 3 |
-
from .
|
|
|
|
| 4 |
from .db_manipulation import (clean_db, ingest_files_to_db,
|
| 5 |
retrive_files_from_db)
|
| 6 |
|
| 7 |
-
__all__ = ["
|
|
|
|
| 1 |
"""This module initializes the database package."""
|
| 2 |
|
| 3 |
+
from . import database
|
| 4 |
+
from .database import init_chroma_client
|
| 5 |
from .db_manipulation import (clean_db, ingest_files_to_db,
|
| 6 |
retrive_files_from_db)
|
| 7 |
|
| 8 |
+
__all__ = ["init_chroma_client", "ingest_files_to_db", "retrive_files_from_db", "clean_db"]
|
src/database/database.py
CHANGED
|
@@ -39,6 +39,10 @@ class GeminiEmbeddingFunction(EmbeddingFunction):
|
|
| 39 |
embed_fn = GeminiEmbeddingFunction()
|
| 40 |
embed_fn.document_mode = True
|
| 41 |
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
embed_fn = GeminiEmbeddingFunction()
|
| 40 |
embed_fn.document_mode = True
|
| 41 |
|
| 42 |
+
chroma_client = None
|
| 43 |
+
|
| 44 |
+
def init_chroma_client():
|
| 45 |
+
global chroma_client
|
| 46 |
+
chroma_path = Path.cwd() / "tmp"
|
| 47 |
+
chroma_path.mkdir(parents=True, exist_ok=True)
|
| 48 |
+
chroma_client = chromadb.PersistentClient(path=str(chroma_path))
|
src/database/db_manipulation.py
CHANGED
|
@@ -6,7 +6,7 @@ from langfuse.decorators import langfuse_context, observe
|
|
| 6 |
|
| 7 |
from ..processing import process_documents
|
| 8 |
from ..telemetry import Logger
|
| 9 |
-
from .
|
| 10 |
|
| 11 |
LOGGER = Logger.get_logger()
|
| 12 |
|
|
@@ -19,8 +19,8 @@ def ingest_files_to_db(pdf_files, session_id) -> list[str]:
|
|
| 19 |
session_id (str): Unique identifier for the current session.
|
| 20 |
"""
|
| 21 |
# Get or create the database collection
|
| 22 |
-
db = chroma_client.get_or_create_collection(
|
| 23 |
-
name=session_id, embedding_function=embed_fn
|
| 24 |
)
|
| 25 |
|
| 26 |
# Get all the existing entries and extract the known filenames
|
|
@@ -76,15 +76,15 @@ def retrive_files_from_db(
|
|
| 76 |
- list[dict]: List of metadata associated with the documents.
|
| 77 |
"""
|
| 78 |
# Get or create the database collection
|
| 79 |
-
db = chroma_client.get_or_create_collection(
|
| 80 |
-
name=session_id, embedding_function=embed_fn
|
| 81 |
)
|
| 82 |
if not db:
|
| 83 |
LOGGER.error("β Database collection not found or could not be created.")
|
| 84 |
return [], []
|
| 85 |
|
| 86 |
LOGGER.info("π Retrieving relevant documents...")
|
| 87 |
-
embed_fn.document_mode = False
|
| 88 |
query_oneline = topic.replace("\n", " ")
|
| 89 |
|
| 90 |
n_yelded_docs = 2
|
|
@@ -107,9 +107,9 @@ def clean_db(session_id: str) -> None:
|
|
| 107 |
Args:
|
| 108 |
session_id (str): Unique identifier for the current session.
|
| 109 |
"""
|
| 110 |
-
collections = chroma_client.list_collections()
|
| 111 |
if any(col.name == session_id for col in collections):
|
| 112 |
-
chroma_client.delete_collection(name=session_id)
|
| 113 |
LOGGER.info("ποΈ Cleaning database for session ID: %s", session_id)
|
| 114 |
else:
|
| 115 |
LOGGER.warning(
|
|
|
|
| 6 |
|
| 7 |
from ..processing import process_documents
|
| 8 |
from ..telemetry import Logger
|
| 9 |
+
from . import database
|
| 10 |
|
| 11 |
LOGGER = Logger.get_logger()
|
| 12 |
|
|
|
|
| 19 |
session_id (str): Unique identifier for the current session.
|
| 20 |
"""
|
| 21 |
# Get or create the database collection
|
| 22 |
+
db = database.chroma_client.get_or_create_collection(
|
| 23 |
+
name=session_id, embedding_function=database.embed_fn
|
| 24 |
)
|
| 25 |
|
| 26 |
# Get all the existing entries and extract the known filenames
|
|
|
|
| 76 |
- list[dict]: List of metadata associated with the documents.
|
| 77 |
"""
|
| 78 |
# Get or create the database collection
|
| 79 |
+
db = database.chroma_client.get_or_create_collection(
|
| 80 |
+
name=session_id, embedding_function=database.embed_fn
|
| 81 |
)
|
| 82 |
if not db:
|
| 83 |
LOGGER.error("β Database collection not found or could not be created.")
|
| 84 |
return [], []
|
| 85 |
|
| 86 |
LOGGER.info("π Retrieving relevant documents...")
|
| 87 |
+
database.embed_fn.document_mode = False
|
| 88 |
query_oneline = topic.replace("\n", " ")
|
| 89 |
|
| 90 |
n_yelded_docs = 2
|
|
|
|
| 107 |
Args:
|
| 108 |
session_id (str): Unique identifier for the current session.
|
| 109 |
"""
|
| 110 |
+
collections = database.chroma_client.list_collections()
|
| 111 |
if any(col.name == session_id for col in collections):
|
| 112 |
+
database.chroma_client.delete_collection(name=session_id)
|
| 113 |
LOGGER.info("ποΈ Cleaning database for session ID: %s", session_id)
|
| 114 |
else:
|
| 115 |
LOGGER.warning(
|