KajetanNiewczas commited on
Commit
d34dafd
Β·
1 Parent(s): 7929974

Trying to fix the new chromadb client approach

Browse files
ragntex.py CHANGED
@@ -2,7 +2,7 @@
2
 
3
  import os
4
 
5
- from src import demo, init_telemetry
6
 
7
  # Setup OpenTelemetry
8
  _ = init_telemetry()
@@ -10,9 +10,11 @@ _ = init_telemetry()
10
  # Run Gradio
11
  if os.getenv("IN_DOCKER") == "true":
12
  os.chdir("/")
 
13
  demo.launch(
14
  server_name="0.0.0.0", # nosec: intentional bind all interfaces for HuggingFace Spaces
15
  server_port=7860,
16
  )
17
  else:
 
18
  demo.launch(share=True)
 
2
 
3
  import os
4
 
5
+ from src import demo, init_telemetry, database
6
 
7
  # Setup OpenTelemetry
8
  _ = init_telemetry()
 
10
  # Run Gradio
11
  if os.getenv("IN_DOCKER") == "true":
12
  os.chdir("/")
13
+ database.init_chroma_client()
14
  demo.launch(
15
  server_name="0.0.0.0", # nosec: intentional bind all interfaces for HuggingFace Spaces
16
  server_port=7860,
17
  )
18
  else:
19
+ database.init_chroma_client()
20
  demo.launch(share=True)
src/__init__.py CHANGED
@@ -7,5 +7,5 @@ from .telemetry import init_telemetry
7
  __all__ = [
8
  "demo",
9
  "init_telemetry",
10
- "generate_presentation",
11
  ]
 
7
  __all__ = [
8
  "demo",
9
  "init_telemetry",
10
+ "generate_presentation"
11
  ]
src/database/__init__.py CHANGED
@@ -1,7 +1,8 @@
1
  """This module initializes the database package."""
2
 
3
- from .database import chroma_client, embed_fn
 
4
  from .db_manipulation import (clean_db, ingest_files_to_db,
5
  retrive_files_from_db)
6
 
7
- __all__ = ["chroma_client", "embed_fn", "ingest_files_to_db", "retrive_files_from_db", "clean_db"]
 
1
  """This module initializes the database package."""
2
 
3
+ from . import database
4
+ from .database import init_chroma_client
5
  from .db_manipulation import (clean_db, ingest_files_to_db,
6
  retrive_files_from_db)
7
 
8
+ __all__ = ["init_chroma_client", "ingest_files_to_db", "retrive_files_from_db", "clean_db"]
src/database/database.py CHANGED
@@ -39,6 +39,10 @@ class GeminiEmbeddingFunction(EmbeddingFunction):
39
  embed_fn = GeminiEmbeddingFunction()
40
  embed_fn.document_mode = True
41
 
42
- chroma_path = Path.cwd() / "tmp"
43
- chroma_path.mkdir(parents=True, exist_ok=True)
44
- chroma_client = chromadb.PersistentClient(path=str(chroma_path))
 
 
 
 
 
39
  embed_fn = GeminiEmbeddingFunction()
40
  embed_fn.document_mode = True
41
 
42
+ chroma_client = None
43
+
44
+ def init_chroma_client():
45
+ global chroma_client
46
+ chroma_path = Path.cwd() / "tmp"
47
+ chroma_path.mkdir(parents=True, exist_ok=True)
48
+ chroma_client = chromadb.PersistentClient(path=str(chroma_path))
src/database/db_manipulation.py CHANGED
@@ -6,7 +6,7 @@ from langfuse.decorators import langfuse_context, observe
6
 
7
  from ..processing import process_documents
8
  from ..telemetry import Logger
9
- from .database import chroma_client, embed_fn
10
 
11
  LOGGER = Logger.get_logger()
12
 
@@ -19,8 +19,8 @@ def ingest_files_to_db(pdf_files, session_id) -> list[str]:
19
  session_id (str): Unique identifier for the current session.
20
  """
21
  # Get or create the database collection
22
- db = chroma_client.get_or_create_collection(
23
- name=session_id, embedding_function=embed_fn
24
  )
25
 
26
  # Get all the existing entries and extract the known filenames
@@ -76,15 +76,15 @@ def retrive_files_from_db(
76
  - list[dict]: List of metadata associated with the documents.
77
  """
78
  # Get or create the database collection
79
- db = chroma_client.get_or_create_collection(
80
- name=session_id, embedding_function=embed_fn
81
  )
82
  if not db:
83
  LOGGER.error("❌ Database collection not found or could not be created.")
84
  return [], []
85
 
86
  LOGGER.info("πŸ” Retrieving relevant documents...")
87
- embed_fn.document_mode = False
88
  query_oneline = topic.replace("\n", " ")
89
 
90
  n_yelded_docs = 2
@@ -107,9 +107,9 @@ def clean_db(session_id: str) -> None:
107
  Args:
108
  session_id (str): Unique identifier for the current session.
109
  """
110
- collections = chroma_client.list_collections()
111
  if any(col.name == session_id for col in collections):
112
- chroma_client.delete_collection(name=session_id)
113
  LOGGER.info("πŸ—‘οΈ Cleaning database for session ID: %s", session_id)
114
  else:
115
  LOGGER.warning(
 
6
 
7
  from ..processing import process_documents
8
  from ..telemetry import Logger
9
+ from . import database
10
 
11
  LOGGER = Logger.get_logger()
12
 
 
19
  session_id (str): Unique identifier for the current session.
20
  """
21
  # Get or create the database collection
22
+ db = database.chroma_client.get_or_create_collection(
23
+ name=session_id, embedding_function=database.embed_fn
24
  )
25
 
26
  # Get all the existing entries and extract the known filenames
 
76
  - list[dict]: List of metadata associated with the documents.
77
  """
78
  # Get or create the database collection
79
+ db = database.chroma_client.get_or_create_collection(
80
+ name=session_id, embedding_function=database.embed_fn
81
  )
82
  if not db:
83
  LOGGER.error("❌ Database collection not found or could not be created.")
84
  return [], []
85
 
86
  LOGGER.info("πŸ” Retrieving relevant documents...")
87
+ database.embed_fn.document_mode = False
88
  query_oneline = topic.replace("\n", " ")
89
 
90
  n_yelded_docs = 2
 
107
  Args:
108
  session_id (str): Unique identifier for the current session.
109
  """
110
+ collections = database.chroma_client.list_collections()
111
  if any(col.name == session_id for col in collections):
112
+ database.chroma_client.delete_collection(name=session_id)
113
  LOGGER.info("πŸ—‘οΈ Cleaning database for session ID: %s", session_id)
114
  else:
115
  LOGGER.warning(