prithvi55 commited on
Commit
13f3ee2
Β·
verified Β·
1 Parent(s): b165d5f

Upload chroma_utils.py

Browse files
Files changed (1) hide show
  1. chroma_utils.py +215 -0
chroma_utils.py ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import chromadb
2
+ import os
3
+ import tempfile
4
+
5
+ CHROMA_DIR = "./chromadb_local"
6
+ COLLECTION_NAME = "prompts"
7
+ EMBEDDING_MODEL = "all-MiniLM-L6-v2"
8
+
9
+ import chromadb
10
+ import os
11
+ import tempfile
12
+ import streamlit as st
13
+
14
+ CHROMA_DIR = "./chromadb_persistent" # More descriptive name
15
+ COLLECTION_NAME = "revenue_prompts" # Unique collection name
16
+ EMBEDDING_MODEL = "all-MiniLM-L6-v2"
17
+
18
+ @st.cache_resource
19
+ def get_chroma_client():
20
+ """Initialize ChromaDB client with proper persistence (cached)"""
21
+ try:
22
+ # Ensure directory exists
23
+ os.makedirs(CHROMA_DIR, exist_ok=True)
24
+
25
+ # Use PersistentClient for true persistence
26
+ client = chromadb.PersistentClient(path=CHROMA_DIR)
27
+
28
+ # Test the client connection
29
+ try:
30
+ client.heartbeat()
31
+ print(f"βœ… Connected to existing ChromaDB at: {CHROMA_DIR}")
32
+ except:
33
+ print(f"βœ… Created new ChromaDB at: {CHROMA_DIR}")
34
+
35
+ return client
36
+ except Exception as e:
37
+ print(f"❌ ChromaDB connection error: {e}")
38
+ st.warning(f"ChromaDB error: {e}")
39
+ # Fallback to in-memory client (will be lost on restart)
40
+ return chromadb.Client()
41
+
42
+ @st.cache_resource
43
+ def get_chroma_collection(_client):
44
+ """Get or create ChromaDB collection with persistence (cached)"""
45
+ try:
46
+ # Check if collection already exists
47
+ existing_collections = _client.list_collections()
48
+ collection_exists = any(col.name == COLLECTION_NAME for col in existing_collections)
49
+
50
+ if collection_exists:
51
+ collection = _client.get_collection(name=COLLECTION_NAME)
52
+ existing_count = collection.count()
53
+ print(f"βœ… Using existing collection '{COLLECTION_NAME}' with {existing_count} items")
54
+ st.info(f"πŸ“š Loaded existing vector database with {existing_count} prompts")
55
+ else:
56
+ collection = _client.create_collection(
57
+ name=COLLECTION_NAME,
58
+ metadata={"description": "Revenue app prompt-SQL pairs"}
59
+ )
60
+ print(f"βœ… Created new collection '{COLLECTION_NAME}'")
61
+ st.info("πŸ“š Created new vector database")
62
+
63
+ return collection
64
+ except Exception as e:
65
+ print(f"❌ Collection error: {e}")
66
+ st.error(f"Collection error: {e}")
67
+ return None
68
+
69
+ @st.cache_resource
70
+ def get_embedding_model():
71
+ """Get embedding model with caching to avoid reloading"""
72
+ try:
73
+ from sentence_transformers import SentenceTransformer
74
+ import torch
75
+
76
+ device = "cpu" # Use CPU to avoid CUDA issues
77
+
78
+ # Use a persistent cache directory
79
+ cache_dir = os.path.join(CHROMA_DIR, "model_cache")
80
+ os.makedirs(cache_dir, exist_ok=True)
81
+
82
+ print(f"πŸ”„ Loading embedding model: {EMBEDDING_MODEL}")
83
+ model = SentenceTransformer(EMBEDDING_MODEL, device=device, cache_folder=cache_dir)
84
+ print(f"βœ… Embedding model loaded successfully")
85
+
86
+ return model
87
+
88
+ except Exception as e:
89
+ print(f"❌ Embedding model error: {e}")
90
+ st.warning(f"Using fallback embedder due to: {e}")
91
+ # Return a simple fallback embedder
92
+ return SimpleEmbedder()
93
+
94
+ def check_database_stats():
95
+ """Check and display database statistics"""
96
+ try:
97
+ client = get_chroma_client()
98
+ collection = get_chroma_collection(client)
99
+
100
+ if collection:
101
+ count = collection.count()
102
+ return {
103
+ "collection_name": COLLECTION_NAME,
104
+ "total_items": count,
105
+ "database_path": CHROMA_DIR,
106
+ "status": "healthy"
107
+ }
108
+ else:
109
+ return {
110
+ "collection_name": COLLECTION_NAME,
111
+ "total_items": 0,
112
+ "database_path": CHROMA_DIR,
113
+ "status": "error"
114
+ }
115
+ except Exception as e:
116
+ return {
117
+ "collection_name": COLLECTION_NAME,
118
+ "total_items": 0,
119
+ "database_path": CHROMA_DIR,
120
+ "status": f"error: {e}"
121
+ }
122
+
123
+ def initialize_from_jsonl(jsonl_path="prompts.jsonl"):
124
+ """Initialize ChromaDB from existing JSONL file if database is empty"""
125
+ try:
126
+ if not os.path.exists(jsonl_path):
127
+ return False
128
+
129
+ client = get_chroma_client()
130
+ collection = get_chroma_collection(client)
131
+ embedding_model = get_embedding_model()
132
+
133
+ if not collection or not embedding_model:
134
+ return False
135
+
136
+ # Check if collection is empty
137
+ if collection.count() > 0:
138
+ print(f"βœ… Database already has {collection.count()} items")
139
+ return True
140
+
141
+ # Load from JSONL
142
+ import json
143
+ prompts_loaded = 0
144
+
145
+ with open(jsonl_path, 'r', encoding='utf-8') as f:
146
+ for i, line in enumerate(f):
147
+ try:
148
+ data = json.loads(line)
149
+ prompt = data.get('prompt', '')
150
+ sql = data.get('sql', '')
151
+
152
+ if prompt and sql:
153
+ # Generate embedding
154
+ embedding = embedding_model.encode([prompt])
155
+
156
+ # Add to collection
157
+ collection.add(
158
+ embeddings=embedding,
159
+ documents=[prompt],
160
+ metadatas=[{"sql": sql, "source": "jsonl_import"}],
161
+ ids=[f"import_{i}"]
162
+ )
163
+ prompts_loaded += 1
164
+
165
+ except Exception as e:
166
+ print(f"⚠️ Error loading line {i}: {e}")
167
+ continue
168
+
169
+ print(f"βœ… Loaded {prompts_loaded} prompts from {jsonl_path}")
170
+ st.success(f"πŸ”„ Imported {prompts_loaded} existing prompts into vector database")
171
+ return True
172
+
173
+ except Exception as e:
174
+ print(f"❌ Error initializing from JSONL: {e}")
175
+ return False
176
+ """Get embedding model with fallback options"""
177
+ try:
178
+ from sentence_transformers import SentenceTransformer
179
+ import torch
180
+
181
+ device = "cpu" # Use CPU to avoid CUDA issues
182
+ cache_dir = tempfile.mkdtemp()
183
+
184
+ # Try to load the model with error handling
185
+ model = SentenceTransformer(EMBEDDING_MODEL, device=device, cache_folder=cache_dir)
186
+ return model
187
+
188
+ except Exception as e:
189
+ print(f"Embedding model error: {e}")
190
+ # Return a simple fallback embedder
191
+ return SimpleEmbedder()
192
+
193
+ class SimpleEmbedder:
194
+ """Simple fallback embedding function"""
195
+ def encode(self, texts, convert_to_numpy=True):
196
+ import hashlib
197
+ import numpy as np
198
+
199
+ if isinstance(texts, str):
200
+ texts = [texts]
201
+
202
+ embeddings = []
203
+ for text in texts:
204
+ # Create a simple 384-dimensional embedding from text hash
205
+ hash_obj = hashlib.md5(text.encode())
206
+ hash_hex = hash_obj.hexdigest()
207
+ # Convert hex to numbers and pad to 384 dimensions
208
+ embedding = [ord(c) / 255.0 for c in hash_hex]
209
+ # Repeat pattern to get 384 dimensions
210
+ while len(embedding) < 384:
211
+ embedding.extend(embedding[:384-len(embedding)])
212
+ embedding = embedding[:384]
213
+ embeddings.append(embedding)
214
+
215
+ return np.array(embeddings) if convert_to_numpy else embeddings