talk2data / chroma_utils.py
prithvi55's picture
Upload chroma_utils.py
13f3ee2 verified
Raw History Blame Contribute Delete
7.85 kB
import chromadb
import os
import tempfile
CHROMA_DIR = "./chromadb_local"
COLLECTION_NAME = "prompts"
EMBEDDING_MODEL = "all-MiniLM-L6-v2"
import chromadb
import os
import tempfile
import streamlit as st
CHROMA_DIR = "./chromadb_persistent" # More descriptive name
COLLECTION_NAME = "revenue_prompts" # Unique collection name
EMBEDDING_MODEL = "all-MiniLM-L6-v2"
@st.cache_resource
def get_chroma_client():
"""Initialize ChromaDB client with proper persistence (cached)"""
try:
# Ensure directory exists
os.makedirs(CHROMA_DIR, exist_ok=True)
# Use PersistentClient for true persistence
client = chromadb.PersistentClient(path=CHROMA_DIR)
# Test the client connection
try:
client.heartbeat()
print(f"βœ… Connected to existing ChromaDB at: {CHROMA_DIR}")
except:
print(f"βœ… Created new ChromaDB at: {CHROMA_DIR}")
return client
except Exception as e:
print(f"❌ ChromaDB connection error: {e}")
st.warning(f"ChromaDB error: {e}")
# Fallback to in-memory client (will be lost on restart)
return chromadb.Client()
@st.cache_resource
def get_chroma_collection(_client):
"""Get or create ChromaDB collection with persistence (cached)"""
try:
# Check if collection already exists
existing_collections = _client.list_collections()
collection_exists = any(col.name == COLLECTION_NAME for col in existing_collections)
if collection_exists:
collection = _client.get_collection(name=COLLECTION_NAME)
existing_count = collection.count()
print(f"βœ… Using existing collection '{COLLECTION_NAME}' with {existing_count} items")
st.info(f"πŸ“š Loaded existing vector database with {existing_count} prompts")
else:
collection = _client.create_collection(
name=COLLECTION_NAME,
metadata={"description": "Revenue app prompt-SQL pairs"}
)
print(f"βœ… Created new collection '{COLLECTION_NAME}'")
st.info("πŸ“š Created new vector database")
return collection
except Exception as e:
print(f"❌ Collection error: {e}")
st.error(f"Collection error: {e}")
return None
@st.cache_resource
def get_embedding_model():
"""Get embedding model with caching to avoid reloading"""
try:
from sentence_transformers import SentenceTransformer
import torch
device = "cpu" # Use CPU to avoid CUDA issues
# Use a persistent cache directory
cache_dir = os.path.join(CHROMA_DIR, "model_cache")
os.makedirs(cache_dir, exist_ok=True)
print(f"πŸ”„ Loading embedding model: {EMBEDDING_MODEL}")
model = SentenceTransformer(EMBEDDING_MODEL, device=device, cache_folder=cache_dir)
print(f"βœ… Embedding model loaded successfully")
return model
except Exception as e:
print(f"❌ Embedding model error: {e}")
st.warning(f"Using fallback embedder due to: {e}")
# Return a simple fallback embedder
return SimpleEmbedder()
def check_database_stats():
"""Check and display database statistics"""
try:
client = get_chroma_client()
collection = get_chroma_collection(client)
if collection:
count = collection.count()
return {
"collection_name": COLLECTION_NAME,
"total_items": count,
"database_path": CHROMA_DIR,
"status": "healthy"
}
else:
return {
"collection_name": COLLECTION_NAME,
"total_items": 0,
"database_path": CHROMA_DIR,
"status": "error"
}
except Exception as e:
return {
"collection_name": COLLECTION_NAME,
"total_items": 0,
"database_path": CHROMA_DIR,
"status": f"error: {e}"
}
def initialize_from_jsonl(jsonl_path="prompts.jsonl"):
"""Initialize ChromaDB from existing JSONL file if database is empty"""
try:
if not os.path.exists(jsonl_path):
return False
client = get_chroma_client()
collection = get_chroma_collection(client)
embedding_model = get_embedding_model()
if not collection or not embedding_model:
return False
# Check if collection is empty
if collection.count() > 0:
print(f"βœ… Database already has {collection.count()} items")
return True
# Load from JSONL
import json
prompts_loaded = 0
with open(jsonl_path, 'r', encoding='utf-8') as f:
for i, line in enumerate(f):
try:
data = json.loads(line)
prompt = data.get('prompt', '')
sql = data.get('sql', '')
if prompt and sql:
# Generate embedding
embedding = embedding_model.encode([prompt])
# Add to collection
collection.add(
embeddings=embedding,
documents=[prompt],
metadatas=[{"sql": sql, "source": "jsonl_import"}],
ids=[f"import_{i}"]
)
prompts_loaded += 1
except Exception as e:
print(f"⚠️ Error loading line {i}: {e}")
continue
print(f"βœ… Loaded {prompts_loaded} prompts from {jsonl_path}")
st.success(f"πŸ”„ Imported {prompts_loaded} existing prompts into vector database")
return True
except Exception as e:
print(f"❌ Error initializing from JSONL: {e}")
return False
"""Get embedding model with fallback options"""
try:
from sentence_transformers import SentenceTransformer
import torch
device = "cpu" # Use CPU to avoid CUDA issues
cache_dir = tempfile.mkdtemp()
# Try to load the model with error handling
model = SentenceTransformer(EMBEDDING_MODEL, device=device, cache_folder=cache_dir)
return model
except Exception as e:
print(f"Embedding model error: {e}")
# Return a simple fallback embedder
return SimpleEmbedder()
class SimpleEmbedder:
"""Simple fallback embedding function"""
def encode(self, texts, convert_to_numpy=True):
import hashlib
import numpy as np
if isinstance(texts, str):
texts = [texts]
embeddings = []
for text in texts:
# Create a simple 384-dimensional embedding from text hash
hash_obj = hashlib.md5(text.encode())
hash_hex = hash_obj.hexdigest()
# Convert hex to numbers and pad to 384 dimensions
embedding = [ord(c) / 255.0 for c in hash_hex]
# Repeat pattern to get 384 dimensions
while len(embedding) < 384:
embedding.extend(embedding[:384-len(embedding)])
embedding = embedding[:384]
embeddings.append(embedding)
return np.array(embeddings) if convert_to_numpy else embeddings