import chromadb import os import tempfile CHROMA_DIR = "./chromadb_local" COLLECTION_NAME = "prompts" EMBEDDING_MODEL = "all-MiniLM-L6-v2" import chromadb import os import tempfile import streamlit as st CHROMA_DIR = "./chromadb_persistent" # More descriptive name COLLECTION_NAME = "revenue_prompts" # Unique collection name EMBEDDING_MODEL = "all-MiniLM-L6-v2" @st.cache_resource def get_chroma_client(): """Initialize ChromaDB client with proper persistence (cached)""" try: # Ensure directory exists os.makedirs(CHROMA_DIR, exist_ok=True) # Use PersistentClient for true persistence client = chromadb.PersistentClient(path=CHROMA_DIR) # Test the client connection try: client.heartbeat() print(f"✅ Connected to existing ChromaDB at: {CHROMA_DIR}") except: print(f"✅ Created new ChromaDB at: {CHROMA_DIR}") return client except Exception as e: print(f"❌ ChromaDB connection error: {e}") st.warning(f"ChromaDB error: {e}") # Fallback to in-memory client (will be lost on restart) return chromadb.Client() @st.cache_resource def get_chroma_collection(_client): """Get or create ChromaDB collection with persistence (cached)""" try: # Check if collection already exists existing_collections = _client.list_collections() collection_exists = any(col.name == COLLECTION_NAME for col in existing_collections) if collection_exists: collection = _client.get_collection(name=COLLECTION_NAME) existing_count = collection.count() print(f"✅ Using existing collection '{COLLECTION_NAME}' with {existing_count} items") st.info(f"📚 Loaded existing vector database with {existing_count} prompts") else: collection = _client.create_collection( name=COLLECTION_NAME, metadata={"description": "Revenue app prompt-SQL pairs"} ) print(f"✅ Created new collection '{COLLECTION_NAME}'") st.info("📚 Created new vector database") return collection except Exception as e: print(f"❌ Collection error: {e}") st.error(f"Collection error: {e}") return None @st.cache_resource def get_embedding_model(): """Get embedding model with caching to avoid reloading""" try: from sentence_transformers import SentenceTransformer import torch device = "cpu" # Use CPU to avoid CUDA issues # Use a persistent cache directory cache_dir = os.path.join(CHROMA_DIR, "model_cache") os.makedirs(cache_dir, exist_ok=True) print(f"🔄 Loading embedding model: {EMBEDDING_MODEL}") model = SentenceTransformer(EMBEDDING_MODEL, device=device, cache_folder=cache_dir) print(f"✅ Embedding model loaded successfully") return model except Exception as e: print(f"❌ Embedding model error: {e}") st.warning(f"Using fallback embedder due to: {e}") # Return a simple fallback embedder return SimpleEmbedder() def check_database_stats(): """Check and display database statistics""" try: client = get_chroma_client() collection = get_chroma_collection(client) if collection: count = collection.count() return { "collection_name": COLLECTION_NAME, "total_items": count, "database_path": CHROMA_DIR, "status": "healthy" } else: return { "collection_name": COLLECTION_NAME, "total_items": 0, "database_path": CHROMA_DIR, "status": "error" } except Exception as e: return { "collection_name": COLLECTION_NAME, "total_items": 0, "database_path": CHROMA_DIR, "status": f"error: {e}" } def initialize_from_jsonl(jsonl_path="prompts.jsonl"): """Initialize ChromaDB from existing JSONL file if database is empty""" try: if not os.path.exists(jsonl_path): return False client = get_chroma_client() collection = get_chroma_collection(client) embedding_model = get_embedding_model() if not collection or not embedding_model: return False # Check if collection is empty if collection.count() > 0: print(f"✅ Database already has {collection.count()} items") return True # Load from JSONL import json prompts_loaded = 0 with open(jsonl_path, 'r', encoding='utf-8') as f: for i, line in enumerate(f): try: data = json.loads(line) prompt = data.get('prompt', '') sql = data.get('sql', '') if prompt and sql: # Generate embedding embedding = embedding_model.encode([prompt]) # Add to collection collection.add( embeddings=embedding, documents=[prompt], metadatas=[{"sql": sql, "source": "jsonl_import"}], ids=[f"import_{i}"] ) prompts_loaded += 1 except Exception as e: print(f"⚠️ Error loading line {i}: {e}") continue print(f"✅ Loaded {prompts_loaded} prompts from {jsonl_path}") st.success(f"🔄 Imported {prompts_loaded} existing prompts into vector database") return True except Exception as e: print(f"❌ Error initializing from JSONL: {e}") return False """Get embedding model with fallback options""" try: from sentence_transformers import SentenceTransformer import torch device = "cpu" # Use CPU to avoid CUDA issues cache_dir = tempfile.mkdtemp() # Try to load the model with error handling model = SentenceTransformer(EMBEDDING_MODEL, device=device, cache_folder=cache_dir) return model except Exception as e: print(f"Embedding model error: {e}") # Return a simple fallback embedder return SimpleEmbedder() class SimpleEmbedder: """Simple fallback embedding function""" def encode(self, texts, convert_to_numpy=True): import hashlib import numpy as np if isinstance(texts, str): texts = [texts] embeddings = [] for text in texts: # Create a simple 384-dimensional embedding from text hash hash_obj = hashlib.md5(text.encode()) hash_hex = hash_obj.hexdigest() # Convert hex to numbers and pad to 384 dimensions embedding = [ord(c) / 255.0 for c in hash_hex] # Repeat pattern to get 384 dimensions while len(embedding) < 384: embedding.extend(embedding[:384-len(embedding)]) embedding = embedding[:384] embeddings.append(embedding) return np.array(embeddings) if convert_to_numpy else embeddings