Spaces:
Sleeping
Sleeping
File size: 7,848 Bytes
13f3ee2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 | import chromadb
import os
import tempfile
CHROMA_DIR = "./chromadb_local"
COLLECTION_NAME = "prompts"
EMBEDDING_MODEL = "all-MiniLM-L6-v2"
import chromadb
import os
import tempfile
import streamlit as st
CHROMA_DIR = "./chromadb_persistent" # More descriptive name
COLLECTION_NAME = "revenue_prompts" # Unique collection name
EMBEDDING_MODEL = "all-MiniLM-L6-v2"
@st.cache_resource
def get_chroma_client():
"""Initialize ChromaDB client with proper persistence (cached)"""
try:
# Ensure directory exists
os.makedirs(CHROMA_DIR, exist_ok=True)
# Use PersistentClient for true persistence
client = chromadb.PersistentClient(path=CHROMA_DIR)
# Test the client connection
try:
client.heartbeat()
print(f"β
Connected to existing ChromaDB at: {CHROMA_DIR}")
except:
print(f"β
Created new ChromaDB at: {CHROMA_DIR}")
return client
except Exception as e:
print(f"β ChromaDB connection error: {e}")
st.warning(f"ChromaDB error: {e}")
# Fallback to in-memory client (will be lost on restart)
return chromadb.Client()
@st.cache_resource
def get_chroma_collection(_client):
"""Get or create ChromaDB collection with persistence (cached)"""
try:
# Check if collection already exists
existing_collections = _client.list_collections()
collection_exists = any(col.name == COLLECTION_NAME for col in existing_collections)
if collection_exists:
collection = _client.get_collection(name=COLLECTION_NAME)
existing_count = collection.count()
print(f"β
Using existing collection '{COLLECTION_NAME}' with {existing_count} items")
st.info(f"π Loaded existing vector database with {existing_count} prompts")
else:
collection = _client.create_collection(
name=COLLECTION_NAME,
metadata={"description": "Revenue app prompt-SQL pairs"}
)
print(f"β
Created new collection '{COLLECTION_NAME}'")
st.info("π Created new vector database")
return collection
except Exception as e:
print(f"β Collection error: {e}")
st.error(f"Collection error: {e}")
return None
@st.cache_resource
def get_embedding_model():
"""Get embedding model with caching to avoid reloading"""
try:
from sentence_transformers import SentenceTransformer
import torch
device = "cpu" # Use CPU to avoid CUDA issues
# Use a persistent cache directory
cache_dir = os.path.join(CHROMA_DIR, "model_cache")
os.makedirs(cache_dir, exist_ok=True)
print(f"π Loading embedding model: {EMBEDDING_MODEL}")
model = SentenceTransformer(EMBEDDING_MODEL, device=device, cache_folder=cache_dir)
print(f"β
Embedding model loaded successfully")
return model
except Exception as e:
print(f"β Embedding model error: {e}")
st.warning(f"Using fallback embedder due to: {e}")
# Return a simple fallback embedder
return SimpleEmbedder()
def check_database_stats():
"""Check and display database statistics"""
try:
client = get_chroma_client()
collection = get_chroma_collection(client)
if collection:
count = collection.count()
return {
"collection_name": COLLECTION_NAME,
"total_items": count,
"database_path": CHROMA_DIR,
"status": "healthy"
}
else:
return {
"collection_name": COLLECTION_NAME,
"total_items": 0,
"database_path": CHROMA_DIR,
"status": "error"
}
except Exception as e:
return {
"collection_name": COLLECTION_NAME,
"total_items": 0,
"database_path": CHROMA_DIR,
"status": f"error: {e}"
}
def initialize_from_jsonl(jsonl_path="prompts.jsonl"):
"""Initialize ChromaDB from existing JSONL file if database is empty"""
try:
if not os.path.exists(jsonl_path):
return False
client = get_chroma_client()
collection = get_chroma_collection(client)
embedding_model = get_embedding_model()
if not collection or not embedding_model:
return False
# Check if collection is empty
if collection.count() > 0:
print(f"β
Database already has {collection.count()} items")
return True
# Load from JSONL
import json
prompts_loaded = 0
with open(jsonl_path, 'r', encoding='utf-8') as f:
for i, line in enumerate(f):
try:
data = json.loads(line)
prompt = data.get('prompt', '')
sql = data.get('sql', '')
if prompt and sql:
# Generate embedding
embedding = embedding_model.encode([prompt])
# Add to collection
collection.add(
embeddings=embedding,
documents=[prompt],
metadatas=[{"sql": sql, "source": "jsonl_import"}],
ids=[f"import_{i}"]
)
prompts_loaded += 1
except Exception as e:
print(f"β οΈ Error loading line {i}: {e}")
continue
print(f"β
Loaded {prompts_loaded} prompts from {jsonl_path}")
st.success(f"π Imported {prompts_loaded} existing prompts into vector database")
return True
except Exception as e:
print(f"β Error initializing from JSONL: {e}")
return False
"""Get embedding model with fallback options"""
try:
from sentence_transformers import SentenceTransformer
import torch
device = "cpu" # Use CPU to avoid CUDA issues
cache_dir = tempfile.mkdtemp()
# Try to load the model with error handling
model = SentenceTransformer(EMBEDDING_MODEL, device=device, cache_folder=cache_dir)
return model
except Exception as e:
print(f"Embedding model error: {e}")
# Return a simple fallback embedder
return SimpleEmbedder()
class SimpleEmbedder:
"""Simple fallback embedding function"""
def encode(self, texts, convert_to_numpy=True):
import hashlib
import numpy as np
if isinstance(texts, str):
texts = [texts]
embeddings = []
for text in texts:
# Create a simple 384-dimensional embedding from text hash
hash_obj = hashlib.md5(text.encode())
hash_hex = hash_obj.hexdigest()
# Convert hex to numbers and pad to 384 dimensions
embedding = [ord(c) / 255.0 for c in hash_hex]
# Repeat pattern to get 384 dimensions
while len(embedding) < 384:
embedding.extend(embedding[:384-len(embedding)])
embedding = embedding[:384]
embeddings.append(embedding)
return np.array(embeddings) if convert_to_numpy else embeddings
|