File size: 7,848 Bytes
13f3ee2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
import chromadb
import os
import tempfile

CHROMA_DIR = "./chromadb_local"
COLLECTION_NAME = "prompts"
EMBEDDING_MODEL = "all-MiniLM-L6-v2"

import chromadb
import os
import tempfile
import streamlit as st

CHROMA_DIR = "./chromadb_persistent"  # More descriptive name
COLLECTION_NAME = "revenue_prompts"   # Unique collection name
EMBEDDING_MODEL = "all-MiniLM-L6-v2"

@st.cache_resource
def get_chroma_client():
    """Initialize ChromaDB client with proper persistence (cached)"""
    try:
        # Ensure directory exists
        os.makedirs(CHROMA_DIR, exist_ok=True)
        
        # Use PersistentClient for true persistence
        client = chromadb.PersistentClient(path=CHROMA_DIR)
        
        # Test the client connection
        try:
            client.heartbeat()
            print(f"βœ… Connected to existing ChromaDB at: {CHROMA_DIR}")
        except:
            print(f"βœ… Created new ChromaDB at: {CHROMA_DIR}")
        
        return client
    except Exception as e:
        print(f"❌ ChromaDB connection error: {e}")
        st.warning(f"ChromaDB error: {e}")
        # Fallback to in-memory client (will be lost on restart)
        return chromadb.Client()

@st.cache_resource
def get_chroma_collection(_client):
    """Get or create ChromaDB collection with persistence (cached)"""
    try:
        # Check if collection already exists
        existing_collections = _client.list_collections()
        collection_exists = any(col.name == COLLECTION_NAME for col in existing_collections)
        
        if collection_exists:
            collection = _client.get_collection(name=COLLECTION_NAME)
            existing_count = collection.count()
            print(f"βœ… Using existing collection '{COLLECTION_NAME}' with {existing_count} items")
            st.info(f"πŸ“š Loaded existing vector database with {existing_count} prompts")
        else:
            collection = _client.create_collection(
                name=COLLECTION_NAME,
                metadata={"description": "Revenue app prompt-SQL pairs"}
            )
            print(f"βœ… Created new collection '{COLLECTION_NAME}'")
            st.info("πŸ“š Created new vector database")
        
        return collection
    except Exception as e:
        print(f"❌ Collection error: {e}")
        st.error(f"Collection error: {e}")
        return None

@st.cache_resource
def get_embedding_model():
    """Get embedding model with caching to avoid reloading"""
    try:
        from sentence_transformers import SentenceTransformer
        import torch
        
        device = "cpu"  # Use CPU to avoid CUDA issues
        
        # Use a persistent cache directory
        cache_dir = os.path.join(CHROMA_DIR, "model_cache")
        os.makedirs(cache_dir, exist_ok=True)
        
        print(f"πŸ”„ Loading embedding model: {EMBEDDING_MODEL}")
        model = SentenceTransformer(EMBEDDING_MODEL, device=device, cache_folder=cache_dir)
        print(f"βœ… Embedding model loaded successfully")
        
        return model
        
    except Exception as e:
        print(f"❌ Embedding model error: {e}")
        st.warning(f"Using fallback embedder due to: {e}")
        # Return a simple fallback embedder
        return SimpleEmbedder()

def check_database_stats():
    """Check and display database statistics"""
    try:
        client = get_chroma_client()
        collection = get_chroma_collection(client)
        
        if collection:
            count = collection.count()
            return {
                "collection_name": COLLECTION_NAME,
                "total_items": count,
                "database_path": CHROMA_DIR,
                "status": "healthy"
            }
        else:
            return {
                "collection_name": COLLECTION_NAME,
                "total_items": 0,
                "database_path": CHROMA_DIR,
                "status": "error"
            }
    except Exception as e:
        return {
            "collection_name": COLLECTION_NAME,
            "total_items": 0,
            "database_path": CHROMA_DIR,
            "status": f"error: {e}"
        }

def initialize_from_jsonl(jsonl_path="prompts.jsonl"):
    """Initialize ChromaDB from existing JSONL file if database is empty"""
    try:
        if not os.path.exists(jsonl_path):
            return False
            
        client = get_chroma_client()
        collection = get_chroma_collection(client)
        embedding_model = get_embedding_model()
        
        if not collection or not embedding_model:
            return False
        
        # Check if collection is empty
        if collection.count() > 0:
            print(f"βœ… Database already has {collection.count()} items")
            return True
        
        # Load from JSONL
        import json
        prompts_loaded = 0
        
        with open(jsonl_path, 'r', encoding='utf-8') as f:
            for i, line in enumerate(f):
                try:
                    data = json.loads(line)
                    prompt = data.get('prompt', '')
                    sql = data.get('sql', '')
                    
                    if prompt and sql:
                        # Generate embedding
                        embedding = embedding_model.encode([prompt])
                        
                        # Add to collection
                        collection.add(
                            embeddings=embedding,
                            documents=[prompt],
                            metadatas=[{"sql": sql, "source": "jsonl_import"}],
                            ids=[f"import_{i}"]
                        )
                        prompts_loaded += 1
                        
                except Exception as e:
                    print(f"⚠️ Error loading line {i}: {e}")
                    continue
        
        print(f"βœ… Loaded {prompts_loaded} prompts from {jsonl_path}")
        st.success(f"πŸ”„ Imported {prompts_loaded} existing prompts into vector database")
        return True
        
    except Exception as e:
        print(f"❌ Error initializing from JSONL: {e}")
        return False
    """Get embedding model with fallback options"""
    try:
        from sentence_transformers import SentenceTransformer
        import torch
        
        device = "cpu"  # Use CPU to avoid CUDA issues
        cache_dir = tempfile.mkdtemp()
        
        # Try to load the model with error handling
        model = SentenceTransformer(EMBEDDING_MODEL, device=device, cache_folder=cache_dir)
        return model
        
    except Exception as e:
        print(f"Embedding model error: {e}")
        # Return a simple fallback embedder
        return SimpleEmbedder()

class SimpleEmbedder:
    """Simple fallback embedding function"""
    def encode(self, texts, convert_to_numpy=True):
        import hashlib
        import numpy as np
        
        if isinstance(texts, str):
            texts = [texts]
        
        embeddings = []
        for text in texts:
            # Create a simple 384-dimensional embedding from text hash
            hash_obj = hashlib.md5(text.encode())
            hash_hex = hash_obj.hexdigest()
            # Convert hex to numbers and pad to 384 dimensions
            embedding = [ord(c) / 255.0 for c in hash_hex]
            # Repeat pattern to get 384 dimensions
            while len(embedding) < 384:
                embedding.extend(embedding[:384-len(embedding)])
            embedding = embedding[:384]
            embeddings.append(embedding)
        
        return np.array(embeddings) if convert_to_numpy else embeddings