Spaces:
Sleeping
Sleeping
Download chroma_utils.py from prithvi55/talk2data: direct link, hf CLI and curl.
- Browser
- Download file 7.85 kB
-
https://huggingface.co/spaces/prithvi55/talk2data/resolve/refs%2Fpr%2F5/chroma_utils.py
- Command line
-
hf download hf://spaces/prithvi55/talk2data@refs/pr/5/chroma_utils.py
-
curl -L -o chroma_utils.py https://huggingface.co/spaces/prithvi55/talk2data/resolve/refs%2Fpr%2F5/chroma_utils.py
7.85 kB
| import chromadb | |
| import os | |
| import tempfile | |
| CHROMA_DIR = "./chromadb_local" | |
| COLLECTION_NAME = "prompts" | |
| EMBEDDING_MODEL = "all-MiniLM-L6-v2" | |
| import chromadb | |
| import os | |
| import tempfile | |
| import streamlit as st | |
| CHROMA_DIR = "./chromadb_persistent" # More descriptive name | |
| COLLECTION_NAME = "revenue_prompts" # Unique collection name | |
| EMBEDDING_MODEL = "all-MiniLM-L6-v2" | |
| def get_chroma_client(): | |
| """Initialize ChromaDB client with proper persistence (cached)""" | |
| try: | |
| # Ensure directory exists | |
| os.makedirs(CHROMA_DIR, exist_ok=True) | |
| # Use PersistentClient for true persistence | |
| client = chromadb.PersistentClient(path=CHROMA_DIR) | |
| # Test the client connection | |
| try: | |
| client.heartbeat() | |
| print(f"β Connected to existing ChromaDB at: {CHROMA_DIR}") | |
| except: | |
| print(f"β Created new ChromaDB at: {CHROMA_DIR}") | |
| return client | |
| except Exception as e: | |
| print(f"β ChromaDB connection error: {e}") | |
| st.warning(f"ChromaDB error: {e}") | |
| # Fallback to in-memory client (will be lost on restart) | |
| return chromadb.Client() | |
| def get_chroma_collection(_client): | |
| """Get or create ChromaDB collection with persistence (cached)""" | |
| try: | |
| # Check if collection already exists | |
| existing_collections = _client.list_collections() | |
| collection_exists = any(col.name == COLLECTION_NAME for col in existing_collections) | |
| if collection_exists: | |
| collection = _client.get_collection(name=COLLECTION_NAME) | |
| existing_count = collection.count() | |
| print(f"β Using existing collection '{COLLECTION_NAME}' with {existing_count} items") | |
| st.info(f"π Loaded existing vector database with {existing_count} prompts") | |
| else: | |
| collection = _client.create_collection( | |
| name=COLLECTION_NAME, | |
| metadata={"description": "Revenue app prompt-SQL pairs"} | |
| ) | |
| print(f"β Created new collection '{COLLECTION_NAME}'") | |
| st.info("π Created new vector database") | |
| return collection | |
| except Exception as e: | |
| print(f"β Collection error: {e}") | |
| st.error(f"Collection error: {e}") | |
| return None | |
| def get_embedding_model(): | |
| """Get embedding model with caching to avoid reloading""" | |
| try: | |
| from sentence_transformers import SentenceTransformer | |
| import torch | |
| device = "cpu" # Use CPU to avoid CUDA issues | |
| # Use a persistent cache directory | |
| cache_dir = os.path.join(CHROMA_DIR, "model_cache") | |
| os.makedirs(cache_dir, exist_ok=True) | |
| print(f"π Loading embedding model: {EMBEDDING_MODEL}") | |
| model = SentenceTransformer(EMBEDDING_MODEL, device=device, cache_folder=cache_dir) | |
| print(f"β Embedding model loaded successfully") | |
| return model | |
| except Exception as e: | |
| print(f"β Embedding model error: {e}") | |
| st.warning(f"Using fallback embedder due to: {e}") | |
| # Return a simple fallback embedder | |
| return SimpleEmbedder() | |
| def check_database_stats(): | |
| """Check and display database statistics""" | |
| try: | |
| client = get_chroma_client() | |
| collection = get_chroma_collection(client) | |
| if collection: | |
| count = collection.count() | |
| return { | |
| "collection_name": COLLECTION_NAME, | |
| "total_items": count, | |
| "database_path": CHROMA_DIR, | |
| "status": "healthy" | |
| } | |
| else: | |
| return { | |
| "collection_name": COLLECTION_NAME, | |
| "total_items": 0, | |
| "database_path": CHROMA_DIR, | |
| "status": "error" | |
| } | |
| except Exception as e: | |
| return { | |
| "collection_name": COLLECTION_NAME, | |
| "total_items": 0, | |
| "database_path": CHROMA_DIR, | |
| "status": f"error: {e}" | |
| } | |
| def initialize_from_jsonl(jsonl_path="prompts.jsonl"): | |
| """Initialize ChromaDB from existing JSONL file if database is empty""" | |
| try: | |
| if not os.path.exists(jsonl_path): | |
| return False | |
| client = get_chroma_client() | |
| collection = get_chroma_collection(client) | |
| embedding_model = get_embedding_model() | |
| if not collection or not embedding_model: | |
| return False | |
| # Check if collection is empty | |
| if collection.count() > 0: | |
| print(f"β Database already has {collection.count()} items") | |
| return True | |
| # Load from JSONL | |
| import json | |
| prompts_loaded = 0 | |
| with open(jsonl_path, 'r', encoding='utf-8') as f: | |
| for i, line in enumerate(f): | |
| try: | |
| data = json.loads(line) | |
| prompt = data.get('prompt', '') | |
| sql = data.get('sql', '') | |
| if prompt and sql: | |
| # Generate embedding | |
| embedding = embedding_model.encode([prompt]) | |
| # Add to collection | |
| collection.add( | |
| embeddings=embedding, | |
| documents=[prompt], | |
| metadatas=[{"sql": sql, "source": "jsonl_import"}], | |
| ids=[f"import_{i}"] | |
| ) | |
| prompts_loaded += 1 | |
| except Exception as e: | |
| print(f"β οΈ Error loading line {i}: {e}") | |
| continue | |
| print(f"β Loaded {prompts_loaded} prompts from {jsonl_path}") | |
| st.success(f"π Imported {prompts_loaded} existing prompts into vector database") | |
| return True | |
| except Exception as e: | |
| print(f"β Error initializing from JSONL: {e}") | |
| return False | |
| """Get embedding model with fallback options""" | |
| try: | |
| from sentence_transformers import SentenceTransformer | |
| import torch | |
| device = "cpu" # Use CPU to avoid CUDA issues | |
| cache_dir = tempfile.mkdtemp() | |
| # Try to load the model with error handling | |
| model = SentenceTransformer(EMBEDDING_MODEL, device=device, cache_folder=cache_dir) | |
| return model | |
| except Exception as e: | |
| print(f"Embedding model error: {e}") | |
| # Return a simple fallback embedder | |
| return SimpleEmbedder() | |
| class SimpleEmbedder: | |
| """Simple fallback embedding function""" | |
| def encode(self, texts, convert_to_numpy=True): | |
| import hashlib | |
| import numpy as np | |
| if isinstance(texts, str): | |
| texts = [texts] | |
| embeddings = [] | |
| for text in texts: | |
| # Create a simple 384-dimensional embedding from text hash | |
| hash_obj = hashlib.md5(text.encode()) | |
| hash_hex = hash_obj.hexdigest() | |
| # Convert hex to numbers and pad to 384 dimensions | |
| embedding = [ord(c) / 255.0 for c in hash_hex] | |
| # Repeat pattern to get 384 dimensions | |
| while len(embedding) < 384: | |
| embedding.extend(embedding[:384-len(embedding)]) | |
| embedding = embedding[:384] | |
| embeddings.append(embedding) | |
| return np.array(embeddings) if convert_to_numpy else embeddings | |