Spaces:
Sleeping
Sleeping
Download build_chromadb.py from prithvi55/talk2data: direct link, hf CLI and curl.
- Browser
- Download file 2.49 kB
-
https://huggingface.co/spaces/prithvi55/talk2data/resolve/main/build_chromadb.py
- Command line
-
hf download hf://spaces/prithvi55/talk2data/build_chromadb.py
-
curl -L -o build_chromadb.py https://huggingface.co/spaces/prithvi55/talk2data/resolve/main/build_chromadb.py
2.49 kB
| import json | |
| import chromadb | |
| from chromadb.config import Settings | |
| from sentence_transformers import SentenceTransformer | |
| import os | |
| PROMPTS_FILE = "prompts.jsonl" | |
| CHROMA_DIR = "chromadb_local" | |
| COLLECTION_NAME = "prompts" | |
| EMBEDDING_MODEL = "all-MiniLM-L6-v2" # You can change to another local model if desired | |
| def load_prompts(prompts_file): | |
| prompts = [] | |
| with open(prompts_file, "r", encoding="utf-8") as f: | |
| for line in f: | |
| if line.strip(): | |
| data = json.loads(line) | |
| prompts.append(data) | |
| return prompts | |
| def build_chromadb(prompts, chroma_dir, collection_name, embedding_model): | |
| print(f"[DEBUG] Connecting to ChromaDB at: {chroma_dir}, collection: {collection_name}") | |
| # Initialize ChromaDB client (local, no cloud) | |
| client = chromadb.Client(Settings(persist_directory=chroma_dir)) | |
| print(f"[DEBUG] Existing collections: {[c.name for c in client.list_collections()]}") | |
| # Remove old collection if exists | |
| if collection_name in [c.name for c in client.list_collections()]: | |
| print(f"[DEBUG] Deleting existing collection: {collection_name}") | |
| client.delete_collection(collection_name) | |
| collection = client.create_collection(collection_name) | |
| print(f"[DEBUG] Created collection: {collection_name}") | |
| # Load embedding model | |
| print(f"[DEBUG] Loading embedding model: {embedding_model}") | |
| model = SentenceTransformer(embedding_model) | |
| # Prepare data | |
| texts = [p["prompt"] for p in prompts] | |
| metadatas = [{"sql": p["sql"]} for p in prompts] | |
| ids = [str(i) for i in range(len(prompts))] | |
| print(f"[DEBUG] Number of prompts: {len(texts)}") | |
| # Compute embeddings | |
| print(f"[DEBUG] Computing embeddings...") | |
| embeddings = model.encode(texts, show_progress_bar=True, convert_to_numpy=True) | |
| # Add to ChromaDB | |
| print(f"[DEBUG] Adding embeddings to ChromaDB...") | |
| collection.add( | |
| embeddings=embeddings, | |
| documents=texts, | |
| metadatas=metadatas, | |
| ids=ids | |
| ) | |
| #client.persist() | |
| print(f"ChromaDB built with {len(prompts)} prompts at '{chroma_dir}' (collection: '{collection_name}')") | |
| def main(): | |
| if not os.path.exists(PROMPTS_FILE): | |
| print(f"File not found: {PROMPTS_FILE}") | |
| return | |
| prompts = load_prompts(PROMPTS_FILE) | |
| build_chromadb(prompts, CHROMA_DIR, COLLECTION_NAME, EMBEDDING_MODEL) | |
| if __name__ == "__main__": | |
| main() | |