import json import chromadb from chromadb.config import Settings from sentence_transformers import SentenceTransformer import os PROMPTS_FILE = "prompts.jsonl" CHROMA_DIR = "chromadb_local" COLLECTION_NAME = "prompts" EMBEDDING_MODEL = "all-MiniLM-L6-v2" # You can change to another local model if desired def load_prompts(prompts_file): prompts = [] with open(prompts_file, "r", encoding="utf-8") as f: for line in f: if line.strip(): data = json.loads(line) prompts.append(data) return prompts def build_chromadb(prompts, chroma_dir, collection_name, embedding_model): print(f"[DEBUG] Connecting to ChromaDB at: {chroma_dir}, collection: {collection_name}") # Initialize ChromaDB client (local, no cloud) client = chromadb.Client(Settings(persist_directory=chroma_dir)) print(f"[DEBUG] Existing collections: {[c.name for c in client.list_collections()]}") # Remove old collection if exists if collection_name in [c.name for c in client.list_collections()]: print(f"[DEBUG] Deleting existing collection: {collection_name}") client.delete_collection(collection_name) collection = client.create_collection(collection_name) print(f"[DEBUG] Created collection: {collection_name}") # Load embedding model print(f"[DEBUG] Loading embedding model: {embedding_model}") model = SentenceTransformer(embedding_model) # Prepare data texts = [p["prompt"] for p in prompts] metadatas = [{"sql": p["sql"]} for p in prompts] ids = [str(i) for i in range(len(prompts))] print(f"[DEBUG] Number of prompts: {len(texts)}") # Compute embeddings print(f"[DEBUG] Computing embeddings...") embeddings = model.encode(texts, show_progress_bar=True, convert_to_numpy=True) # Add to ChromaDB print(f"[DEBUG] Adding embeddings to ChromaDB...") collection.add( embeddings=embeddings, documents=texts, metadatas=metadatas, ids=ids ) #client.persist() print(f"ChromaDB built with {len(prompts)} prompts at '{chroma_dir}' (collection: '{collection_name}')") def main(): if not os.path.exists(PROMPTS_FILE): print(f"File not found: {PROMPTS_FILE}") return prompts = load_prompts(PROMPTS_FILE) build_chromadb(prompts, CHROMA_DIR, COLLECTION_NAME, EMBEDDING_MODEL) if __name__ == "__main__": main()