talk2data / build_chromadb.py
prithvi55's picture
adding files
d1a818a verified
Raw History Blame Contribute Delete
2.49 kB
import json
import chromadb
from chromadb.config import Settings
from sentence_transformers import SentenceTransformer
import os
PROMPTS_FILE = "prompts.jsonl"
CHROMA_DIR = "chromadb_local"
COLLECTION_NAME = "prompts"
EMBEDDING_MODEL = "all-MiniLM-L6-v2" # You can change to another local model if desired
def load_prompts(prompts_file):
prompts = []
with open(prompts_file, "r", encoding="utf-8") as f:
for line in f:
if line.strip():
data = json.loads(line)
prompts.append(data)
return prompts
def build_chromadb(prompts, chroma_dir, collection_name, embedding_model):
print(f"[DEBUG] Connecting to ChromaDB at: {chroma_dir}, collection: {collection_name}")
# Initialize ChromaDB client (local, no cloud)
client = chromadb.Client(Settings(persist_directory=chroma_dir))
print(f"[DEBUG] Existing collections: {[c.name for c in client.list_collections()]}")
# Remove old collection if exists
if collection_name in [c.name for c in client.list_collections()]:
print(f"[DEBUG] Deleting existing collection: {collection_name}")
client.delete_collection(collection_name)
collection = client.create_collection(collection_name)
print(f"[DEBUG] Created collection: {collection_name}")
# Load embedding model
print(f"[DEBUG] Loading embedding model: {embedding_model}")
model = SentenceTransformer(embedding_model)
# Prepare data
texts = [p["prompt"] for p in prompts]
metadatas = [{"sql": p["sql"]} for p in prompts]
ids = [str(i) for i in range(len(prompts))]
print(f"[DEBUG] Number of prompts: {len(texts)}")
# Compute embeddings
print(f"[DEBUG] Computing embeddings...")
embeddings = model.encode(texts, show_progress_bar=True, convert_to_numpy=True)
# Add to ChromaDB
print(f"[DEBUG] Adding embeddings to ChromaDB...")
collection.add(
embeddings=embeddings,
documents=texts,
metadatas=metadatas,
ids=ids
)
#client.persist()
print(f"ChromaDB built with {len(prompts)} prompts at '{chroma_dir}' (collection: '{collection_name}')")
def main():
if not os.path.exists(PROMPTS_FILE):
print(f"File not found: {PROMPTS_FILE}")
return
prompts = load_prompts(PROMPTS_FILE)
build_chromadb(prompts, CHROMA_DIR, COLLECTION_NAME, EMBEDDING_MODEL)
if __name__ == "__main__":
main()