import chromadb from chromadb.utils import embedding_functions import pandas as pd import csv import numpy as np import gradio as gr import os # Instantiate chromadb instance. Data is stored in memory only. # chroma_client = chromadb.Client() # Instantiate chromadb instance. Data is stored on disk (a folder named 'my_vectordb' will be created in the same folder as this file). chroma_client = chromadb.PersistentClient(path="my_vectordb") collection_delete = chroma_client.list_collections() for collection in collection_delete: chroma_client.delete_collection(name=collection.name) # Select the embedding model to use. # List of model names can be found here https://www.sbert.net/docs/pretrained_models.html sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name="BAAI/bge-m3") # Create the collection vector collection = [] collection_filename = [] def push_data(CSV_File): # Create the collection, aka vector database. Or, if database already exist, then use it. Specify the model that we want to use to do the embedding. collection_name = os.path.splitext(os.path.basename(CSV_File))[0] if collection_name in collection_filename: chroma_client.delete_collection(name=collection_name) # Update the collections list to remove the deleted collection collection[:] = [coll for coll in collection if coll.name != collection_name] # Update the collection filenames list to remove the deleted collection's name collection_filename[:] = [name for name in collection_filename if name != collection_name] collection_tmp = chroma_client.create_collection(name=collection_name, embedding_function=sentence_transformer_ef) with open(CSV_File) as file: lines = csv.reader(file) # Store the name of the menu items in this array. In Chroma, a "document" is a string i.e. name, sentence, paragraph, etc. documents = [] # Store the corresponding menu item IDs in this array. metadatas = [] # Each "document" needs a unique ID. This is like the primary key of a relational database. We'll start at 1 and increment from there. ids = [] # Loop thru each line and populate the 3 arrays. for i, line in enumerate(lines): if i==0: # Skip the first row (the column headers) continue documents.append(line[1] + ", " + line[3]) metadatas.append({"item_id": line[0], "name": line[1], "price": float(line[2])}) ids.append(line[0]) # Add all the data to the vector database. ChromaDB automatically converts and stores the text as vector embeddings. This may take a few minutes. collection_tmp.add(documents=documents, metadatas=metadatas, ids=ids) # Add to collection collection.append(collection_tmp) collection_filename.append(collection_name) df = pd.read_csv(CSV_File.name, encoding='utf-8') return df.to_csv(index=False), gr.Dropdown(choices=collection_filename, interactive=True), gr.Dropdown(choices=collection_filename, interactive=True) def delete_data(filename): is_delete = "Cannot find " + filename if filename in collection_filename: chroma_client.delete_collection(name=filename) # Update the collections list to remove the deleted collection collection[:] = [coll for coll in collection if coll.name != filename] # Update the collection filenames list to remove the deleted collection's name collection_filename[:] = [name for name in collection_filename if name != filename] is_delete = "Deleted " + filename return is_delete, gr.Dropdown(choices=collection_filename, interactive=True), gr.Dropdown(choices=collection_filename, interactive=True) def result_for_semantic_search(Filename, Description, Input_constraint, Input_price1, Input_price2, Input_id): # Input the description for semantic search input_description = Description # Input the price and its constraint for keyword search input_constraint = Input_constraint input_price1 = float(Input_price1) input_price2 = float(Input_price2) # Input the id for keyword search input_id = Input_id results = None target_collection = None for coll in collection: if coll.name == Filename: target_collection = coll break if target_collection is None: return f"No collection found with the name {Filename}" # Check if both description and price are provided if input_id == "": if input_constraint == "Higher": query_condition = {"price": {"$gte": input_price1}} elif input_constraint == "Lower": query_condition = {"price": {"$lte": input_price1}} elif input_constraint == "Between": query_condition = {"$and": [{"price": {"$gte": min(input_price1, input_price2)}}, {"price": {"$lte": max(input_price1, input_price2)}}]} elif input_constraint == "Equal": query_condition = {"price": {"$eq": input_price1}} else: query_condition = {} results = target_collection.query( query_texts=[input_description], n_results=3, include=['documents', 'distances', 'metadatas'], where=query_condition ) else: results = target_collection.query( query_texts=[input_description], n_results=3, include=['documents', 'distances', 'metadatas'], where={"item_id": {"$eq": input_id}} ) # Process the results to the desired output format formatted_results = [] for idx, metadata in enumerate(results['metadatas'][0]): item_id = metadata['item_id'] price = metadata['price'] name = metadata['name'] document = results['documents'][0][idx] distance = results['distances'][0][idx] formatted_results.append(f"ID: {item_id}, Price: {price}, Name: {name}, Description: {document}, Distance: {distance}") return '\n'.join(formatted_results) demo = gr.Blocks() with demo: gr.Markdown("### Search engine.") with gr.Tabs(): with gr.TabItem("Push data"): gr.Markdown(""" ### Instructions for Push Data - Use this tab to upload your CSV file. - Once the file is uploaded, you can move to the 'Search' tab to perform searches. """) with gr.Row(): file_input = gr.File() push_data_button = gr.Button("Upload") output_csv = gr.Textbox(label="Result") with gr.TabItem("Delete"): gr.Markdown(""" ### Instructions for Delete - Use this tab to delete your CSV file. """) with gr.Row(): file_delete = gr.Dropdown(label="File to delete", choices=collection_filename) delete_data_button = gr.Button("Delete") output_delete = gr.Textbox(label="Result") with gr.TabItem("Search"): gr.Markdown(""" ### Instructions for Search - **Keyword:** Provide text input for semantic search. If empty, it will be treated as "". - **Constraint:** Can be one of the following: - "Lower" for values less than the given price. - "Higher" for values greater than the given price. - "Equal" for values equal to the given price. - "Between" for values between two given prices. - "None" if no constraint is needed. - **From price:** Input a float value as the first price. - **To price:** Input a float value as the second price (only used if constraint is "Between"). - **Product ID:** Input the unique ID to search for an exact match. If empty, it will be treated as "". """) with gr.Row(): file_search = gr.Dropdown(label="File to search", choices=collection_filename) description_input = gr.Textbox(label="Keyword") constraint_input = gr.Dropdown(label="Price filter type", choices=["Between", "Lower", "Higher", "Equal", "None"]) price1 = gr.Number(label="From price") price2 = gr.Number(label="To price") id_input = gr.Textbox(label="Product ID", type="text") search_button = gr.Button("Search") output = gr.Textbox(label="Result") push_data_button.click(push_data, inputs=file_input, outputs=[output_csv, file_delete, file_search]) delete_data_button.click(delete_data, inputs=file_delete, outputs=[output_delete, file_delete, file_search]) search_button.click(api_name="click", fn=result_for_semantic_search, inputs=[file_search, description_input, constraint_input, price1, price2, id_input], outputs=output) demo.launch( debug=True, show_error=True, share=True, show_api=True, max_threads=100 )