FF / app.py
Kaitoune's picture
Create app.py
a173c38 verified
Raw History Blame Contribute Delete
9.1 kB
import chromadb
from chromadb.utils import embedding_functions
import pandas as pd
import csv
import numpy as np
import gradio as gr
import os
# Instantiate chromadb instance. Data is stored in memory only.
# chroma_client = chromadb.Client()
# Instantiate chromadb instance. Data is stored on disk (a folder named 'my_vectordb' will be created in the same folder as this file).
chroma_client = chromadb.PersistentClient(path="my_vectordb")
collection_delete = chroma_client.list_collections()
for collection in collection_delete:
chroma_client.delete_collection(name=collection.name)
# Select the embedding model to use.
# List of model names can be found here https://www.sbert.net/docs/pretrained_models.html
sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name="BAAI/bge-m3")
# Create the collection vector
collection = []
collection_filename = []
def push_data(CSV_File):
# Create the collection, aka vector database. Or, if database already exist, then use it. Specify the model that we want to use to do the embedding.
collection_name = os.path.splitext(os.path.basename(CSV_File))[0]
if collection_name in collection_filename:
chroma_client.delete_collection(name=collection_name)
# Update the collections list to remove the deleted collection
collection[:] = [coll for coll in collection if coll.name != collection_name]
# Update the collection filenames list to remove the deleted collection's name
collection_filename[:] = [name for name in collection_filename if name != collection_name]
collection_tmp = chroma_client.create_collection(name=collection_name, embedding_function=sentence_transformer_ef)
with open(CSV_File) as file:
lines = csv.reader(file)
# Store the name of the menu items in this array. In Chroma, a "document" is a string i.e. name, sentence, paragraph, etc.
documents = []
# Store the corresponding menu item IDs in this array.
metadatas = []
# Each "document" needs a unique ID. This is like the primary key of a relational database. We'll start at 1 and increment from there.
ids = []
# Loop thru each line and populate the 3 arrays.
for i, line in enumerate(lines):
if i==0:
# Skip the first row (the column headers)
continue
documents.append(line[1] + ", " + line[3])
metadatas.append({"item_id": line[0], "name": line[1], "price": float(line[2])})
ids.append(line[0])
# Add all the data to the vector database. ChromaDB automatically converts and stores the text as vector embeddings. This may take a few minutes.
collection_tmp.add(documents=documents, metadatas=metadatas, ids=ids)
# Add to collection
collection.append(collection_tmp)
collection_filename.append(collection_name)
df = pd.read_csv(CSV_File.name, encoding='utf-8')
return df.to_csv(index=False), gr.Dropdown(choices=collection_filename, interactive=True), gr.Dropdown(choices=collection_filename, interactive=True)
def delete_data(filename):
is_delete = "Cannot find " + filename
if filename in collection_filename:
chroma_client.delete_collection(name=filename)
# Update the collections list to remove the deleted collection
collection[:] = [coll for coll in collection if coll.name != filename]
# Update the collection filenames list to remove the deleted collection's name
collection_filename[:] = [name for name in collection_filename if name != filename]
is_delete = "Deleted " + filename
return is_delete, gr.Dropdown(choices=collection_filename, interactive=True), gr.Dropdown(choices=collection_filename, interactive=True)
def result_for_semantic_search(Filename, Description, Input_constraint, Input_price1, Input_price2, Input_id):
# Input the description for semantic search
input_description = Description
# Input the price and its constraint for keyword search
input_constraint = Input_constraint
input_price1 = float(Input_price1)
input_price2 = float(Input_price2)
# Input the id for keyword search
input_id = Input_id
results = None
target_collection = None
for coll in collection:
if coll.name == Filename:
target_collection = coll
break
if target_collection is None:
return f"No collection found with the name {Filename}"
# Check if both description and price are provided
if input_id == "":
if input_constraint == "Higher":
query_condition = {"price": {"$gte": input_price1}}
elif input_constraint == "Lower":
query_condition = {"price": {"$lte": input_price1}}
elif input_constraint == "Between":
query_condition = {"$and": [{"price": {"$gte": min(input_price1, input_price2)}}, {"price": {"$lte": max(input_price1, input_price2)}}]}
elif input_constraint == "Equal":
query_condition = {"price": {"$eq": input_price1}}
else:
query_condition = {}
results = target_collection.query(
query_texts=[input_description],
n_results=3,
include=['documents', 'distances', 'metadatas'],
where=query_condition
)
else:
results = target_collection.query(
query_texts=[input_description],
n_results=3,
include=['documents', 'distances', 'metadatas'],
where={"item_id": {"$eq": input_id}}
)
# Process the results to the desired output format
formatted_results = []
for idx, metadata in enumerate(results['metadatas'][0]):
item_id = metadata['item_id']
price = metadata['price']
name = metadata['name']
document = results['documents'][0][idx]
distance = results['distances'][0][idx]
formatted_results.append(f"ID: {item_id}, Price: {price}, Name: {name}, Description: {document}, Distance: {distance}")
return '\n'.join(formatted_results)
demo = gr.Blocks()
with demo:
gr.Markdown("### Search engine.")
with gr.Tabs():
with gr.TabItem("Push data"):
gr.Markdown("""
### Instructions for Push Data
- Use this tab to upload your CSV file.
- Once the file is uploaded, you can move to the 'Search' tab to perform searches.
""")
with gr.Row():
file_input = gr.File()
push_data_button = gr.Button("Upload")
output_csv = gr.Textbox(label="Result")
with gr.TabItem("Delete"):
gr.Markdown("""
### Instructions for Delete
- Use this tab to delete your CSV file.
""")
with gr.Row():
file_delete = gr.Dropdown(label="File to delete", choices=collection_filename)
delete_data_button = gr.Button("Delete")
output_delete = gr.Textbox(label="Result")
with gr.TabItem("Search"):
gr.Markdown("""
### Instructions for Search
- **Keyword:** Provide text input for semantic search. If empty, it will be treated as "".
- **Constraint:** Can be one of the following:
- "Lower" for values less than the given price.
- "Higher" for values greater than the given price.
- "Equal" for values equal to the given price.
- "Between" for values between two given prices.
- "None" if no constraint is needed.
- **From price:** Input a float value as the first price.
- **To price:** Input a float value as the second price (only used if constraint is "Between").
- **Product ID:** Input the unique ID to search for an exact match. If empty, it will be treated as "".
""")
with gr.Row():
file_search = gr.Dropdown(label="File to search", choices=collection_filename)
description_input = gr.Textbox(label="Keyword")
constraint_input = gr.Dropdown(label="Price filter type", choices=["Between", "Lower", "Higher", "Equal", "None"])
price1 = gr.Number(label="From price")
price2 = gr.Number(label="To price")
id_input = gr.Textbox(label="Product ID", type="text")
search_button = gr.Button("Search")
output = gr.Textbox(label="Result")
push_data_button.click(push_data, inputs=file_input, outputs=[output_csv, file_delete, file_search])
delete_data_button.click(delete_data, inputs=file_delete, outputs=[output_delete, file_delete, file_search])
search_button.click(api_name="click", fn=result_for_semantic_search, inputs=[file_search, description_input, constraint_input, price1, price2, id_input], outputs=output)
demo.launch(
debug=True,
show_error=True,
share=True,
show_api=True,
max_threads=100
)