Download app.py from Kaitoune/FF: direct link, hf CLI and curl.
- Browser
- Download file 9.1 kB
-
https://huggingface.co/spaces/Kaitoune/FF/resolve/main/app.py
- Command line
-
hf download hf://spaces/Kaitoune/FF/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Kaitoune/FF/resolve/main/app.py
9.1 kB
| import chromadb | |
| from chromadb.utils import embedding_functions | |
| import pandas as pd | |
| import csv | |
| import numpy as np | |
| import gradio as gr | |
| import os | |
| # Instantiate chromadb instance. Data is stored in memory only. | |
| # chroma_client = chromadb.Client() | |
| # Instantiate chromadb instance. Data is stored on disk (a folder named 'my_vectordb' will be created in the same folder as this file). | |
| chroma_client = chromadb.PersistentClient(path="my_vectordb") | |
| collection_delete = chroma_client.list_collections() | |
| for collection in collection_delete: | |
| chroma_client.delete_collection(name=collection.name) | |
| # Select the embedding model to use. | |
| # List of model names can be found here https://www.sbert.net/docs/pretrained_models.html | |
| sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name="BAAI/bge-m3") | |
| # Create the collection vector | |
| collection = [] | |
| collection_filename = [] | |
| def push_data(CSV_File): | |
| # Create the collection, aka vector database. Or, if database already exist, then use it. Specify the model that we want to use to do the embedding. | |
| collection_name = os.path.splitext(os.path.basename(CSV_File))[0] | |
| if collection_name in collection_filename: | |
| chroma_client.delete_collection(name=collection_name) | |
| # Update the collections list to remove the deleted collection | |
| collection[:] = [coll for coll in collection if coll.name != collection_name] | |
| # Update the collection filenames list to remove the deleted collection's name | |
| collection_filename[:] = [name for name in collection_filename if name != collection_name] | |
| collection_tmp = chroma_client.create_collection(name=collection_name, embedding_function=sentence_transformer_ef) | |
| with open(CSV_File) as file: | |
| lines = csv.reader(file) | |
| # Store the name of the menu items in this array. In Chroma, a "document" is a string i.e. name, sentence, paragraph, etc. | |
| documents = [] | |
| # Store the corresponding menu item IDs in this array. | |
| metadatas = [] | |
| # Each "document" needs a unique ID. This is like the primary key of a relational database. We'll start at 1 and increment from there. | |
| ids = [] | |
| # Loop thru each line and populate the 3 arrays. | |
| for i, line in enumerate(lines): | |
| if i==0: | |
| # Skip the first row (the column headers) | |
| continue | |
| documents.append(line[1] + ", " + line[3]) | |
| metadatas.append({"item_id": line[0], "name": line[1], "price": float(line[2])}) | |
| ids.append(line[0]) | |
| # Add all the data to the vector database. ChromaDB automatically converts and stores the text as vector embeddings. This may take a few minutes. | |
| collection_tmp.add(documents=documents, metadatas=metadatas, ids=ids) | |
| # Add to collection | |
| collection.append(collection_tmp) | |
| collection_filename.append(collection_name) | |
| df = pd.read_csv(CSV_File.name, encoding='utf-8') | |
| return df.to_csv(index=False), gr.Dropdown(choices=collection_filename, interactive=True), gr.Dropdown(choices=collection_filename, interactive=True) | |
| def delete_data(filename): | |
| is_delete = "Cannot find " + filename | |
| if filename in collection_filename: | |
| chroma_client.delete_collection(name=filename) | |
| # Update the collections list to remove the deleted collection | |
| collection[:] = [coll for coll in collection if coll.name != filename] | |
| # Update the collection filenames list to remove the deleted collection's name | |
| collection_filename[:] = [name for name in collection_filename if name != filename] | |
| is_delete = "Deleted " + filename | |
| return is_delete, gr.Dropdown(choices=collection_filename, interactive=True), gr.Dropdown(choices=collection_filename, interactive=True) | |
| def result_for_semantic_search(Filename, Description, Input_constraint, Input_price1, Input_price2, Input_id): | |
| # Input the description for semantic search | |
| input_description = Description | |
| # Input the price and its constraint for keyword search | |
| input_constraint = Input_constraint | |
| input_price1 = float(Input_price1) | |
| input_price2 = float(Input_price2) | |
| # Input the id for keyword search | |
| input_id = Input_id | |
| results = None | |
| target_collection = None | |
| for coll in collection: | |
| if coll.name == Filename: | |
| target_collection = coll | |
| break | |
| if target_collection is None: | |
| return f"No collection found with the name {Filename}" | |
| # Check if both description and price are provided | |
| if input_id == "": | |
| if input_constraint == "Higher": | |
| query_condition = {"price": {"$gte": input_price1}} | |
| elif input_constraint == "Lower": | |
| query_condition = {"price": {"$lte": input_price1}} | |
| elif input_constraint == "Between": | |
| query_condition = {"$and": [{"price": {"$gte": min(input_price1, input_price2)}}, {"price": {"$lte": max(input_price1, input_price2)}}]} | |
| elif input_constraint == "Equal": | |
| query_condition = {"price": {"$eq": input_price1}} | |
| else: | |
| query_condition = {} | |
| results = target_collection.query( | |
| query_texts=[input_description], | |
| n_results=3, | |
| include=['documents', 'distances', 'metadatas'], | |
| where=query_condition | |
| ) | |
| else: | |
| results = target_collection.query( | |
| query_texts=[input_description], | |
| n_results=3, | |
| include=['documents', 'distances', 'metadatas'], | |
| where={"item_id": {"$eq": input_id}} | |
| ) | |
| # Process the results to the desired output format | |
| formatted_results = [] | |
| for idx, metadata in enumerate(results['metadatas'][0]): | |
| item_id = metadata['item_id'] | |
| price = metadata['price'] | |
| name = metadata['name'] | |
| document = results['documents'][0][idx] | |
| distance = results['distances'][0][idx] | |
| formatted_results.append(f"ID: {item_id}, Price: {price}, Name: {name}, Description: {document}, Distance: {distance}") | |
| return '\n'.join(formatted_results) | |
| demo = gr.Blocks() | |
| with demo: | |
| gr.Markdown("### Search engine.") | |
| with gr.Tabs(): | |
| with gr.TabItem("Push data"): | |
| gr.Markdown(""" | |
| ### Instructions for Push Data | |
| - Use this tab to upload your CSV file. | |
| - Once the file is uploaded, you can move to the 'Search' tab to perform searches. | |
| """) | |
| with gr.Row(): | |
| file_input = gr.File() | |
| push_data_button = gr.Button("Upload") | |
| output_csv = gr.Textbox(label="Result") | |
| with gr.TabItem("Delete"): | |
| gr.Markdown(""" | |
| ### Instructions for Delete | |
| - Use this tab to delete your CSV file. | |
| """) | |
| with gr.Row(): | |
| file_delete = gr.Dropdown(label="File to delete", choices=collection_filename) | |
| delete_data_button = gr.Button("Delete") | |
| output_delete = gr.Textbox(label="Result") | |
| with gr.TabItem("Search"): | |
| gr.Markdown(""" | |
| ### Instructions for Search | |
| - **Keyword:** Provide text input for semantic search. If empty, it will be treated as "". | |
| - **Constraint:** Can be one of the following: | |
| - "Lower" for values less than the given price. | |
| - "Higher" for values greater than the given price. | |
| - "Equal" for values equal to the given price. | |
| - "Between" for values between two given prices. | |
| - "None" if no constraint is needed. | |
| - **From price:** Input a float value as the first price. | |
| - **To price:** Input a float value as the second price (only used if constraint is "Between"). | |
| - **Product ID:** Input the unique ID to search for an exact match. If empty, it will be treated as "". | |
| """) | |
| with gr.Row(): | |
| file_search = gr.Dropdown(label="File to search", choices=collection_filename) | |
| description_input = gr.Textbox(label="Keyword") | |
| constraint_input = gr.Dropdown(label="Price filter type", choices=["Between", "Lower", "Higher", "Equal", "None"]) | |
| price1 = gr.Number(label="From price") | |
| price2 = gr.Number(label="To price") | |
| id_input = gr.Textbox(label="Product ID", type="text") | |
| search_button = gr.Button("Search") | |
| output = gr.Textbox(label="Result") | |
| push_data_button.click(push_data, inputs=file_input, outputs=[output_csv, file_delete, file_search]) | |
| delete_data_button.click(delete_data, inputs=file_delete, outputs=[output_delete, file_delete, file_search]) | |
| search_button.click(api_name="click", fn=result_for_semantic_search, inputs=[file_search, description_input, constraint_input, price1, price2, id_input], outputs=output) | |
| demo.launch( | |
| debug=True, | |
| show_error=True, | |
| share=True, | |
| show_api=True, | |
| max_threads=100 | |
| ) |