Spaces:
Build error
Build error
File size: 2,711 Bytes
eca6f5e a8d5aa3 256fe2d aeee918 a8d5aa3 aeee918 501796b aeee918 eca6f5e 64aad0d a8d5aa3 64aad0d a8d5aa3 61e0bab 64aad0d 9e462d0 64aad0d 6ab18b3 64aad0d 9e462d0 64aad0d a8d5aa3 64aad0d a8d5aa3 64aad0d a8d5aa3 64aad0d a8d5aa3 61e0bab 64aad0d 61e0bab 64aad0d 61e0bab 64aad0d a8d5aa3 64aad0d a8d5aa3 501796b b795379 501796b b795379 e4f5b29 b795379 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | from dotenv import load_dotenv
from pinecone import Pinecone
from transformers import AutoTokenizer, AutoModel
import torch
import pandas as pd
import pandas.testing as pdt
import boto3
import os
from io import StringIO
from gradio_client import Client
load_dotenv()
MODEL_NAME = "sentence-transformers/all-mpnet-base-v2"
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
model = AutoModel.from_pretrained(MODEL_NAME)
pc = Pinecone(api_key=os.environ["PINECONE_KEY"])
index = pc.Index("product-recommender")
def compare_s3_to_setupfiles(file_name: str) -> bool:
expected_table = pd.read_csv("setupfiles/test_" + file_name)
actual_table = pd.read_csv(
StringIO(
boto3.client("s3")
.get_object(Bucket="charityshop-media-assistant", Key=file_name)["Body"]
.read()
.decode("utf-8")
)
)
actual_test_table = actual_table[
actual_table.store_name == "TestStore"
].reset_index(drop=True)
return pdt.assert_frame_equal(expected_table, actual_test_table)
def text_to_embedding(text):
inputs = tokenizer(text, padding=True, truncation=True, return_tensors="pt")
with torch.no_grad():
outputs = model(**inputs)
embeddings = outputs.last_hidden_state.mean(dim=1).numpy()
return embeddings.flatten().tolist()
def find_similar_media(query_text, top_k=1):
query_embedding = text_to_embedding(query_text)
matches = index.query(
vector=query_embedding,
top_k=top_k,
include_metadata=True,
filter={"vector_type": "test"},
)["matches"]
df = pd.DataFrame(
{"barcode": [m["id"] for m in matches], "score": [m["score"] for m in matches]}
)
for col in ["artist", "title", "media_type", "description", "vector_type"]:
df[col] = [m["metadata"][col] for m in matches]
return df
def test_InventoryDatabaseMatchesSampleRecords():
compare_s3_to_setupfiles("inventory.csv")
def test_ChatHistoryDatabaseMatchesSampleRecords():
compare_s3_to_setupfiles("chat_history.csv")
def test_VectorDatabaseMatchesSampleRecords():
expected_table = pd.read_csv(
"setupfiles/test_vector_database_query.csv", dtype={"barcode": str}
)
actual_test_table = find_similar_media("denver", top_k=5).drop(columns=["score"])
pdt.assert_frame_equal(expected_table, actual_test_table)
def test_CanQueryLLM():
current_site = "media_assistant"
client = Client("ml-usecases/" + current_site, hf_token=os.environ["HF_TOKEN"])
try:
result = client.predict(
user_query="Hello!!", api_name="/combined_recommendation"
)
return len(str(result)) > 0
except:
return False
|