import torch from pinecone import Pinecone from torchvision.transforms import Compose, Resize, ToTensor, Normalize from PIL import Image from transformers import CLIPProcessor, CLIPModel # Initialize the Pinecone client pc = Pinecone( api_key="pcsk_6w79Cx_MfeLRKxmSKerQhKiyrjZieguMKdiSVV1M3no8fTzAygLT8Qpx8F7AS7iMSzUEX1", environment="us-east-1" # Ensure this matches your region ) # Connect to the existing index index_name = "multimodal-search" if index_name not in pc.list_indexes().names(): raise ValueError(f"Index {index_name} does not exist. Please create it in your Pinecone dashboard.") index = pc.Index(index_name) # Initialize CLIP model and processor device = "cuda" if torch.cuda.is_available() else "cpu" model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device) processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") # Define global image transformation pipeline transform = Compose([ Resize((224, 224)), ToTensor(), Normalize(mean=(0.481, 0.457, 0.408), std=(0.268, 0.261, 0.275)) # CLIP-specific normalization ]) def query_pinecone_with_filters(query, mode="text-to-image", top_k=5, filter_metadata=None): """ Perform text-to-image or image-to-image search with metadata filtering using Pinecone and CLIP. Args: query: String (for text-to-image) or file path to image (for image-to-image). mode: "text-to-image" or "image-to-image". top_k: Number of top results to retrieve. filter_metadata: Dictionary of filters to apply on metadata (e.g., {"type": "image", "captions": "dog"}). Returns: List of matches with metadata. """ if mode == "text-to-image": # Convert text query to embedding text_inputs = processor(text=[query], return_tensors="pt", padding=True).to(device) query_embedding = model.get_text_features(**text_inputs) query_embedding = query_embedding / query_embedding.norm(dim=-1, keepdim=True) # Normalize elif mode == "image-to-image": # Convert image to embedding query_image = Image.open(query).convert("RGB") image_tensor = transform(query_image).unsqueeze(0).to(device) query_embedding = model.get_image_features(image_tensor) query_embedding = query_embedding / query_embedding.norm(dim=-1, keepdim=True) # Normalize else: raise ValueError("Invalid mode. Choose 'text-to-image' or 'image-to-image'.") # Query Pinecone query_vector = query_embedding.cpu().detach().numpy().flatten().tolist() results = index.query( vector=query_vector, top_k=top_k, include_metadata=True, filter=filter_metadata ) # Update image paths to use URLs for match in results["matches"]: image_filename = match["metadata"]["image_path"].split("\\")[-1] # Extract filename match["metadata"]["image_path"] = f"https://Dev24910-VisionVerse-Search.hf.space/file/workspace/images/{image_filename}" return results["matches"]