🎨 QuickDraw Classifier
Draw anything and let AI guess what it is! This app uses CLIP for zero-shot classification of your drawings across 345+ categories from Google's QuickDraw dataset.
import gradio as gr import torch import clip from PIL import Image import numpy as np import json from typing import List, Tuple import io import base64 # Load CLIP model device = "cuda" if torch.cuda.is_available() else "cpu" model, preprocess = clip.load("ViT-B/32", device=device) # Load categories def load_categories(): """Load drawing categories from categories.txt file""" try: with open("categories.txt", "r") as f: categories = [line.strip() for line in f.readlines()] return categories except FileNotFoundError: # Fallback to a subset of common categories if file not found return [ "airplane", "apple", "car", "cat", "dog", "flower", "house", "tree", "sun", "moon", "star", "bird", "fish", "butterfly", "bicycle", "boat" ] # Load categories CATEGORIES = load_categories() def preprocess_drawing(image): """Preprocess the drawing image for CLIP""" if image is None: return None print(image) # Convert to PIL Image if it's not already if isinstance(image, np.ndarray): image = Image.fromarray(image) # print(image) # Convert to RGB if it's not already if image.mode != 'RGB': image = image.convert('RGB') # Save the image to a temporary file for debugging image.save("temp_drawing.png") # Preprocess for CLIP return preprocess(image).unsqueeze(0).to(device) def classify_drawing(image, top_k: int = 5) -> List[Tuple[str, float]]: """ Classify a quickdraw-style drawing using CLIP zero-shot classification Args: image: PIL Image or numpy array of the drawing top_k: Number of top predictions to return Returns: List of tuples (category, confidence_score) """ if image is None: print("Here") return [] try: # Preprocess the image if isinstance(image, dict): print("Image is a dictionary") # return [] print(image) image_input = preprocess_drawing(image['composite']) else: image_input = preprocess_drawing(image) if image_input is None: # print("Here2") return [] # Create text prompts for all categories text_prompts = [f"a drawing of a {category}" for category in CATEGORIES] text_inputs = clip.tokenize(text_prompts).to(device) # Get features with torch.no_grad(): image_features = model.encode_image(image_input) text_features = model.encode_text(text_inputs) # Normalize features image_features = image_features / image_features.norm(dim=-1, keepdim=True) text_features = text_features / text_features.norm(dim=-1, keepdim=True) # Calculate similarities similarities = (100.0 * image_features @ text_features.T).softmax(dim=-1) values, indices = similarities[0].topk(top_k) # Format results results = [] for i in range(top_k): category = CATEGORIES[indices[i]] confidence = float(values[i]) results.append((category, confidence)) return results except Exception as e: print(f"Error in classification: {str(e)}") return [] def classify_drawing1(image, top_k: int = 5,word: str = "dog") -> List[Tuple[str, float]]: """ Classify a quickdraw-style drawing using CLIP zero-shot classification Args: image: PIL Image or numpy array of the drawing top_k: Number of top predictions to return Returns: List of tuples (category, confidence_score) """ if image is None: print("Here") return [] try: # Preprocess the image if isinstance(image, dict): print("Image is a dictionary") # return [] print(image) image_input = preprocess_drawing(image['composite']) else: image_input = preprocess_drawing(image) if image_input is None: # print("Here2") return [] if word.lower() not in [cat.lower() for cat in CATEGORIES]: CATEGORIES.append(word) # print(f"Added new category: {word}") # Create text prompts for all categories text_prompts = [f"a drawing of a {category}" for category in CATEGORIES] text_inputs = clip.tokenize(text_prompts).to(device) # Get features with torch.no_grad(): image_features = model.encode_image(image_input) text_features = model.encode_text(text_inputs) # Normalize features image_features = image_features / image_features.norm(dim=-1, keepdim=True) text_features = text_features / text_features.norm(dim=-1, keepdim=True) # Calculate similarities similarities = (100.0 * image_features @ text_features.T).softmax(dim=-1) values, indices = similarities[0].topk(top_k) # Format results results = [] for i in range(top_k): category = CATEGORIES[indices[i]] confidence = float(values[i]) results.append((category, confidence)) if word.lower() not in [cat.lower() for cat, _ in results]: # Find the original confidence score for the word word_idx = CATEGORIES.index(word) word_confidence = float(similarities[0][word_idx]) # Remove the last result if we have top_k results if len(results) >= top_k: results.pop() # Add the word with its original confidence score results.append((word, word_confidence)) return results except Exception as e: print(f"Error in classification: {str(e)}") return [] def gradio_classify(image): """Gradio interface function""" if image is None: return "Please draw something first!" results = classify_drawing(image, top_k=5) if not results: return "Sorry, couldn't classify the drawing. Please try again." # Format results for display output = "🎨 **Drawing Classification Results:**\n\n" for i, (category, confidence) in enumerate(results, 1): percentage = confidence * 100 output += f"{i}. **{category.title()}** - {percentage:.1f}%\n" return output # Create Gradio interface def create_interface(): """Create the Gradio interface with both drawing and API tabs""" with gr.Blocks( title="QuickDraw Classifier", theme=gr.themes.Soft(), css=""" .main-header { text-align: center; margin-bottom: 2rem; } .description { text-align: center; margin-bottom: 1.5rem; font-size: 1.1rem; } """ ) as demo: gr.HTML("""
Draw anything and let AI guess what it is! This app uses CLIP for zero-shot classification of your drawings across 345+ categories from Google's QuickDraw dataset.
• Draw simple, clear shapes
• Try common objects like animals, vehicles, or everyday items
• The AI knows 345+ categories including: airplane, cat, house, tree, car, flower, etc.
Test the API endpoint with base64 encoded images