import os import torch import clip # Ensure you installed via: pip install git+https://github.com/openai/CLIP.git from PIL import Image import numpy as np import cv2 import mediapipe as mp import matplotlib.pyplot as plt # Set device and load the CLIP model (CLIP automatically downloads the correct weights) device = "cuda" if torch.cuda.is_available() else "cpu" model, preprocess = clip.load("ViT-B/32", device=device) def isolate_subject(pil_image: Image.Image) -> Image.Image: """ Isolates the subject from the background using Mediapipe. """ image_np = np.array(pil_image) image_bgr = cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR) mp_selfie_segmentation = mp.solutions.selfie_segmentation with mp_selfie_segmentation.SelfieSegmentation(model_selection=1) as segmenter: results = segmenter.process(cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)) mask = results.segmentation_mask condition = mask > 0.5 # Threshold segmentation bg_color = (255, 255, 255) output_image = np.where(condition[..., None], image_np, bg_color) output_image = output_image.astype(np.uint8) return Image.fromarray(output_image) def zero_shot_classification(image_features, candidate_texts): """ Classify image features against candidate texts and return the best matching candidate. """ text_tokens = clip.tokenize(candidate_texts).to(device) with torch.no_grad(): text_features = model.encode_text(text_tokens) text_features /= text_features.norm(dim=-1, keepdim=True) similarities = (image_features @ text_features.T).squeeze(0).cpu().numpy() best_idx = int(np.argmax(similarities)) return candidate_texts[best_idx] def extract_skin_tone(pil_image: Image.Image): """ Extracts the average RGB color from the central region of the image. """ image = np.array(pil_image) h, w, _ = image.shape x_start, y_start = w // 4, h // 4 x_end, y_end = x_start + w // 2, y_start + h // 2 face_region = image[y_start:y_end, x_start:x_end] avg_color = face_region.mean(axis=(0, 1)) return tuple(int(c) for c in avg_color) def extract_features(image_path: str) -> dict: """ Process the image and extract features including: - Gender, Body Type, Hair Type, Hair Color, - Eyeball Color, Glasses status, and Skin Tone. """ pil_image = Image.open(image_path).convert("RGB") isolated_image = isolate_subject(pil_image) image_input = preprocess(isolated_image).unsqueeze(0).to(device) with torch.no_grad(): image_features = model.encode_image(image_input) image_features /= image_features.norm(dim=-1, keepdim=True) gender = zero_shot_classification(image_features, ["man", "woman"]) body_type = zero_shot_classification(image_features, ["slim", "moderate", "heavyset"]) hair_type = zero_shot_classification(image_features, ["bald", "short", "normal", "long"]) glasses = zero_shot_classification(image_features, ["wearing eyeglasses", "not wearing eyeglasses"]) glasses = "Yes" if glasses == "wearing eyeglasses" else "No" eyeball_color = zero_shot_classification(image_features, ["blue", "brown", "green", "hazel"]) hair_color = zero_shot_classification(image_features, ["blonde", "brown", "black", "red"]) skin_tone = extract_skin_tone(isolated_image) skin_tone_str = f"{skin_tone[0]},{skin_tone[1]},{skin_tone[2]}" return { "gender": gender, "body_type": body_type, "hair_type": hair_type, "hair_color": hair_color, "eyeball_color": eyeball_color, "glasses": glasses, "skin_tone": skin_tone_str, }