Feature_Extraction / models /feature_extractor.py
kabeerahmad's picture
Upload feature_extractor.py
b8337ef verified
Raw History Blame Contribute Delete
3.67 kB
import os
import torch
import clip # Ensure you installed via: pip install git+https://github.com/openai/CLIP.git
from PIL import Image
import numpy as np
import cv2
import mediapipe as mp
import matplotlib.pyplot as plt
# Set device and load the CLIP model (CLIP automatically downloads the correct weights)
device = "cuda" if torch.cuda.is_available() else "cpu"
model, preprocess = clip.load("ViT-B/32", device=device)
def isolate_subject(pil_image: Image.Image) -> Image.Image:
"""
Isolates the subject from the background using Mediapipe.
"""
image_np = np.array(pil_image)
image_bgr = cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR)
mp_selfie_segmentation = mp.solutions.selfie_segmentation
with mp_selfie_segmentation.SelfieSegmentation(model_selection=1) as segmenter:
results = segmenter.process(cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB))
mask = results.segmentation_mask
condition = mask > 0.5 # Threshold segmentation
bg_color = (255, 255, 255)
output_image = np.where(condition[..., None], image_np, bg_color)
output_image = output_image.astype(np.uint8)
return Image.fromarray(output_image)
def zero_shot_classification(image_features, candidate_texts):
"""
Classify image features against candidate texts and return the best matching candidate.
"""
text_tokens = clip.tokenize(candidate_texts).to(device)
with torch.no_grad():
text_features = model.encode_text(text_tokens)
text_features /= text_features.norm(dim=-1, keepdim=True)
similarities = (image_features @ text_features.T).squeeze(0).cpu().numpy()
best_idx = int(np.argmax(similarities))
return candidate_texts[best_idx]
def extract_skin_tone(pil_image: Image.Image):
"""
Extracts the average RGB color from the central region of the image.
"""
image = np.array(pil_image)
h, w, _ = image.shape
x_start, y_start = w // 4, h // 4
x_end, y_end = x_start + w // 2, y_start + h // 2
face_region = image[y_start:y_end, x_start:x_end]
avg_color = face_region.mean(axis=(0, 1))
return tuple(int(c) for c in avg_color)
def extract_features(image_path: str) -> dict:
"""
Process the image and extract features including:
- Gender, Body Type, Hair Type, Hair Color,
- Eyeball Color, Glasses status, and Skin Tone.
"""
pil_image = Image.open(image_path).convert("RGB")
isolated_image = isolate_subject(pil_image)
image_input = preprocess(isolated_image).unsqueeze(0).to(device)
with torch.no_grad():
image_features = model.encode_image(image_input)
image_features /= image_features.norm(dim=-1, keepdim=True)
gender = zero_shot_classification(image_features, ["man", "woman"])
body_type = zero_shot_classification(image_features, ["slim", "moderate", "heavyset"])
hair_type = zero_shot_classification(image_features, ["bald", "short", "normal", "long"])
glasses = zero_shot_classification(image_features, ["wearing eyeglasses", "not wearing eyeglasses"])
glasses = "Yes" if glasses == "wearing eyeglasses" else "No"
eyeball_color = zero_shot_classification(image_features, ["blue", "brown", "green", "hazel"])
hair_color = zero_shot_classification(image_features, ["blonde", "brown", "black", "red"])
skin_tone = extract_skin_tone(isolated_image)
skin_tone_str = f"{skin_tone[0]},{skin_tone[1]},{skin_tone[2]}"
return {
"gender": gender,
"body_type": body_type,
"hair_type": hair_type,
"hair_color": hair_color,
"eyeball_color": eyeball_color,
"glasses": glasses,
"skin_tone": skin_tone_str,
}