kabeerahmad commited on
Commit
47d59d2
·
verified ·
1 Parent(s): 4136ef3

Update models/feature_extractor.py

Browse files
Files changed (1) hide show
  1. models/feature_extractor.py +91 -99
models/feature_extractor.py CHANGED
@@ -1,100 +1,92 @@
1
- {\rtf1\ansi\ansicpg1252\cocoartf2821
2
- \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;}
3
- {\colortbl;\red255\green255\blue255;}
4
- {\*\expandedcolortbl;;}
5
- \paperw11900\paperh16840\margl1440\margr1440\vieww11520\viewh8400\viewkind0
6
- \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0
7
 
8
- \f0\fs24 \cf0 # models/feature_extractor.py\
9
- \
10
- import os\
11
- import torch\
12
- import clip\
13
- from PIL import Image\
14
- import numpy as np\
15
- import cv2\
16
- import mediapipe as mp\
17
- import matplotlib.pyplot as plt\
18
- \
19
- # Set device and load CLIP model\
20
- device = "cuda" if torch.cuda.is_available() else "cpu"\
21
- model, preprocess = clip.load("ViT-B/32", device=device)\
22
- \
23
- def isolate_subject(pil_image: Image.Image) -> Image.Image:\
24
- """\
25
- Isolate the subject from the background using Mediapipe.\
26
- """\
27
- image_np = np.array(pil_image)\
28
- image_bgr = cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR)\
29
- mp_selfie_segmentation = mp.solutions.selfie_segmentation\
30
- with mp_selfie_segmentation.SelfieSegmentation(model_selection=1) as segmenter:\
31
- results = segmenter.process(cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB))\
32
- mask = results.segmentation_mask\
33
- condition = mask > 0.5 # Threshold segmentation\
34
- bg_color = (255, 255, 255)\
35
- output_image = np.where(condition[..., None], image_np, bg_color)\
36
- output_image = output_image.astype(np.uint8)\
37
- return Image.fromarray(output_image)\
38
- \
39
- def zero_shot_classification(image_features, candidate_texts):\
40
- """\
41
- Classify image features against candidate texts and return the best match.\
42
- """\
43
- text_tokens = clip.tokenize(candidate_texts).to(device)\
44
- with torch.no_grad():\
45
- text_features = model.encode_text(text_tokens)\
46
- text_features /= text_features.norm(dim=-1, keepdim=True)\
47
- similarities = (image_features @ text_features.T).squeeze(0).cpu().numpy()\
48
- best_idx = int(np.argmax(similarities))\
49
- return candidate_texts[best_idx]\
50
- \
51
- def extract_skin_tone(pil_image: Image.Image):\
52
- """\
53
- Extracts the average RGB color from the central region of the image.\
54
- """\
55
- image = np.array(pil_image)\
56
- h, w, _ = image.shape\
57
- x_start, y_start = w // 4, h // 4\
58
- x_end, y_end = x_start + w // 2, y_start + h // 2\
59
- face_region = image[y_start:y_end, x_start:x_end]\
60
- avg_color = face_region.mean(axis=(0, 1))\
61
- return tuple(int(c) for c in avg_color)\
62
- \
63
- def extract_features(image_path: str) -> dict:\
64
- """\
65
- Process the image and extract features including gender, body type, hair type, hair color,\
66
- eyeball color, glasses status, and skin tone.\
67
- """\
68
- pil_image = Image.open(image_path).convert("RGB")\
69
- isolated_image = isolate_subject(pil_image)\
70
- image_input = preprocess(isolated_image).unsqueeze(0).to(device)\
71
- with torch.no_grad():\
72
- image_features = model.encode_image(image_input)\
73
- image_features /= image_features.norm(dim=-1, keepdim=True)\
74
- \
75
- # Extract features using zero-shot classification\
76
- gender = zero_shot_classification(image_features, ["man", "woman"])\
77
- body_type = zero_shot_classification(image_features, ["slim", "moderate", "heavyset"])\
78
- hair_type = zero_shot_classification(image_features, ["bald", "short", "normal", "long"])\
79
- glasses = zero_shot_classification(image_features, ["wearing eyeglasses", "not wearing eyeglasses"])\
80
- glasses = "Yes" if "wearing" in glasses else "No"\
81
- eyeball_color = zero_shot_classification(image_features, ["blue", "brown", "green", "hazel"])\
82
- hair_color = zero_shot_classification(image_features, ["blonde", "brown", "black", "red"])\
83
- skin_tone = extract_skin_tone(isolated_image)\
84
- skin_tone_str = f"\{skin_tone[0]\},\{skin_tone[1]\},\{skin_tone[2]\}"\
85
- \
86
- return \{\
87
- "gender": gender,\
88
- "body_type": body_type,\
89
- "hair_type": hair_type,\
90
- "hair_color": hair_color,\
91
- "eyeball_color": eyeball_color,\
92
- "glasses": glasses,\
93
- "skin_tone": skin_tone_str,\
94
- \}\
95
- \
96
- if __name__ == "__main__":\
97
- test_image_path = "queen.jpeg" # Replace with a valid image path for local testing\
98
- features = extract_features(test_image_path)\
99
- print("Extracted features:", features)\
100
- }
 
1
+ # models/feature_extractor.py
 
 
 
 
 
2
 
3
+ import os
4
+ import torch
5
+ import clip
6
+ from PIL import Image
7
+ import numpy as np
8
+ import cv2
9
+ import mediapipe as mp
10
+ import matplotlib.pyplot as plt
11
+
12
+ # Set device and load CLIP model
13
+ device = "cuda" if torch.cuda.is_available() else "cpu"
14
+ model, preprocess = clip.load("ViT-B/32", device=device)
15
+
16
+ def isolate_subject(pil_image: Image.Image) -> Image.Image:
17
+ """
18
+ Isolate the subject from the background using Mediapipe.
19
+ """
20
+ image_np = np.array(pil_image)
21
+ image_bgr = cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR)
22
+ mp_selfie_segmentation = mp.solutions.selfie_segmentation
23
+ with mp_selfie_segmentation.SelfieSegmentation(model_selection=1) as segmenter:
24
+ results = segmenter.process(cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB))
25
+ mask = results.segmentation_mask
26
+ condition = mask > 0.5 # Threshold segmentation
27
+ bg_color = (255, 255, 255)
28
+ output_image = np.where(condition[..., None], image_np, bg_color)
29
+ output_image = output_image.astype(np.uint8)
30
+ return Image.fromarray(output_image)
31
+
32
+ def zero_shot_classification(image_features, candidate_texts):
33
+ """
34
+ Classify image features against candidate texts and return the best match.
35
+ """
36
+ text_tokens = clip.tokenize(candidate_texts).to(device)
37
+ with torch.no_grad():
38
+ text_features = model.encode_text(text_tokens)
39
+ text_features /= text_features.norm(dim=-1, keepdim=True)
40
+ similarities = (image_features @ text_features.T).squeeze(0).cpu().numpy()
41
+ best_idx = int(np.argmax(similarities))
42
+ return candidate_texts[best_idx]
43
+
44
+ def extract_skin_tone(pil_image: Image.Image):
45
+ """
46
+ Extracts the average RGB color from the central region of the image.
47
+ """
48
+ image = np.array(pil_image)
49
+ h, w, _ = image.shape
50
+ x_start, y_start = w // 4, h // 4
51
+ x_end, y_end = x_start + w // 2, y_start + h // 2
52
+ face_region = image[y_start:y_end, x_start:x_end]
53
+ avg_color = face_region.mean(axis=(0, 1))
54
+ return tuple(int(c) for c in avg_color)
55
+
56
+ def extract_features(image_path: str) -> dict:
57
+ """
58
+ Process the image and extract features including gender, body type, hair type, hair color,
59
+ eyeball color, glasses status, and skin tone.
60
+ """
61
+ pil_image = Image.open(image_path).convert("RGB")
62
+ isolated_image = isolate_subject(pil_image)
63
+ image_input = preprocess(isolated_image).unsqueeze(0).to(device)
64
+ with torch.no_grad():
65
+ image_features = model.encode_image(image_input)
66
+ image_features /= image_features.norm(dim=-1, keepdim=True)
67
+
68
+ # Extract features using zero-shot classification
69
+ gender = zero_shot_classification(image_features, ["man", "woman"])
70
+ body_type = zero_shot_classification(image_features, ["slim", "moderate", "heavyset"])
71
+ hair_type = zero_shot_classification(image_features, ["bald", "short", "normal", "long"])
72
+ glasses = zero_shot_classification(image_features, ["wearing eyeglasses", "not wearing eyeglasses"])
73
+ glasses = "Yes" if "wearing" in glasses else "No"
74
+ eyeball_color = zero_shot_classification(image_features, ["blue", "brown", "green", "hazel"])
75
+ hair_color = zero_shot_classification(image_features, ["blonde", "brown", "black", "red"])
76
+ skin_tone = extract_skin_tone(isolated_image)
77
+ skin_tone_str = f"{skin_tone[0]},{skin_tone[1]},{skin_tone[2]}"
78
+
79
+ return {
80
+ "gender": gender,
81
+ "body_type": body_type,
82
+ "hair_type": hair_type,
83
+ "hair_color": hair_color,
84
+ "eyeball_color": eyeball_color,
85
+ "glasses": glasses,
86
+ "skin_tone": skin_tone_str,
87
+ }
88
+
89
+ if __name__ == "__main__":
90
+ test_image_path = "queen.jpeg" # Replace with a valid image path for local testing
91
+ features = extract_features(test_image_path)
92
+ print("Extracted features:", features)