File size: 5,197 Bytes
c280886 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 | """Model wrappers: fashion embeddings (open_clip) and clothes segmentation.
Both wrappers import torch lazily so the rest of the package (and the test
suite) runs without it. Anything with the same methods can be swapped in,
which is how the tests use small fakes.
"""
from __future__ import annotations
import os
from typing import Protocol, Sequence
import numpy as np
from PIL import Image
DEFAULT_EMBED_MODEL = "hf-hub:Marqo/marqo-fashionSigLIP"
DEFAULT_SEG_MODEL = "mattmdjaga/segformer_b2_clothes"
class Embedder(Protocol):
name: str
dim: int
def embed_images(self, images: Sequence[Image.Image]) -> np.ndarray: ...
def embed_texts(self, texts: Sequence[str]) -> np.ndarray: ...
class Segmenter(Protocol):
def segment(self, image: Image.Image) -> np.ndarray:
"""Return an (H, W) uint8 label map aligned with ``image``."""
...
def l2_normalize(x: np.ndarray, axis: int = -1) -> np.ndarray:
x = np.asarray(x, dtype=np.float32)
norm = np.linalg.norm(x, axis=axis, keepdims=True)
return x / np.maximum(norm, 1e-12)
def _pick_device(device: str | None) -> str:
if device:
return device
try:
import torch
return "cuda" if torch.cuda.is_available() else "cpu"
except Exception: # pragma: no cover - torch missing
return "cpu"
def _set_threads() -> None:
try:
import torch
torch.set_num_threads(max(1, os.cpu_count() or 1))
except Exception: # pragma: no cover
pass
class OpenClipEmbedder:
"""Image/text embeddings from any open_clip model.
``spec`` is either an ``hf-hub:`` name (e.g. the default Marqo
FashionSigLIP) or ``"<architecture>|<pretrained tag>"`` such as
``"ViT-B-16-SigLIP|webli"`` for the generic baselines in the benchmark.
"""
def __init__(self, spec: str = DEFAULT_EMBED_MODEL, device: str | None = None, batch_size: int = 32):
import open_clip
import torch
_set_threads()
self.name = spec
self.device = _pick_device(device)
self.batch_size = batch_size
if "|" in spec:
arch, pretrained = spec.split("|", 1)
model, _, preprocess = open_clip.create_model_and_transforms(arch, pretrained=pretrained)
tokenizer = open_clip.get_tokenizer(arch)
else:
model, _, preprocess = open_clip.create_model_and_transforms(spec)
tokenizer = open_clip.get_tokenizer(spec)
self.model = model.to(self.device).eval()
self.preprocess = preprocess
self.tokenizer = tokenizer
self._torch = torch
with torch.inference_mode():
probe = self.model.encode_text(self.tokenizer(["probe"]).to(self.device))
self.dim = int(probe.shape[-1])
def embed_images(self, images: Sequence[Image.Image]) -> np.ndarray:
torch = self._torch
out: list[np.ndarray] = []
for start in range(0, len(images), self.batch_size):
batch = [self.preprocess(im.convert("RGB")) for im in images[start : start + self.batch_size]]
if not batch:
continue
x = torch.stack(batch).to(self.device)
with torch.inference_mode():
feats = self.model.encode_image(x)
out.append(feats.float().cpu().numpy())
if not out:
return np.zeros((0, self.dim), dtype=np.float32)
return l2_normalize(np.concatenate(out))
def embed_texts(self, texts: Sequence[str]) -> np.ndarray:
torch = self._torch
out: list[np.ndarray] = []
for start in range(0, len(texts), self.batch_size):
tokens = self.tokenizer(list(texts[start : start + self.batch_size])).to(self.device)
with torch.inference_mode():
feats = self.model.encode_text(tokens)
out.append(feats.float().cpu().numpy())
if not out:
return np.zeros((0, self.dim), dtype=np.float32)
return l2_normalize(np.concatenate(out))
class SegformerClothesSegmenter:
"""Pixel-level clothes parsing with a SegFormer fine-tuned on ATR."""
def __init__(self, model_id: str = DEFAULT_SEG_MODEL, device: str | None = None, model=None, processor=None):
import torch
from transformers import AutoModelForSemanticSegmentation, SegformerImageProcessor
_set_threads()
self.device = _pick_device(device)
self.processor = processor or SegformerImageProcessor.from_pretrained(model_id)
self.model = (model or AutoModelForSemanticSegmentation.from_pretrained(model_id)).to(self.device).eval()
self._torch = torch
def segment(self, image: Image.Image) -> np.ndarray:
torch = self._torch
image = image.convert("RGB")
inputs = self.processor(images=image, return_tensors="pt").to(self.device)
with torch.inference_mode():
logits = self.model(**inputs).logits # (1, C, h/4, w/4)
upsampled = torch.nn.functional.interpolate(
logits, size=(image.height, image.width), mode="bilinear", align_corners=False
)
return upsampled.argmax(dim=1)[0].to(torch.uint8).cpu().numpy()
|