File size: 3,769 Bytes
82dfc77 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | """Next Match: search a catalog with a reference image plus a text edit ("bunun kirmizi olani", "same but striped").
Shared by the notebook, inference.py and (ported) web/core.js:
* tokenize: Turkish-aware lower-casing, punctuation removal, longest-prefix stem matching against a fixed vocabulary
* encode_query: image features + text features -> fused, L2-normalised query vector
* encode_item: catalog image -> L2-normalised key vector
The catalog is synthetic: procedurally drawn product icons (8 categories x 8 colours x 3 patterns), rendered in the notebook.
"""
import numpy as np
from nlrt import forward
CATEGORIES = ["tshirt", "mug", "sneaker", "bag", "hat", "chair", "lamp", "bottle"]
COLORS = ["red", "blue", "green", "yellow", "black", "white", "purple", "orange"]
PATTERNS = ["plain", "striped", "dotted"]
RGB = {"red": (214, 40, 40), "blue": (40, 90, 210), "green": (40, 160, 70), "yellow": (240, 200, 30),
"black": (30, 30, 34), "white": (250, 250, 250), "purple": (130, 60, 170), "orange": (245, 130, 30)}
TR = {"red": "kırmızı", "blue": "mavi", "green": "yeşil", "yellow": "sarı", "black": "siyah", "white": "beyaz", "purple": "mor",
"orange": "turuncu", "plain": "düz", "striped": "çizgili", "dotted": "puantiyeli", "tshirt": "tişört", "mug": "kupa",
"sneaker": "ayakkabı", "bag": "çanta", "hat": "şapka", "chair": "sandalye", "lamp": "lamba", "bottle": "şişe"}
EN = {"tshirt": "t-shirt", "mug": "mug", "sneaker": "sneaker", "bag": "bag", "hat": "hat", "chair": "chair", "lamp": "lamp", "bottle": "bottle",
"plain": "plain", "striped": "striped", "dotted": "dotted"}
SYNONYMS = {"tshirt": ["tshirt", "shirt", "tee"], "dotted": ["dotted", "dots", "polka"], "striped": ["striped", "stripes", "stripe"],
"sneaker": ["sneaker", "shoe", "shoes"], "plain": ["plain", "solid"]}
def lower_tr(s):
return s.replace("I", "ı").replace("İ", "i").lower()
def tokenize(text, vocab):
"""Return token ids (unknown words dropped). Matches whole words, then the longest vocabulary prefix of >= 4 letters."""
clean = "".join(ch if (ch.isalnum() or ch == " ") else " " for ch in lower_tr(text).replace("-", ""))
ids = []
for w in clean.split():
if w in vocab:
ids.append(vocab[w])
continue
best = None
for k in range(len(w) - 1, 3, -1):
if w[:k] in vocab:
best = vocab[w[:k]]
break
if best is not None:
ids.append(best)
return ids
def text_vector(ids, emb):
if not ids:
return np.zeros(emb.shape[1])
return np.mean(emb[ids], axis=0)
def l2n(x):
return x / max(float(np.linalg.norm(x)), 1e-12)
def image_features(spec, images, net="img"):
"""images: (N, 3, 48, 48) floats in [0, 1]."""
return forward(spec["nets"][net], np.asarray(images, dtype=np.float64))
def encode_item(spec, images, net="img"):
f = image_features(spec, images, net)
return f / np.maximum(np.linalg.norm(f, axis=1, keepdims=True), 1e-12)
def encode_query(spec, image, text, mode="combined"):
"""mode: combined | image | text. image: (3, 48, 48) in [0, 1]."""
vocab = spec["meta"]["vocab"]
if mode == "combined":
fi = image_features(spec, image[None], "img")[0]
ft = forward(spec["nets"]["txt"], text_vector(tokenize(text, vocab), spec["tensors"]["emb"])[None])[0]
return l2n(fi + forward(spec["nets"]["fuse"], np.concatenate([fi, ft])[None])[0])
if mode == "image":
return l2n(forward(spec["nets"]["img_only_head"], image_features(spec, image[None], "img_only")[0][None])[0])
ft = forward(spec["nets"]["txt_only"], text_vector(tokenize(text, vocab), spec["tensors"]["emb_only"])[None])[0]
return l2n(ft)
|