File size: 3,769 Bytes
82dfc77
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
"""Next Match: search a catalog with a reference image plus a text edit ("bunun kirmizi olani", "same but striped").

Shared by the notebook, inference.py and (ported) web/core.js:
* tokenize: Turkish-aware lower-casing, punctuation removal, longest-prefix stem matching against a fixed vocabulary
* encode_query: image features + text features -> fused, L2-normalised query vector
* encode_item: catalog image -> L2-normalised key vector
The catalog is synthetic: procedurally drawn product icons (8 categories x 8 colours x 3 patterns), rendered in the notebook.
"""
import numpy as np

from nlrt import forward

CATEGORIES = ["tshirt", "mug", "sneaker", "bag", "hat", "chair", "lamp", "bottle"]
COLORS = ["red", "blue", "green", "yellow", "black", "white", "purple", "orange"]
PATTERNS = ["plain", "striped", "dotted"]
RGB = {"red": (214, 40, 40), "blue": (40, 90, 210), "green": (40, 160, 70), "yellow": (240, 200, 30),
       "black": (30, 30, 34), "white": (250, 250, 250), "purple": (130, 60, 170), "orange": (245, 130, 30)}
TR = {"red": "kırmızı", "blue": "mavi", "green": "yeşil", "yellow": "sarı", "black": "siyah", "white": "beyaz", "purple": "mor",
      "orange": "turuncu", "plain": "düz", "striped": "çizgili", "dotted": "puantiyeli", "tshirt": "tişört", "mug": "kupa",
      "sneaker": "ayakkabı", "bag": "çanta", "hat": "şapka", "chair": "sandalye", "lamp": "lamba", "bottle": "şişe"}
EN = {"tshirt": "t-shirt", "mug": "mug", "sneaker": "sneaker", "bag": "bag", "hat": "hat", "chair": "chair", "lamp": "lamp", "bottle": "bottle",
      "plain": "plain", "striped": "striped", "dotted": "dotted"}
SYNONYMS = {"tshirt": ["tshirt", "shirt", "tee"], "dotted": ["dotted", "dots", "polka"], "striped": ["striped", "stripes", "stripe"],
            "sneaker": ["sneaker", "shoe", "shoes"], "plain": ["plain", "solid"]}


def lower_tr(s):
    return s.replace("I", "ı").replace("İ", "i").lower()


def tokenize(text, vocab):
    """Return token ids (unknown words dropped). Matches whole words, then the longest vocabulary prefix of >= 4 letters."""
    clean = "".join(ch if (ch.isalnum() or ch == " ") else " " for ch in lower_tr(text).replace("-", ""))
    ids = []
    for w in clean.split():
        if w in vocab:
            ids.append(vocab[w])
            continue
        best = None
        for k in range(len(w) - 1, 3, -1):
            if w[:k] in vocab:
                best = vocab[w[:k]]
                break
        if best is not None:
            ids.append(best)
    return ids


def text_vector(ids, emb):
    if not ids:
        return np.zeros(emb.shape[1])
    return np.mean(emb[ids], axis=0)


def l2n(x):
    return x / max(float(np.linalg.norm(x)), 1e-12)


def image_features(spec, images, net="img"):
    """images: (N, 3, 48, 48) floats in [0, 1]."""
    return forward(spec["nets"][net], np.asarray(images, dtype=np.float64))


def encode_item(spec, images, net="img"):
    f = image_features(spec, images, net)
    return f / np.maximum(np.linalg.norm(f, axis=1, keepdims=True), 1e-12)


def encode_query(spec, image, text, mode="combined"):
    """mode: combined | image | text. image: (3, 48, 48) in [0, 1]."""
    vocab = spec["meta"]["vocab"]
    if mode == "combined":
        fi = image_features(spec, image[None], "img")[0]
        ft = forward(spec["nets"]["txt"], text_vector(tokenize(text, vocab), spec["tensors"]["emb"])[None])[0]
        return l2n(fi + forward(spec["nets"]["fuse"], np.concatenate([fi, ft])[None])[0])
    if mode == "image":
        return l2n(forward(spec["nets"]["img_only_head"], image_features(spec, image[None], "img_only")[0][None])[0])
    ft = forward(spec["nets"]["txt_only"], text_vector(tokenize(text, vocab), spec["tensors"]["emb_only"])[None])[0]
    return l2n(ft)