File size: 8,059 Bytes
c280886
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
"""Retrieval benchmark for Drape.

Known-item search under photo conditions: take catalog products, degrade them
the way a garment looks when cropped out of a real photo (see
``drape/augment.py``), and check whether the original product comes back.
Every model is evaluated on the same gallery and the same degraded queries.

    python scripts/benchmark.py --index-dir index_out

Writes ``benchmark/results.json`` and ``benchmark/results.md``.
"""

from __future__ import annotations

import argparse
import datetime as dt
import json
import sys
import time
from pathlib import Path

import numpy as np

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from drape.augment import augment  # noqa: E402
from drape.index import CatalogIndex  # noqa: E402
from drape.models import DEFAULT_EMBED_MODEL, OpenClipEmbedder, l2_normalize  # noqa: E402
from drape.taxonomy import GROUP_PROMPTS, GROUPS, OTHER  # noqa: E402

DEFAULT_MODELS = [
    DEFAULT_EMBED_MODEL,  # fashion-tuned SigLIP (what the app ships)
    "ViT-B-16-SigLIP|webli",  # same architecture, generic web pre-training
    "ViT-B-32|laion2b_s34b_b79k",  # classic generic CLIP
]
NICE = {
    DEFAULT_EMBED_MODEL: "Marqo-FashionSigLIP",
    "ViT-B-16-SigLIP|webli": "SigLIP ViT-B/16 (generic)",
    "ViT-B-32|laion2b_s34b_b79k": "CLIP ViT-B/32 (generic)",
}


def ranks_of_targets(q: np.ndarray, g: np.ndarray, targets: np.ndarray, allowed: list[np.ndarray] | None) -> np.ndarray:
    """1-based rank of each query's target in the gallery."""
    sims = q @ g.T  # (Q, G)
    ranks = np.empty(len(q), dtype=np.int64)
    for i in range(len(q)):
        s = sims[i]
        if allowed is not None:
            s = np.where(allowed[i], s, -np.inf)
        ranks[i] = int((s > s[targets[i]]).sum()) + 1
    return ranks


def metrics(ranks: np.ndarray) -> dict:
    n = len(ranks)
    out = {f"recall@{k}": float((ranks <= k).mean()) for k in (1, 5, 10)}
    out["mrr"] = float((1.0 / ranks).mean())
    p = out["recall@1"]
    out["recall@1_ci95"] = float(1.96 * np.sqrt(p * (1 - p) / n))
    return out


def zero_shot_accuracy(embedder, embs: np.ndarray, labels: np.ndarray) -> float:
    names = [*GROUPS, OTHER]
    protos = np.stack([l2_normalize(embedder.embed_texts(GROUP_PROMPTS[g]).mean(axis=0)) for g in names])
    scores = embs @ protos.T
    scores[:, names.index(OTHER)] = -np.inf  # the catalog has no "other" items left
    pred = np.array(names)[np.argmax(scores, axis=1)]
    return float((pred == labels).mean())


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--index-dir", default="index_out")
    ap.add_argument("--gallery", type=int, default=5000)
    ap.add_argument("--queries", type=int, default=1000)
    ap.add_argument("--seed", type=int, default=2026)
    ap.add_argument("--models", nargs="+", default=DEFAULT_MODELS)
    ap.add_argument("--out-dir", default="benchmark")
    args = ap.parse_args()

    index = CatalogIndex.load(args.index_dir)
    rng = np.random.default_rng(args.seed)
    gallery_rows = np.sort(rng.choice(len(index), size=min(args.gallery, len(index)), replace=False))
    q_local = rng.choice(len(gallery_rows), size=min(args.queries, len(gallery_rows)), replace=False)
    groups = index.groups[gallery_rows]
    allowed = [groups == groups[t] for t in q_local]
    kw = index.meta["group_kw"].to_numpy()[gallery_rows] if "group_kw" in index.meta else np.array([None] * len(gallery_rows))
    labelled = np.array([v is not None and v == v for v in kw])  # not None / NaN

    print(f"[bench] gallery {len(gallery_rows):,} items, {len(q_local):,} queries, seed {args.seed}")
    gallery_imgs = [index.thumbnail(int(r)) for r in gallery_rows]
    queries = {}
    for setting in ("clean", "wild"):
        queries[setting] = [
            augment(gallery_imgs[t], np.random.default_rng([args.seed, i, setting == "wild"]), wild=(setting == "wild"))
            for i, t in enumerate(q_local)
        ]
    Path(args.out_dir).mkdir(parents=True, exist_ok=True)
    for setting in ("clean", "wild"):  # save a few examples for the README
        strip = queries[setting][:6]
        from PIL import Image

        canvas = Image.new("RGB", (6 * 160, 2 * 160), (255, 255, 255))
        for j, t in enumerate(q_local[:6]):
            canvas.paste(gallery_imgs[t].resize((160, 160)), (j * 160, 0))
            q = strip[j].copy()
            q.thumbnail((160, 160))
            canvas.paste(q, (j * 160, 160))
        canvas.save(Path(args.out_dir) / f"queries_{setting}.jpg", quality=88)

    results = {"config": vars(args) | {"date": dt.date.today().isoformat()}, "models": {}}
    for spec in args.models:
        t0 = time.time()
        emb = OpenClipEmbedder(spec)
        g = emb.embed_images(gallery_imgs)
        res = {"device": emb.device}
        for setting, imgs in queries.items():
            q = emb.embed_images(imgs)
            res[setting] = {
                "all_catalog": metrics(ranks_of_targets(q, g, q_local, None)),
                "category_routed": metrics(ranks_of_targets(q, g, q_local, allowed)),
            }
        if labelled.any():
            res["zero_shot_group_accuracy"] = zero_shot_accuracy(emb, g[labelled], kw[labelled].astype(str))
            res["zero_shot_n"] = int(labelled.sum())
        res["seconds"] = round(time.time() - t0, 1)
        results["models"][spec] = res
        print(f"[bench] {NICE.get(spec, spec)}: {json.dumps(res, indent=None)[:400]}")
        del emb

    (Path(args.out_dir) / "results.json").write_text(json.dumps(results, indent=2))
    (Path(args.out_dir) / "results.md").write_text(render_markdown(results, index))
    print((Path(args.out_dir) / "results.md").read_text())


def _pct(x: float) -> str:
    return f"{100 * x:.1f}"


def render_markdown(results: dict, index: CatalogIndex) -> str:
    cfg = results["config"]
    lines = [
        "## Benchmark",
        "",
        f"Known-item retrieval: {cfg['queries']:,} catalog products, degraded to look like garments cropped from "
        f"real photos, searched against a {cfg['gallery']:,}-product gallery (seed {cfg['seed']}, {cfg['date']}). "
        "*Clean*: off-centre crop, tilt, lighting/colour cast, blur, JPEG. *Wild*: also a non-white background "
        "and a 6–20% occlusion. *Routed*: search restricted to the garment's category, as the app does.",
        "",
        "| Model | Setting | R@1 (±95% CI) | R@5 | R@10 | MRR | R@1 routed | R@10 routed |",
        "|---|---|---|---|---|---|---|---|",
    ]
    for spec, res in results["models"].items():
        for setting in ("clean", "wild"):
            a, r = res[setting]["all_catalog"], res[setting]["category_routed"]
            lines.append(
                f"| {NICE.get(spec, spec)} | {setting} | {_pct(a['recall@1'])} ± {_pct(a['recall@1_ci95'])} | "
                f"{_pct(a['recall@5'])} | {_pct(a['recall@10'])} | {a['mrr']:.3f} | "
                f"{_pct(r['recall@1'])} | {_pct(r['recall@10'])} |"
            )
    zs = [(spec, res) for spec, res in results["models"].items() if "zero_shot_group_accuracy" in res]
    if zs:
        lines += [
            "",
            "**Zero-shot category routing** (used when a photo has no person in it): accuracy against "
            f"keyword-derived labels on {zs[0][1]['zero_shot_n']:,} gallery items, {len(GROUPS)} categories.",
            "",
            "| Model | Accuracy |",
            "|---|---|",
        ]
        lines += [f"| {NICE.get(s, s)} | {_pct(r['zero_shot_group_accuracy'])}% |" for s, r in zs]
    lines += [
        "",
        "**Caveats.** Marqo-FashionSigLIP was fine-tuned on data that includes Polyvore product images, so its "
        "absolute scores on this catalog may be optimistic; the generic models were not. Synthetic degradations "
        "approximate, but are not, real street photos — a consumer-to-shop evaluation is on the roadmap.",
    ]
    return "\n".join(lines) + "\n"


if __name__ == "__main__":
    main()