{ "project": "Next Match", "profile": "full", "seed": 73, "device": { "python": "3.13.16", "platform": "Linux-6.6.159+-x86_64-with-glibc2.39", "cpu_count": 12, "numpy": "2.4.6", "torch": "2.14.1+cu130", "cuda": true, "gpu": "NVIDIA L4", "colab": true }, "data": { "type": "synthetic catalog rendered by render.py", "combinations": 192, "heldout_combinations": [ [ "bag", "green", "dotted" ], [ "bag", "green", "plain" ], [ "bag", "green", "striped" ], [ "bag", "orange", "dotted" ], [ "bag", "white", "striped" ], [ "bottle", "yellow", "dotted" ], [ "chair", "black", "plain" ], [ "chair", "green", "dotted" ], [ "chair", "green", "plain" ], [ "hat", "green", "plain" ], [ "lamp", "black", "plain" ], [ "lamp", "purple", "dotted" ], [ "lamp", "red", "plain" ], [ "lamp", "red", "striped" ], [ "lamp", "white", "striped" ], [ "mug", "green", "dotted" ], [ "mug", "orange", "plain" ], [ "mug", "yellow", "plain" ], [ "sneaker", "black", "plain" ], [ "sneaker", "red", "plain" ], [ "sneaker", "white", "striped" ], [ "tshirt", "black", "dotted" ], [ "tshirt", "orange", "plain" ], [ "tshirt", "red", "striped" ] ], "train_instances_per_combo": 10, "gallery": "1 unseen rendering per combination (seed 2)", "references": "unseen renderings (seed 3)", "templates": { "train": { "color": { "tr": [ "bunun {c} olanı", "{c} renklisi", "aynısının {c} olanı", "{c} olsun", "rengi {c} olsun" ], "en": [ "the {c} one", "make it {c}", "same but {c}", "in {c}" ] }, "pattern": { "tr": [ "{p} olanı", "{p} olsun", "aynısı ama {p}" ], "en": [ "the {p} one", "make it {p}", "same but {p}" ] }, "category": { "tr": [ "aynısı ama {k}", "bunun {k} olanı", "aynı desende {k}" ], "en": [ "same but a {k}", "a {k} like this", "same style {k}" ] }, "color+pattern": { "tr": [ "{c} ve {p} olanı", "{p} ve {c} olsun" ], "en": [ "{c} and {p}", "make it {p} and {c}" ] } }, "heldout": { "color": { "tr": [ "{c} olanını göster", "bundan {c} var mı" ], "en": [ "show me the {c} one", "do you have it in {c}" ] }, "pattern": { "tr": [ "{p} modelini göster" ], "en": [ "show a {p} version" ] }, "category": { "tr": [ "bunun gibi bir {k} istiyorum" ], "en": [ "I want a {k} like this" ] }, "color+pattern": { "tr": [ "{c} renkli ve {p} göster" ], "en": [ "show it {p} in {c}" ] } } }, "vocabulary_size": 61 }, "training": { "combined": { "seconds": 161.6, "final_loss": 0.5543771022558213, "steps": 6000, "batch": 128 }, "image": { "seconds": 158.2, "final_loss": 3.290284743309021, "steps": 6000, "batch": 128 }, "text": { "seconds": 138.7, "final_loss": 2.8225105476379393, "steps": 6000, "batch": 128 } }, "budget": "same encoder architecture, batch, steps, optimiser and schedule for all three models", "results": { "all": { "n": 1600, "combined": { "R@1": 0.9325, "R@5": 0.96625, "R@10": 0.979375 }, "image": { "R@1": 0.0, "R@5": 0.315625, "R@10": 0.578125 }, "text": { "R@1": 0.054375, "R@5": 0.263125, "R@10": 0.47125 } }, "edit=color": { "n": 648, "combined": { "R@1": 0.9938271604938271, "R@5": 1.0, "R@10": 1.0 }, "image": { "R@1": 0.0, "R@5": 0.35802469135802467, "R@10": 0.9583333333333334 }, "text": { "R@1": 0.05092592592592592, "R@5": 0.2175925925925926, "R@10": 0.39197530864197533 } }, "edit=pattern": { "n": 311, "combined": { "R@1": 0.8360128617363344, "R@5": 0.8971061093247589, "R@10": 0.9356913183279743 }, "image": { "R@1": 0.0, "R@5": 0.8778135048231511, "R@10": 0.9678456591639871 }, "text": { "R@1": 0.01607717041800643, "R@5": 0.07395498392282958, "R@10": 0.19614147909967847 } }, "edit=category": { "n": 322, "combined": { "R@1": 1.0, "R@5": 1.0, "R@10": 1.0 }, "image": { "R@1": 0.0, "R@5": 0.0, "R@10": 0.0 }, "text": { "R@1": 0.031055900621118012, "R@5": 0.18322981366459629, "R@10": 0.37267080745341613 } }, "edit=color+pattern": { "n": 319, "combined": { "R@1": 0.8338557993730408, "R@5": 0.9310344827586207, "R@10": 0.9592476489028213 }, "image": { "R@1": 0.0, "R@5": 0.0, "R@10": 0.009404388714733543 }, "text": { "R@1": 0.12225705329153605, "R@5": 0.6206896551724138, "R@10": 1.0 } }, "lang=tr": { "n": 747, "combined": { "R@1": 0.9946452476572959, "R@5": 1.0, "R@10": 1.0 }, "image": { "R@1": 0.0, "R@5": 0.3172690763052209, "R@10": 0.570281124497992 }, "text": { "R@1": 0.050870147255689425, "R@5": 0.2583668005354752, "R@10": 0.47523427041499333 } }, "lang=en": { "n": 853, "combined": { "R@1": 0.8780773739742087, "R@5": 0.936694021101993, "R@10": 0.9613130128956624 }, "image": { "R@1": 0.0, "R@5": 0.3141852286049238, "R@10": 0.5849941383352872 }, "text": { "R@1": 0.05744431418522861, "R@5": 0.2672919109026964, "R@10": 0.46776084407971863 } }, "template=seen": { "n": 1104, "combined": { "R@1": 1.0, "R@5": 1.0, "R@10": 1.0 }, "image": { "R@1": 0.0, "R@5": 0.3233695652173913, "R@10": 0.5769927536231884 }, "text": { "R@1": 0.057065217391304345, "R@5": 0.2509057971014493, "R@10": 0.45742753623188404 } }, "template=held-out": { "n": 496, "combined": { "R@1": 0.782258064516129, "R@5": 0.8911290322580645, "R@10": 0.9334677419354839 }, "image": { "R@1": 0.0, "R@5": 0.29838709677419356, "R@10": 0.5806451612903226 }, "text": { "R@1": 0.04838709677419355, "R@5": 0.2903225806451613, "R@10": 0.5020161290322581 } }, "combo=seen": { "n": 1192, "combined": { "R@1": 0.9328859060402684, "R@5": 0.9664429530201343, "R@10": 0.9773489932885906 }, "image": { "R@1": 0.0, "R@5": 0.34815436241610737, "R@10": 0.5956375838926175 }, "text": { "R@1": 0.05704697986577181, "R@5": 0.2961409395973154, "R@10": 0.5016778523489933 } }, "combo=held-out": { "n": 408, "combined": { "R@1": 0.9313725490196079, "R@5": 0.9656862745098039, "R@10": 0.9852941176470589 }, "image": { "R@1": 0.0, "R@5": 0.22058823529411764, "R@10": 0.5269607843137255 }, "text": { "R@1": 0.04656862745098039, "R@5": 0.16666666666666666, "R@10": 0.38235294117647056 } } }, "failure_examples_combined": [ { "ref": [ "lamp", "black", "striped" ], "target": [ "lamp", "black", "plain" ], "text": "show a plain version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": true, "rank_combined": 15 }, { "ref": [ "mug", "blue", "dotted" ], "target": [ "mug", "blue", "striped" ], "text": "show a striped version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 53 }, { "ref": [ "hat", "yellow", "dotted" ], "target": [ "hat", "black", "striped" ], "text": "show it striped in black", "kind": "color+pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 16 }, { "ref": [ "mug", "blue", "dotted" ], "target": [ "mug", "blue", "striped" ], "text": "show a striped version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 53 }, { "ref": [ "bag", "black", "dotted" ], "target": [ "bag", "orange", "striped" ], "text": "show it striped in orange", "kind": "color+pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 8 }, { "ref": [ "bottle", "green", "plain" ], "target": [ "bottle", "white", "striped" ], "text": "show it striped in white", "kind": "color+pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 50 }, { "ref": [ "bag", "red", "plain" ], "target": [ "bag", "red", "striped" ], "text": "show a striped version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 64 }, { "ref": [ "hat", "purple", "striped" ], "target": [ "hat", "purple", "plain" ], "text": "show a plain version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 25 }, { "ref": [ "bottle", "green", "plain" ], "target": [ "bottle", "white", "striped" ], "text": "show it striped in white", "kind": "color+pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 50 }, { "ref": [ "bag", "purple", "dotted" ], "target": [ "bag", "green", "striped" ], "text": "show it striped in green", "kind": "color+pattern", "lang": "en", "heldout_template": true, "unseen_combo": true, "rank_combined": 6 }, { "ref": [ "mug", "red", "dotted" ], "target": [ "mug", "red", "striped" ], "text": "show a striped version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 28 }, { "ref": [ "sneaker", "white", "plain" ], "target": [ "sneaker", "white", "striped" ], "text": "show a striped version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": true, "rank_combined": 60 }, { "ref": [ "hat", "red", "dotted" ], "target": [ "hat", "black", "striped" ], "text": "show it striped in black", "kind": "color+pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 5 }, { "ref": [ "bottle", "purple", "dotted" ], "target": [ "bottle", "purple", "striped" ], "text": "show a striped version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 8 }, { "ref": [ "bag", "blue", "dotted" ], "target": [ "bag", "blue", "plain" ], "text": "show a plain version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 22 }, { "ref": [ "sneaker", "red", "striped" ], "target": [ "sneaker", "red", "plain" ], "text": "show a plain version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": true, "rank_combined": 6 }, { "ref": [ "mug", "red", "striped" ], "target": [ "mug", "red", "plain" ], "text": "show a plain version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 73 }, { "ref": [ "mug", "purple", "dotted" ], "target": [ "mug", "purple", "plain" ], "text": "show a plain version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 125 }, { "ref": [ "mug", "red", "dotted" ], "target": [ "mug", "red", "plain" ], "text": "show a plain version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 89 }, { "ref": [ "sneaker", "green", "striped" ], "target": [ "sneaker", "green", "dotted" ], "text": "show a dotted version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 8 }, { "ref": [ "lamp", "red", "dotted" ], "target": [ "lamp", "black", "striped" ], "text": "show it striped in black", "kind": "color+pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 5 }, { "ref": [ "tshirt", "white", "plain" ], "target": [ "tshirt", "white", "striped" ], "text": "show a striped version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 15 }, { "ref": [ "sneaker", "black", "dotted" ], "target": [ "sneaker", "black", "striped" ], "text": "show a striped version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 80 }, { "ref": [ "bottle", "black", "plain" ], "target": [ "bottle", "blue", "striped" ], "text": "show it striped in blue", "kind": "color+pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 23 }, { "ref": [ "chair", "black", "striped" ], "target": [ "chair", "black", "plain" ], "text": "show a plain version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": true, "rank_combined": 6 }, { "ref": [ "sneaker", "blue", "plain" ], "target": [ "sneaker", "blue", "striped" ], "text": "show a striped version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 57 }, { "ref": [ "chair", "orange", "striped" ], "target": [ "chair", "orange", "plain" ], "text": "show a plain version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 10 }, { "ref": [ "bag", "green", "dotted" ], "target": [ "bag", "green", "plain" ], "text": "show a plain version", "kind": "pattern", "lang": "en", "heldout_template": true, "unseen_combo": true, "rank_combined": 11 }, { "ref": [ "sneaker", "yellow", "dotted" ], "target": [ "sneaker", "orange", "striped" ], "text": "show it striped in orange", "kind": "color+pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 24 }, { "ref": [ "hat", "orange", "dotted" ], "target": [ "hat", "white", "striped" ], "text": "show it striped in white", "kind": "color+pattern", "lang": "en", "heldout_template": true, "unseen_combo": false, "rank_combined": 12 } ], "tests": { "reload": { "returncode": 0, "seconds": 0.24, "process": "fresh python -E subprocess, cwd = temporary copy of the model folder", "passed": true, "cases": 6, "mismatches": [] }, "node_parity": { "cases": 6, "failed": 0, "maxDiff": 1.6653345369377348e-16, "results": [ { "name": "28:bunun kırmızı olanı", "maxDiff": 1.3877787807814457e-17 }, { "name": "17:çizgili olanı", "maxDiff": 5.551115123125783e-17 }, { "name": "102:same but a lamp", "maxDiff": 2.7755575615628914e-17 }, { "name": "84:make it dotted and yellow", "maxDiff": 1.249000902703301e-16 }, { "name": "190:İSTİYORUM, mor!", "maxDiff": 2.949029909160572e-17 }, { "name": "122:", "maxDiff": 1.6653345369377348e-16 } ], "ms": 749, "runtime": "node v22.23.4", "status": "passed" }, "browser": { "status": "skipped", "reason": "playwright not installed" } } }