File size: 16,650 Bytes
90d354b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
"""Example requests for the playground's "Load example" menu (docs/playground-spec.md).



Each example carries repository-relative image paths so the server can load the images itself;

`GET /examples/{n}/image/{i}` serves them to the UI. `load_examples()` drops any example whose

images are missing from the checkout rather than advertising a broken menu entry.



Every question here was checked against the v1 adapter (see "verified" notes): the menu is meant to

show the model's real behaviour, so an example either gets the answer right or exists to show an

abstention. Two things the v1 model is known to be shaky on are deliberately avoided: score scales

for subjective photo quality with fewer than 4 levels (it puts ~all mass on unknown, because the

training data only ever used a 5-point scale), and asking which state field a *second* image

contradicts (it answers that correctly from one image and flips with two).

"""
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]

STATE_AWARE = "data/decision-v1/state_aware/images"
TALLYQA = "data/decision-v1/tallyqa/images"
ABSTAIN = "data/decision-v1/heldout_abstention/images"
TEXTVQA = "data/decision-v1/textvqa/images"

CUTLERY = f"{STATE_AWARE}/21e6886fae2b31f208de0150a03c9d46af5fed5e6904d50e03f8107c99880278.jpg"
OXFORD = f"{STATE_AWARE}/a0e8fbdd569a7f7c0cec0aafb0c6bf63df05f50ba02a9814261f7d7bf07783cd.jpg"
SNEAKER = f"{STATE_AWARE}/1b3ac44bf8066c9894fefcf53af026a030e61b04280cf2f3386dcbde601f0a87.jpg"

# Photo-quality scores need a 5-level scale; on 3 levels the model abstains with unknown ≈ 0.99.
USABILITY = ["unusable: blurry, dark or badly cropped", "poor", "acceptable", "good",
             "excellent: sharp, well lit, product fills the frame"]

EXAMPLES = [
    {
        "name": "Product listing check",
        "description": "One product photo against a marketplace listing that is correct: colour match, category, photo usability. Verified: true 0.99, cutlery 0.97, usability 2.5 of 4.",
        "images": [CUTLERY],
        "request": {
            "state": {"listing": {"title": "24 Piece Stainless Steel Cutlery Set, Stripes (Contains: 6 Table Spoons, 6 Tea Spoons, 6 Forks, 6 Soup Spoons)", "color": "silver", "product_type": "cutlery"}},
            "questions": {
                "color_matches": {
                    "type": "noul",
                    "instructions": "The product in the photo matches `listing.color`.",
                    "criteria": {"true": "the dominant colour of the product is the listed colour",
                                 "false": "the product is clearly a different colour from the listed one"},
                },
                "product_type": {
                    "type": "choice",
                    "instructions": "Which category does the photographed item belong to?",
                    "criteria": {"cutlery": "knives, forks, spoons", "cookware": "pots and pans", "tableware": "plates, bowls, glasses", "other": None},
                },
                "photo_usability": {
                    "type": "score",
                    "instructions": "How usable is this photo as the main listing image?",
                    "criteria": USABILITY,
                },
            },
        },
    },
    {
        "name": "Listing field contradiction",
        "description": "The same check on a listing that is wrong: the shoe in the photo is black, the listing says brown. Verified: false 0.02, and the choice picks `listing.color` at 0.98.",
        "images": [SNEAKER],
        "request": {
            "state": {"listing": {"title": "Men's Lace-Up Shoes", "color": "brown", "product_type": "shoe"}},
            "questions": {
                "color_matches": {
                    "type": "noul",
                    "instructions": "The product in the photo matches `listing.color`.",
                    "criteria": {"true": "the product is the listed colour",
                                 "false": "the product is clearly a different colour from the listed one"},
                },
                "contradicted_field": {
                    "type": "choice",
                    "instructions": "Which listed field does this photo contradict?",
                    "criteria": {"listing.color": "the photo shows a different colour than `listing.color`",
                                 "listing.product_type": "the photo shows a different kind of product than `listing.product_type`",
                                 "none of them": "the photo is consistent with the listing"},
                },
            },
        },
    },
    {
        "name": "Two-image same-product",
        "description": "Reference listing photo vs a second photo of a different shoe: both questions reason across the two images. Verified: same_item 0.00, the first photo 0.66, black 0.60 — note how much lower the confidence is than on one image.",
        "images": [OXFORD, SNEAKER],
        "request": {
            "state": {"listing": {"title": "Men's Formal Oxford Shoes", "color": "brown", "product_type": "shoe"}},
            "questions": {
                "same_item": {
                    "type": "noul",
                    "instructions": "The second photo shows the item described in `listing`.",
                    "criteria": {"true": "the second photo shows the same product as the listing and the reference photo",
                                 "false": "the second photo shows a different product"},
                },
                "matching_photo": {
                    "type": "choice",
                    "instructions": "Which photo shows the product described in `listing`?",
                    "criteria": {"the first photo": None, "the second photo": None,
                                 "both photos": None, "neither photo": None},
                },
                "second_color": {
                    "type": "choice",
                    "instructions": "What is the dominant colour of the product in the second photo?",
                    "criteria": {"brown": None, "black": None, "white": None, "other": None},
                },
            },
        },
    },
    {
        "name": "Counting as score",
        "description": "An ordinal question used as a counter: the score type gives a distribution over 0-6 people. Verified: 3.01, with 0.78 on 'three people'.",
        "images": [f"{TALLYQA}/dcdc070d27438efb56ae31076174113440103634877d074c9ac5083a5df70bd6.jpg"],
        "request": {
            "state": {},
            "questions": {
                "people": {
                    "type": "score",
                    "instructions": "How many people are visible in the photo?",
                    "criteria": ["no people", "one person", "two people", "three people",
                                 "four people", "five people", "six or more people"],
                },
            },
        },
    },
    {
        "name": "Abstention showcase",
        "description": "Two questions should abstain: there is no dog (false premise) and the shirt colour the model reads, green, is not among the options (not listed). Bag colour and mood are answerable controls. Add an 'other' option to shirt_color to see the model pick it (0.98) instead of abstaining.",
        "images": [f"{ABSTAIN}/dc020c1f7b783a0fbd6d518b0e5e021a5a735312bd9a3b14b940ce8ca02adeb0.jpg"],
        "request": {
            "state": {},
            "questions": {
                "dog_color": {
                    "type": "choice",
                    "instructions": "What colour is the dog standing next to him?",
                    "criteria": {"red": None, "blue": None, "black": None, "white": None},
                },
                "shirt_color": {
                    "type": "choice",
                    "instructions": "What colour is his shirt?",
                    "criteria": {"red": None, "blue": None, "black": None},
                },
                "bag_color": {
                    "type": "choice",
                    "instructions": "What colour is the bag?",
                    "criteria": {"purple": None, "green": None, "gray": None, "black": None},
                },
                "mood": {
                    "type": "choice",
                    "instructions": "Which mood does the figure convey?",
                    "criteria": {"sad": None, "angry": None, "happy": None},
                },
            },
        },
    },
    {
        "name": "Document / scene text",
        "description": "A held-out TextVQA photo: a choice over candidate OCR tokens plus a legibility score. Verified: palm 1.00, legibility 3.3 of 4.",
        # TextVQA's val split, so the answer is read off the photo rather than remembered from training.
        "images": [f"{TEXTVQA}/1739daa601540017c980dce24fadae37fe9f2368790df2ae232854c0d39f2ac3.jpg"],
        "request": {
            "state": {"task": "read the branding on the device"},
            "questions": {
                "brand": {
                    "type": "choice",
                    "instructions": "Which brand name is printed on the device?",
                    "criteria": {"palm": None, "sony": None, "nokia": None, "tokyo bullet": None},
                },
                "text_legible": {
                    "type": "score",
                    "instructions": "How legible is the branding in this photo?",
                    "criteria": ["illegible", "mostly illegible", "partly legible", "mostly legible", "fully legible"],
                },
            },
        },
    },
    {
        "name": "Text-only: support ticket triage",
        "description": "No image at all: a support email as state, routed and triaged in one pass. The model runs the same prompt and decision position as an image request, with the vision tower skipped. Verified: billing 0.89, urgent 0.98, frustration 1.5 of 2.",
        "images": [],
        "request": {
            "state": {
                "ticket": {
                    "id": "TKT-48812",
                    "from": "dana.okafor@northwind-labs.example",
                    "received": "2026-09-21T08:14:00Z",
                    "subject": "Charged twice for the September seats - third email",
                    "body": ("We were billed 480 USD twice on 3 September for the same 12 seats. "
                             "I have now written three times and the only reply was an automated one. "
                             "Our finance close is on Wednesday and I need the duplicate reversed before then. "
                             "Nothing is broken in the product itself."),
                    "plan": "Team (12 seats)",
                    "prior_tickets_90d": 2,
                },
            },
            "questions": {
                "department": {
                    "type": "choice",
                    "instructions": "Which department should own this ticket?",
                    "criteria": {
                        "billing": "payments, invoices, refunds and subscription charges",
                        "technical_support": "the product is broken, erroring or unavailable",
                        "sales": "new seats, upgrades, quotes and renewals",
                        "account_management": "ownership, access and contract administration",
                    },
                },
                "urgent": {
                    "type": "noul",
                    "instructions": "This ticket needs a reply today.",
                    "criteria": {"true": "money is at stake or the customer has a dated deadline within days",
                                 "false": "it can wait for the normal queue"},
                },
                "frustration": {
                    "type": "score",
                    "instructions": "How frustrated does the customer sound?",
                    "criteria": ["calm and matter-of-fact",
                                 "impatient: chasing a reply or repeating themselves",
                                 "angry: threatening to escalate, churn or dispute the charge"],
                },
            },
        },
    },
    {
        "name": "Text-only: resume screening",
        "description": "State as a plain string rather than JSON, and object-form instructions ({\"question\": ..., \"today\": ...}) so the model can do date arithmetic. Six-option choice, a 6-level score and a noul. Verified: experience 3.5 of 5 (the candidate is at about 7 years, so this is one level low), mentorship 0.93, and it picks engineering_management at 0.49 over backend_systems — a good example of a low-confidence choice you would gate rather than act on.",
        "images": [],
        "request": {
            "state": ("Priya Raghunathan - Bengaluru, India. "
                      "2022-03 to now: Senior Backend Engineer, Zeltic Payments - owns the ledger service "
                      "(Go, Postgres, Kafka), led the migration off a monolith, runs the on-call rotation for six engineers "
                      "and writes the internal design-review guide. "
                      "2019-07 to 2022-02: Backend Engineer, Turnstile Health - billing APIs in Python. "
                      "2018-06 to 2019-06: intern, then junior developer, Kaveri Systems. "
                      "B.E. Computer Science, PES University, 2018. "
                      "Speaks at the Bengaluru Go meetup; mentored four interns to full-time offers."),
            "questions": {
                "experience_years": {
                    "type": "score",
                    "instructions": {
                        "question": "How many years of professional software engineering experience does this candidate have?",
                        "today": "September 22, 2026",
                        "note": "Count paid full-time work only; internships do not count.",
                    },
                    "criteria": ["less than 1 year", "1 to 2 years", "3 to 4 years",
                                 "5 to 6 years", "7 to 9 years", "10 years or more"],
                },
                "would_mentor": {
                    "type": "noul",
                    "instructions": "This candidate would be effective mentoring junior engineers.",
                    "criteria": {"true": "there is evidence of teaching, reviewing or growing other engineers",
                                 "false": "the record is individual-contributor work only"},
                },
                "talent_profile": {
                    "type": "choice",
                    "instructions": "Which profile fits this candidate best?",
                    "criteria": {
                        "backend_systems": "services, data stores and distributed systems",
                        "frontend_product": "user interfaces and client applications",
                        "data_ml": "data pipelines, analytics or machine learning",
                        "infrastructure_sre": "platform, deployment and reliability engineering",
                        "security": "application or infrastructure security",
                        "engineering_management": "primarily leads people rather than writes code",
                    },
                },
            },
        },
    },
]


def load_examples(root=ROOT):
    """Examples whose images all exist under `root`, each with UI-ready image descriptors.



    An example with an empty `images` list is a text-only example and is always listed.

    """
    root = Path(root)
    out = []
    for example in EXAMPLES:
        paths = [root / name for name in example["images"]]
        if not all(path.is_file() for path in paths):
            continue
        index = len(out)
        out.append({
            "index": index,
            "name": example["name"],
            "description": example["description"],
            "request": example["request"],
            "images": [{"index": i, "path": name, "name": Path(name).name,
                        "role": "reference" if i == 0 else "target",
                        "bytes": path.stat().st_size,
                        "url": f"/examples/{index}/image/{i}?v={Path(path).stem[:12]}"}  # cache-buster: image files are content-addressed
                       for i, (name, path) in enumerate(zip(example["images"], paths))],
        })
    return out