File size: 16,650 Bytes
90d354b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 | """Example requests for the playground's "Load example" menu (docs/playground-spec.md).
Each example carries repository-relative image paths so the server can load the images itself;
`GET /examples/{n}/image/{i}` serves them to the UI. `load_examples()` drops any example whose
images are missing from the checkout rather than advertising a broken menu entry.
Every question here was checked against the v1 adapter (see "verified" notes): the menu is meant to
show the model's real behaviour, so an example either gets the answer right or exists to show an
abstention. Two things the v1 model is known to be shaky on are deliberately avoided: score scales
for subjective photo quality with fewer than 4 levels (it puts ~all mass on unknown, because the
training data only ever used a 5-point scale), and asking which state field a *second* image
contradicts (it answers that correctly from one image and flips with two).
"""
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
STATE_AWARE = "data/decision-v1/state_aware/images"
TALLYQA = "data/decision-v1/tallyqa/images"
ABSTAIN = "data/decision-v1/heldout_abstention/images"
TEXTVQA = "data/decision-v1/textvqa/images"
CUTLERY = f"{STATE_AWARE}/21e6886fae2b31f208de0150a03c9d46af5fed5e6904d50e03f8107c99880278.jpg"
OXFORD = f"{STATE_AWARE}/a0e8fbdd569a7f7c0cec0aafb0c6bf63df05f50ba02a9814261f7d7bf07783cd.jpg"
SNEAKER = f"{STATE_AWARE}/1b3ac44bf8066c9894fefcf53af026a030e61b04280cf2f3386dcbde601f0a87.jpg"
# Photo-quality scores need a 5-level scale; on 3 levels the model abstains with unknown ≈ 0.99.
USABILITY = ["unusable: blurry, dark or badly cropped", "poor", "acceptable", "good",
"excellent: sharp, well lit, product fills the frame"]
EXAMPLES = [
{
"name": "Product listing check",
"description": "One product photo against a marketplace listing that is correct: colour match, category, photo usability. Verified: true 0.99, cutlery 0.97, usability 2.5 of 4.",
"images": [CUTLERY],
"request": {
"state": {"listing": {"title": "24 Piece Stainless Steel Cutlery Set, Stripes (Contains: 6 Table Spoons, 6 Tea Spoons, 6 Forks, 6 Soup Spoons)", "color": "silver", "product_type": "cutlery"}},
"questions": {
"color_matches": {
"type": "noul",
"instructions": "The product in the photo matches `listing.color`.",
"criteria": {"true": "the dominant colour of the product is the listed colour",
"false": "the product is clearly a different colour from the listed one"},
},
"product_type": {
"type": "choice",
"instructions": "Which category does the photographed item belong to?",
"criteria": {"cutlery": "knives, forks, spoons", "cookware": "pots and pans", "tableware": "plates, bowls, glasses", "other": None},
},
"photo_usability": {
"type": "score",
"instructions": "How usable is this photo as the main listing image?",
"criteria": USABILITY,
},
},
},
},
{
"name": "Listing field contradiction",
"description": "The same check on a listing that is wrong: the shoe in the photo is black, the listing says brown. Verified: false 0.02, and the choice picks `listing.color` at 0.98.",
"images": [SNEAKER],
"request": {
"state": {"listing": {"title": "Men's Lace-Up Shoes", "color": "brown", "product_type": "shoe"}},
"questions": {
"color_matches": {
"type": "noul",
"instructions": "The product in the photo matches `listing.color`.",
"criteria": {"true": "the product is the listed colour",
"false": "the product is clearly a different colour from the listed one"},
},
"contradicted_field": {
"type": "choice",
"instructions": "Which listed field does this photo contradict?",
"criteria": {"listing.color": "the photo shows a different colour than `listing.color`",
"listing.product_type": "the photo shows a different kind of product than `listing.product_type`",
"none of them": "the photo is consistent with the listing"},
},
},
},
},
{
"name": "Two-image same-product",
"description": "Reference listing photo vs a second photo of a different shoe: both questions reason across the two images. Verified: same_item 0.00, the first photo 0.66, black 0.60 — note how much lower the confidence is than on one image.",
"images": [OXFORD, SNEAKER],
"request": {
"state": {"listing": {"title": "Men's Formal Oxford Shoes", "color": "brown", "product_type": "shoe"}},
"questions": {
"same_item": {
"type": "noul",
"instructions": "The second photo shows the item described in `listing`.",
"criteria": {"true": "the second photo shows the same product as the listing and the reference photo",
"false": "the second photo shows a different product"},
},
"matching_photo": {
"type": "choice",
"instructions": "Which photo shows the product described in `listing`?",
"criteria": {"the first photo": None, "the second photo": None,
"both photos": None, "neither photo": None},
},
"second_color": {
"type": "choice",
"instructions": "What is the dominant colour of the product in the second photo?",
"criteria": {"brown": None, "black": None, "white": None, "other": None},
},
},
},
},
{
"name": "Counting as score",
"description": "An ordinal question used as a counter: the score type gives a distribution over 0-6 people. Verified: 3.01, with 0.78 on 'three people'.",
"images": [f"{TALLYQA}/dcdc070d27438efb56ae31076174113440103634877d074c9ac5083a5df70bd6.jpg"],
"request": {
"state": {},
"questions": {
"people": {
"type": "score",
"instructions": "How many people are visible in the photo?",
"criteria": ["no people", "one person", "two people", "three people",
"four people", "five people", "six or more people"],
},
},
},
},
{
"name": "Abstention showcase",
"description": "Two questions should abstain: there is no dog (false premise) and the shirt colour the model reads, green, is not among the options (not listed). Bag colour and mood are answerable controls. Add an 'other' option to shirt_color to see the model pick it (0.98) instead of abstaining.",
"images": [f"{ABSTAIN}/dc020c1f7b783a0fbd6d518b0e5e021a5a735312bd9a3b14b940ce8ca02adeb0.jpg"],
"request": {
"state": {},
"questions": {
"dog_color": {
"type": "choice",
"instructions": "What colour is the dog standing next to him?",
"criteria": {"red": None, "blue": None, "black": None, "white": None},
},
"shirt_color": {
"type": "choice",
"instructions": "What colour is his shirt?",
"criteria": {"red": None, "blue": None, "black": None},
},
"bag_color": {
"type": "choice",
"instructions": "What colour is the bag?",
"criteria": {"purple": None, "green": None, "gray": None, "black": None},
},
"mood": {
"type": "choice",
"instructions": "Which mood does the figure convey?",
"criteria": {"sad": None, "angry": None, "happy": None},
},
},
},
},
{
"name": "Document / scene text",
"description": "A held-out TextVQA photo: a choice over candidate OCR tokens plus a legibility score. Verified: palm 1.00, legibility 3.3 of 4.",
# TextVQA's val split, so the answer is read off the photo rather than remembered from training.
"images": [f"{TEXTVQA}/1739daa601540017c980dce24fadae37fe9f2368790df2ae232854c0d39f2ac3.jpg"],
"request": {
"state": {"task": "read the branding on the device"},
"questions": {
"brand": {
"type": "choice",
"instructions": "Which brand name is printed on the device?",
"criteria": {"palm": None, "sony": None, "nokia": None, "tokyo bullet": None},
},
"text_legible": {
"type": "score",
"instructions": "How legible is the branding in this photo?",
"criteria": ["illegible", "mostly illegible", "partly legible", "mostly legible", "fully legible"],
},
},
},
},
{
"name": "Text-only: support ticket triage",
"description": "No image at all: a support email as state, routed and triaged in one pass. The model runs the same prompt and decision position as an image request, with the vision tower skipped. Verified: billing 0.89, urgent 0.98, frustration 1.5 of 2.",
"images": [],
"request": {
"state": {
"ticket": {
"id": "TKT-48812",
"from": "dana.okafor@northwind-labs.example",
"received": "2026-09-21T08:14:00Z",
"subject": "Charged twice for the September seats - third email",
"body": ("We were billed 480 USD twice on 3 September for the same 12 seats. "
"I have now written three times and the only reply was an automated one. "
"Our finance close is on Wednesday and I need the duplicate reversed before then. "
"Nothing is broken in the product itself."),
"plan": "Team (12 seats)",
"prior_tickets_90d": 2,
},
},
"questions": {
"department": {
"type": "choice",
"instructions": "Which department should own this ticket?",
"criteria": {
"billing": "payments, invoices, refunds and subscription charges",
"technical_support": "the product is broken, erroring or unavailable",
"sales": "new seats, upgrades, quotes and renewals",
"account_management": "ownership, access and contract administration",
},
},
"urgent": {
"type": "noul",
"instructions": "This ticket needs a reply today.",
"criteria": {"true": "money is at stake or the customer has a dated deadline within days",
"false": "it can wait for the normal queue"},
},
"frustration": {
"type": "score",
"instructions": "How frustrated does the customer sound?",
"criteria": ["calm and matter-of-fact",
"impatient: chasing a reply or repeating themselves",
"angry: threatening to escalate, churn or dispute the charge"],
},
},
},
},
{
"name": "Text-only: resume screening",
"description": "State as a plain string rather than JSON, and object-form instructions ({\"question\": ..., \"today\": ...}) so the model can do date arithmetic. Six-option choice, a 6-level score and a noul. Verified: experience 3.5 of 5 (the candidate is at about 7 years, so this is one level low), mentorship 0.93, and it picks engineering_management at 0.49 over backend_systems — a good example of a low-confidence choice you would gate rather than act on.",
"images": [],
"request": {
"state": ("Priya Raghunathan - Bengaluru, India. "
"2022-03 to now: Senior Backend Engineer, Zeltic Payments - owns the ledger service "
"(Go, Postgres, Kafka), led the migration off a monolith, runs the on-call rotation for six engineers "
"and writes the internal design-review guide. "
"2019-07 to 2022-02: Backend Engineer, Turnstile Health - billing APIs in Python. "
"2018-06 to 2019-06: intern, then junior developer, Kaveri Systems. "
"B.E. Computer Science, PES University, 2018. "
"Speaks at the Bengaluru Go meetup; mentored four interns to full-time offers."),
"questions": {
"experience_years": {
"type": "score",
"instructions": {
"question": "How many years of professional software engineering experience does this candidate have?",
"today": "September 22, 2026",
"note": "Count paid full-time work only; internships do not count.",
},
"criteria": ["less than 1 year", "1 to 2 years", "3 to 4 years",
"5 to 6 years", "7 to 9 years", "10 years or more"],
},
"would_mentor": {
"type": "noul",
"instructions": "This candidate would be effective mentoring junior engineers.",
"criteria": {"true": "there is evidence of teaching, reviewing or growing other engineers",
"false": "the record is individual-contributor work only"},
},
"talent_profile": {
"type": "choice",
"instructions": "Which profile fits this candidate best?",
"criteria": {
"backend_systems": "services, data stores and distributed systems",
"frontend_product": "user interfaces and client applications",
"data_ml": "data pipelines, analytics or machine learning",
"infrastructure_sre": "platform, deployment and reliability engineering",
"security": "application or infrastructure security",
"engineering_management": "primarily leads people rather than writes code",
},
},
},
},
},
]
def load_examples(root=ROOT):
"""Examples whose images all exist under `root`, each with UI-ready image descriptors.
An example with an empty `images` list is a text-only example and is always listed.
"""
root = Path(root)
out = []
for example in EXAMPLES:
paths = [root / name for name in example["images"]]
if not all(path.is_file() for path in paths):
continue
index = len(out)
out.append({
"index": index,
"name": example["name"],
"description": example["description"],
"request": example["request"],
"images": [{"index": i, "path": name, "name": Path(name).name,
"role": "reference" if i == 0 else "target",
"bytes": path.stat().st_size,
"url": f"/examples/{index}/image/{i}?v={Path(path).stem[:12]}"} # cache-buster: image files are content-addressed
for i, (name, path) in enumerate(zip(example["images"], paths))],
})
return out
|