{
  "_about": "One example per ability of the strands decider (sources: the strands-decider README and docs/vision.md). `measured` blocks were recorded through app.answer() on the stated model and date -- never typed by hand.",
  "families": [
    {
      "id": "pick",
      "label": "Pick",
      "blurb": "choice: one of N options, with probabilities"
    },
    {
      "id": "rate",
      "label": "Rate",
      "blurb": "score: an ordered rubric, read as an expected value"
    },
    {
      "id": "yesno",
      "label": "Yes-No",
      "blurb": "noul: P(true)"
    },
    {
      "id": "agentic",
      "label": "Agentic",
      "blurb": "routing, tool checks, guardrails, rules"
    },
    {
      "id": "vision",
      "label": "Vision",
      "blurb": "over pixels · Qwen3.5 models only"
    },
    {
      "id": "code",
      "label": "Code",
      "blurb": "yes/no over a function"
    }
  ],
  "examples": [
    {
      "id": "triage-choice",
      "family": "pick",
      "title": "triage choice",
      "golden": true,
      "shows": "Route a support message to the team that should own it.",
      "teaches": "A choice answer is a distribution, not a label: billing wins, 23% of the mass sits elsewhere, and the confidence (0.659) is what you act on.",
      "state": "Help! My payouts have been failing for 3 days!",
      "questions": [
        {
          "type": "choice",
          "instructions": "Which team should handle this?",
          "options": [
            "billing",
            "sales",
            "retail"
          ]
        }
      ],
      "expected_shape": "choice -> billing (confidence ~0.659); billing ~0.773, sales ~0.121, retail ~0.106 (README, cpu)",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "choice_0"
        ],
        "answers": {
          "choice_0": {
            "type": "choice",
            "display": "billing",
            "value": "billing",
            "probs": {
              "billing": 0.7728,
              "sales": 0.1213,
              "retail": 0.106
            },
            "confidence": 0.6591,
            "vote": "billing"
          }
        },
        "input_tokens": 68,
        "latency_ms": 329.141,
        "cold": true
      }
    },
    {
      "id": "urgency-noul",
      "family": "yesno",
      "title": "urgency yes/no",
      "golden": true,
      "shows": "Is this message urgent?",
      "teaches": "noul returns only P(true): with two outcomes the probability already is the uncertainty.",
      "state": "Help! My payouts have been failing for 3 days!",
      "questions": [
        {
          "type": "noul",
          "instructions": "Does this convey urgency?",
          "options": []
        }
      ],
      "expected_shape": "noul ~0.858 (README, cpu)",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "noul_0"
        ],
        "answers": {
          "noul_0": {
            "type": "noul",
            "display": "0.8581",
            "value": 0.8581,
            "probs": {
              "true": 0.8581,
              "false": 0.14190000000000003
            },
            "confidence": null,
            "vote": "true"
          }
        },
        "input_tokens": 86,
        "latency_ms": 262.437,
        "cold": false
      }
    },
    {
      "id": "frustration-score",
      "family": "rate",
      "title": "frustration score",
      "golden": true,
      "shows": "Rate the writer on calm < frustrated < depressed.",
      "teaches": "A score is the expected value over the ordered levels you type, so the levels are the scale.",
      "state": "Help! My payouts have been failing for 3 days!",
      "questions": [
        {
          "type": "score",
          "instructions": "How frustrated is the writer?",
          "options": [
            "calm",
            "frustrated",
            "depressed"
          ]
        }
      ],
      "expected_shape": "score ~1.03 (confidence ~0.509); calm ~0.196, frustrated ~0.574, depressed ~0.230 (README, cpu)",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "score_0"
        ],
        "answers": {
          "score_0": {
            "type": "score",
            "display": "1.0345",
            "value": 1.0345,
            "probs": {
              "calm": 0.1958,
              "frustrated": 0.5738,
              "depressed": 0.2303
            },
            "confidence": 0.5091,
            "legend": {
              "0": "calm",
              "1": "frustrated",
              "2": "depressed"
            },
            "vote": "frustrated"
          }
        },
        "input_tokens": 84,
        "latency_ms": 256.475,
        "cold": false
      }
    },
    {
      "id": "multi-question",
      "family": "pick",
      "title": "three questions, one read",
      "shows": "All three questions on one state, in one request.",
      "teaches": "The state is encoded once, so extra questions about the same text are nearly free.",
      "state": "Help! My payouts have been failing for 3 days!",
      "questions": [
        {
          "type": "noul",
          "instructions": "Does this convey urgency?",
          "options": []
        },
        {
          "type": "choice",
          "instructions": "Which team should handle this?",
          "options": [
            "billing",
            "sales",
            "retail"
          ]
        },
        {
          "type": "score",
          "instructions": "How frustrated is the writer?",
          "options": [
            "calm",
            "frustrated",
            "depressed"
          ]
        }
      ],
      "expected_shape": "noul ~0.858 · choice billing (~0.659) · score ~1.03 -- the same numbers as the three single-question examples",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "noul_0",
          "choice_0",
          "score_0"
        ],
        "answers": {
          "noul_0": {
            "type": "noul",
            "display": "0.8581",
            "value": 0.8581,
            "probs": {
              "true": 0.8581,
              "false": 0.14190000000000003
            },
            "confidence": null,
            "vote": "true"
          },
          "choice_0": {
            "type": "choice",
            "display": "billing",
            "value": "billing",
            "probs": {
              "billing": 0.7728,
              "sales": 0.1213,
              "retail": 0.106
            },
            "confidence": 0.6591,
            "vote": "billing"
          },
          "score_0": {
            "type": "score",
            "display": "1.0345",
            "value": 1.0345,
            "probs": {
              "calm": 0.1958,
              "frustrated": 0.5738,
              "depressed": 0.2303
            },
            "confidence": 0.5091,
            "legend": {
              "0": "calm",
              "1": "frustrated",
              "2": "depressed"
            },
            "vote": "frustrated"
          }
        },
        "input_tokens": 196,
        "latency_ms": 604.615,
        "cold": false
      }
    },
    {
      "id": "model-routing",
      "family": "agentic",
      "title": "model routing",
      "shows": "Pick the LLM for a task: small and fast, or large and careful.",
      "teaches": "Options come from the request, not the weights: rename them to your own tiers.",
      "state": "User request: \"Draft a 3-page legal memo comparing the indemnification clauses in these two vendor contracts and flag any asymmetric liability caps.\"",
      "questions": [
        {
          "type": "choice",
          "instructions": "Which model should handle this request?",
          "options": [
            "small fast model",
            "large careful model"
          ]
        }
      ],
      "expected_shape": "choice -> one of the two tiers, with P(each) and a confidence",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "choice_0"
        ],
        "answers": {
          "choice_0": {
            "type": "choice",
            "display": "large careful model",
            "value": "large careful model",
            "probs": {
              "small fast model": 0.0925,
              "large careful model": 0.9075
            },
            "confidence": 0.8149,
            "vote": "large careful model"
          }
        },
        "input_tokens": 85,
        "latency_ms": 239.644,
        "cold": false
      }
    },
    {
      "id": "argument-check",
      "family": "agentic",
      "title": "argument check",
      "shows": "Verify a tool call's arguments before it runs.",
      "teaches": "JSON goes in as state; gate the call on P(true) with your own threshold.",
      "state": "Tool: send_email(to, subject, body)\nCall: {\"to\": \"\", \"subject\": \"Q3 invoice\", \"body\": \"Hi, attached is the Q3 invoice. Best, Sam\"}",
      "questions": [
        {
          "type": "noul",
          "instructions": "Are the arguments complete and ready to execute?",
          "options": []
        }
      ],
      "expected_shape": "noul P(true) -- low, since `to` is empty",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "noul_0"
        ],
        "answers": {
          "noul_0": {
            "type": "noul",
            "display": "0.1268",
            "value": 0.1268,
            "probs": {
              "true": 0.1268,
              "false": 0.8732
            },
            "confidence": null,
            "vote": "false"
          }
        },
        "input_tokens": 120,
        "latency_ms": 277.906,
        "cold": false
      }
    },
    {
      "id": "grounding-guardrail",
      "family": "agentic",
      "title": "grounding check",
      "shows": "Is a claim supported by the passage it cites?",
      "teaches": "With evidence and claim in one state, noul is a hallucination detector in one pass.",
      "state": "Passage: The clinic opens at 8:00 on weekdays and 9:00 on Saturdays. It is closed on Sundays and public holidays.\nClaim: The clinic is open on Sundays from 9:00.",
      "questions": [
        {
          "type": "noul",
          "instructions": "Is the claim supported by the passage?",
          "options": []
        }
      ],
      "expected_shape": "noul P(true) -- low: the passage says closed on Sundays",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "noul_0"
        ],
        "answers": {
          "noul_0": {
            "type": "noul",
            "display": "0.0423",
            "value": 0.0423,
            "probs": {
              "true": 0.0423,
              "false": 0.9577
            },
            "confidence": null,
            "vote": "false"
          }
        },
        "input_tokens": 125,
        "latency_ms": 285.565,
        "cold": false
      }
    },
    {
      "id": "rule-application",
      "family": "agentic",
      "title": "apply a policy",
      "shows": "A written policy plus a case: which outcome does the policy dictate?",
      "teaches": "Rules written in plain words apply at request time, nothing to retrain.",
      "state": "Policy: Refunds are approved automatically within 14 days of purchase. Between 15 and 30 days a refund needs a manager's approval. After 30 days refunds are denied unless the item arrived defective.\nCase: Purchased 22 days ago, item works, customer changed their mind.",
      "questions": [
        {
          "type": "choice",
          "instructions": "What does the policy say should happen?",
          "options": [
            "approve automatically",
            "needs manager approval",
            "deny"
          ]
        }
      ],
      "expected_shape": "choice -> needs manager approval (22 days falls in the 15-30 band)",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "choice_0"
        ],
        "answers": {
          "choice_0": {
            "type": "choice",
            "display": "needs manager approval",
            "value": "needs manager approval",
            "probs": {
              "approve automatically": 0.0292,
              "needs manager approval": 0.9456,
              "deny": 0.0252
            },
            "confidence": 0.9184,
            "vote": "needs manager approval"
          }
        },
        "input_tokens": 126,
        "latency_ms": 288.074,
        "cold": false
      }
    },
    {
      "id": "language-id",
      "family": "pick",
      "title": "language id",
      "shows": "Which language is 'sihamba ngokushesha'?",
      "teaches": "Options can be anything nameable; a label set of languages works like a label set of teams.",
      "state": "sihamba ngokushesha",
      "questions": [
        {
          "type": "choice",
          "instructions": "What language is this phrase in?",
          "options": [
            "English",
            "Zulu",
            "Dutch"
          ]
        }
      ],
      "expected_shape": "choice -> Zulu",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "choice_0"
        ],
        "answers": {
          "choice_0": {
            "type": "choice",
            "display": "Zulu",
            "value": "Zulu",
            "probs": {
              "English": 0.0444,
              "Zulu": 0.9112,
              "Dutch": 0.0444
            },
            "confidence": 0.8668,
            "vote": "Zulu"
          }
        },
        "input_tokens": 66,
        "latency_ms": 244.617,
        "cold": false
      }
    },
    {
      "id": "judge-score",
      "family": "rate",
      "title": "judge an answer",
      "shows": "Score a model's answer on a rubric from 0 to 4.",
      "teaches": "An ordered rubric makes the decider a cheap judge: the expected value is the grade.",
      "state": "Question: What is the capital of Australia?\nModel answer: The capital of Australia is Sydney, its largest city.\nRubric: 4 = correct and complete, 3 = correct with a minor flaw, 2 = partly correct, 1 = mostly wrong, 0 = wrong or no answer.",
      "questions": [
        {
          "type": "score",
          "instructions": "Grade the model answer against the rubric.",
          "options": [
            "0",
            "1",
            "2",
            "3",
            "4"
          ]
        }
      ],
      "expected_shape": "score near the low end (Canberra is the capital)",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "score_0"
        ],
        "answers": {
          "score_0": {
            "type": "score",
            "display": "2.3892",
            "value": 2.3892,
            "probs": {
              "0": 0.1218,
              "1": 0.156,
              "2": 0.1965,
              "3": 0.2626,
              "4": 0.2631
            },
            "confidence": 0.3895,
            "legend": {
              "0": "0",
              "1": "1",
              "2": "2",
              "3": "3",
              "4": "4"
            },
            "vote": "4"
          }
        },
        "input_tokens": 158,
        "latency_ms": 330.912,
        "cold": false
      }
    },
    {
      "id": "signature-signed",
      "family": "vision",
      "title": "signature: filled",
      "vision_only": true,
      "image": "form_signed.png",
      "shows": "A release form whose signature block is signed and dated.",
      "teaches": "The image enters the state as vision tokens; the readout and confidence are the text model's.",
      "state": "",
      "questions": [
        {
          "type": "noul",
          "instructions": "Is the signature block filled in?",
          "options": []
        }
      ],
      "expected_shape": "noul P(true) -- higher than on the empty form",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "noul_0"
        ],
        "answers": {
          "noul_0": {
            "type": "noul",
            "display": "0.6868",
            "value": 0.6868,
            "probs": {
              "true": 0.6868,
              "false": 0.31320000000000003
            },
            "confidence": null,
            "vote": "true"
          }
        },
        "input_tokens": 232,
        "latency_ms": 1160.211,
        "cold": true
      }
    },
    {
      "id": "signature-empty",
      "family": "vision",
      "title": "signature: empty",
      "vision_only": true,
      "image": "form_unsigned.png",
      "shows": "The same form, signature block empty.",
      "teaches": "The gap between the two P(true) values is what the model reads from the pixels.",
      "state": "",
      "questions": [
        {
          "type": "noul",
          "instructions": "Is the signature block filled in?",
          "options": []
        }
      ],
      "expected_shape": "noul P(true) -- lower than on the signed form",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "noul_0"
        ],
        "answers": {
          "noul_0": {
            "type": "noul",
            "display": "0.1344",
            "value": 0.1344,
            "probs": {
              "true": 0.1344,
              "false": 0.8656
            },
            "confidence": null,
            "vote": "false"
          }
        },
        "input_tokens": 232,
        "latency_ms": 1057.52,
        "cold": false
      }
    },
    {
      "id": "shape-choice",
      "family": "vision",
      "title": "which shape",
      "vision_only": true,
      "image": "shapes.png",
      "shows": "One large shape is drawn; pick which.",
      "teaches": "choice over images works like choice over text: the option tokens follow the image.",
      "state": "",
      "questions": [
        {
          "type": "choice",
          "instructions": "Which shape is drawn in the image?",
          "options": [
            "circle",
            "triangle",
            "square"
          ]
        }
      ],
      "expected_shape": "choice -> triangle",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "choice_0"
        ],
        "answers": {
          "choice_0": {
            "type": "choice",
            "display": "triangle",
            "value": "triangle",
            "probs": {
              "circle": 0.0119,
              "triangle": 0.9766,
              "square": 0.0115
            },
            "confidence": 0.9649,
            "vote": "triangle"
          }
        },
        "input_tokens": 256,
        "latency_ms": 1128.372,
        "cold": false
      }
    },
    {
      "id": "empty-list-code",
      "family": "code",
      "title": "empty list?",
      "shows": "Does this Python function handle the empty list?",
      "teaches": "Code is state; one forward pass answers a property of the function.",
      "state": "def average(xs):\n    return sum(xs) / len(xs)",
      "questions": [
        {
          "type": "noul",
          "instructions": "Does this function handle the empty list?",
          "options": []
        }
      ],
      "expected_shape": "noul P(true) -- low: len([]) == 0 divides by zero",
      "measured": {
        "label": "measured on cpu, strands-decider-2B-qwen3.5-v1-2610, 2026-10-10 02:20Z",
        "model": "strands-decider-2B-qwen3.5-v1-2610",
        "device": "cpu",
        "torch": "2.14.1",
        "question_ids": [
          "noul_0"
        ],
        "answers": {
          "noul_0": {
            "type": "noul",
            "display": "0.2203",
            "value": 0.2203,
            "probs": {
              "true": 0.2203,
              "false": 0.7797000000000001
            },
            "confidence": null,
            "vote": "false"
          }
        },
        "input_tokens": 91,
        "latency_ms": 359.318,
        "cold": false
      }
    }
  ]
}