Compare DecidaBERT-large-v1.1 and v2: ground truth, benchmark case browser, more demos, question builder

README.md

@@ -8,22 +8,30 @@ sdk_version: 5.9.1
 app_file: app.py
 pinned: false
 license: apache-2.0
-short_description: DecidaBERT-large vs v2, typed decisions in one pass
+short_description: DecidaBERT-large-v1.1 vs v2, typed decisions in one pass
 ---
 
-# Decida: DecidaBERT-large vs DecidaBERT-large-v2
+# Decida: DecidaBERT-large-v1.1 vs DecidaBERT-large-v2
 
-A free ZeroGPU demo that runs [DecidaBERT-large](https://huggingface.co/helmo/DecidaBERT-large) and
+A free ZeroGPU demo that runs [DecidaBERT-large-v1.1](https://huggingface.co/helmo/DecidaBERT-large-v1.1) and
 [DecidaBERT-large-v2](https://huggingface.co/helmo/DecidaBERT-large-v2) side by side: send a **state** and typed
-**questions** (`choice`, `score`, `noul`), get a probability distribution per question from each model, no
-generated text.
+**questions** (`choice`, `score`, `noul`), get a probability distribution per question from each model, one forward
+pass each, no generated text.
 
-The examples are chosen to show where the models differ: a long log whose key lines sit past v1's context, twelve
-questions answered in one pass, and decisions whose right answer is the measured middle option. The **Benchmark**
-tab has both models' scores on the Typed Decisions test split.
+- **Benchmark cases:** any of the 400 cases of the Typed Decisions **test** split (never used for training), with the
+  benchmark's gold answers. Each model is marked ✓ or ✗ per question, and the gold distribution is shown beside them.
+- **Hand-written demos:** a long incident log, a pull request with twelve questions, a phishing email, an invoice
+  that does not match its order, and more. Every question carries the demo author's reading of the right answer
+  (a judgment, not a benchmark label), so you can see where the models match it, and where they agree or disagree
+  with each other.
+- **Your own:** write any state (text or JSON) and questions, or build them with the question form; add `"expected"`
+  to a question to have both models marked against it.
 
-This Space's code is independent of the main project: `model.py` is a standalone port of v1's inference and
-`decidabert_v2/` is v2's inference code as published with the model. The local runtime with a REST API, MCP tools
-and a testbench is at **https://github.com/heldernoid/decida**.
+The **Benchmark** tab has both models' scores on the Typed Decisions test split, overall, per question type and per
+workflow. The two are level overall and differ by workflow.
+
+This Space's code is independent of the main project: `decidabert_v2/` is the inference code published with v2, and
+it loads both models (same architecture). The local runtime with a REST API, MCP tools and a testbench is at
+**https://github.com/heldernoid/decida**.
 
 Not affiliated with any other model author.

app.py

@@ -1,20 +1,25 @@
-"""DecidaBERT-large vs DecidaBERT-large-v2, side by side, free on ZeroGPU.
+"""DecidaBERT-large-v1.1 vs DecidaBERT-large-v2, side by side, free on ZeroGPU.
 
 The same state and typed questions go to both models; each answers every question with a probability distribution
-in one forward pass (v1: one sequence per question, up to 2,142 tokens; v2: one sequence for all questions, up to
-8,192 tokens). model.py is the standalone port of v1 used by this Space before; decidabert_v2/ is v2's own
-inference code, as published with the model. Neither imports the `decida` package.
+in one forward pass of up to 8,192 tokens. Both models share one architecture, so one inference package
+(decidabert_v2/, as published with v2) loads both.
+
+Ground truth: every question may carry an "expected" answer. Benchmark cases come from the Typed Decisions test split
+(never used for training) with the benchmark's gold; the hand-written demos carry the demo author's reading of the
+right answer (a judgment, not a benchmark label).
 """
 import html
 import json
 import os
+import random
 import time
 
 import gradio as gr
+import pandas as pd
 import torch
+from huggingface_hub import hf_hub_download
 
-from decidabert_v2 import load as load_v2
-from model import Primitive, Question, load as load_v1, predict as predict_v1
+from decidabert_v2 import load
 
 try:
     import spaces
@@ -23,25 +28,49 @@ except ImportError:
     def GPU(fn):
         return fn
 
-DEVICE = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
-V1, V1_LAYOUT = load_v1(DEVICE)
-V2 = load_v2(os.environ.get("DECIDABERT_V2", "helmo/DecidaBERT-large-v2"), DEVICE)
+DEVICE = os.environ.get("DECIDA_DEVICE") or ("cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu")
+MODELS = [  # (key, display name, repo or folder, colour as r,g,b)
+    ("v11", "DecidaBERT-large-v1.1", os.environ.get("DECIDABERT_V11", "helmo/DecidaBERT-large-v1.1"), "15,118,110"),
+    ("v2", "DecidaBERT-large-v2", os.environ.get("DECIDABERT_V2", "helmo/DecidaBERT-large-v2"), "67,56,202"),
+]
+LOADED = {k: load(path, DEVICE) for k, _, path, _ in MODELS}
+TOK = LOADED["v2"].layout.tok
+
+# ---- benchmark cases: LocalLLaMA/typed-decisions test split ------------------------------------------------------
+BENCH = pd.read_parquet(hf_hub_download("LocalLLaMA/typed-decisions", "all/test-00000-of-00001.parquet", repo_type="dataset"))
+WORKFLOWS = sorted(BENCH.workflow.unique())
+
+
+def _label_to_expected(q: dict, label) -> str:
+    if q["type"] == "noul":
+        return "yes" if str(label).lower() in ("true", "1", "yes") else "no"
+    return str(label)
 
 
+def bench_case(workflow: str, number: int):
+    rows = BENCH[BENCH.workflow == workflow].sort_values("id")
+    r = rows.iloc[int(number) % len(rows)]
+    state, qs, gold = json.loads(r["state"]), json.loads(r["questions"]), json.loads(r["gold"])
+    for qid, q in qs.items():
+        q["expected"] = _label_to_expected(q, gold[qid]["label"])
+    gold_dist = {qid: {"type": g["type"], "probabilities": g["probabilities"]} for qid, g in gold.items()}
+    why = (f"Benchmark case <b>{html.escape(r['id'])}</b> from the Typed Decisions <b>test</b> split, never used to train "
+           f"either model. The gold is the benchmark's teacher answer averaged over several samples; its top answer is "
+           f"the expected one, and its full distribution is shown on the right.")
+    return why, json.dumps(state, indent=1, ensure_ascii=False), json.dumps(qs, indent=2, ensure_ascii=False), gold_dist
+
+
+# ---- hand-written demos -------------------------------------------------------------------------------------------
 def _long_incident_log() -> str:
-    """~3,000 tokens of routine logs; the lines that matter come last, past v1's context and well inside v2's."""
+    """~3,000 tokens of routine logs; the lines that matter come last."""
     lines = []
     for i in range(80):
         mm, ss = divmod(i * 7, 60)
         svc = ["payments-api", "ledger-worker", "auth-gw", "notify"][i % 4]
-        if i % 4 == 0:
-            msg = f"POST /v2/charges 201 latency_ms={38 + (i * 13) % 41} merchant=m_{1000 + i}"
-        elif i % 4 == 1:
-            msg = f"batch {5000 + i} settled rows={200 + i % 57} lag_ms={12 + i % 9}"
-        elif i % 4 == 2:
-            msg = f"token refresh ok client=web-{i % 7} ttl=3600"
-        else:
-            msg = f"queued email receipt id=r_{77000 + i} provider=ok"
+        msg = [f"POST /v2/charges 201 latency_ms={38 + (i * 13) % 41} merchant=m_{1000 + i}",
+               f"batch {5000 + i} settled rows={200 + i % 57} lag_ms={12 + i % 9}",
+               f"token refresh ok client=web-{i % 7} ttl=3600",
+               f"queued email receipt id=r_{77000 + i} provider=ok"][i % 4]
         lines.append(f"2026-11-14T09:{mm:02d}:{ss:02d}Z INFO  {svc:13s} {msg}")
     lines += [
         "2026-11-14T09:28:01Z WARN  ledger-worker db primary latency_ms=2400 (threshold 200)",
@@ -54,102 +83,150 @@ def _long_incident_log() -> str:
     return "\n".join(lines)
 
 
-PRESETS = {
+SEVERITY4 = ["Low", "Medium", "High", "Critical"]
+DEMOS = {
     "Long incident log (~3k tokens)": {
-        "why": "The failure is in the last lines of a long log. v1 reads about the first 2,100 tokens of a request, "
-               "v2 reads up to 8,192, so only v2 sees the outage.",
+        "why": "The outage is in the last six lines of a long, routine log: a model has to read to the end.",
         "state": _long_incident_log(),
         "questions": {
-            "active_outage": {"type": "noul", "instructions": "Is there an active outage affecting customers right now?"},
+            "active_outage": {"type": "noul", "instructions": "Is there an active outage affecting customers right now?", "expected": "yes"},
             "severity": {"type": "score", "instructions": "How severe is the situation described by these logs?",
-                         "criteria": ["Normal operation", "Minor degradation", "Major degradation", "Critical outage"]},
+                         "criteria": ["Normal operation", "Minor degradation", "Major degradation", "Critical outage"], "expected": "3"},
             "next_action": {"type": "choice", "instructions": "What should the on-call engineer do?",
-                            "criteria": {"nothing": "Routine logs, no action needed",
-                                         "watch": "Keep an eye on it, nothing is failing yet",
-                                         "page_incident": "Declare an incident and page the database owners now"}},
+                            "criteria": {"nothing": "Routine logs, no action needed", "watch": "Keep an eye on it, nothing is failing yet",
+                                         "page_incident": "Declare an incident and page the database owners now"}, "expected": "page_incident"},
         },
     },
     "Security alert: suspicious, not confirmed": {
         "why": "The right call is the measured middle option (investigate), not the extremes (close or contain).",
-        "state": ("EDR alert on laptop FIN-2231: powershell.exe launched with an encoded command by excel.exe, "
-                  "15 minutes after the user opened 'Q4_invoice_review.xlsm' from an external sender. No network "
-                  "connection to known-bad addresses, no new services, no credential access observed so far. "
-                  "The user is in accounts payable and says the file came from a supplier they work with."),
+        "state": ("EDR alert on laptop FIN-2231: powershell.exe launched with an encoded command by excel.exe, 15 minutes after "
+                  "the user opened 'Q4_invoice_review.xlsm' from an external sender. No network connection to known-bad "
+                  "addresses, no new services, no credential access observed so far. The user is in accounts payable and says "
+                  "the file came from a supplier they work with."),
         "questions": {
             "disposition": {"type": "choice", "instructions": "How should this alert be handled?",
                             "criteria": {"close_benign": "Expected activity; close without analyst time",
                                          "monitor": "Not clearly malicious; watch for recurrence",
                                          "investigate": "Open an investigation and have an analyst review it",
-                                         "contain": "Isolate the host immediately"}},
-            "true_positive": {"type": "noul", "instructions": "Is this alert likely to be real malicious activity?"},
-            "severity": {"type": "score", "instructions": "How severe is it?", "criteria": ["Low", "Medium", "High", "Critical"]},
+                                         "contain": "Isolate the host immediately"}, "expected": "investigate"},
+            "true_positive": {"type": "noul", "instructions": "Is this alert likely to be real malicious activity?", "expected": "yes"},
+            "severity": {"type": "score", "instructions": "How severe is it?", "criteria": SEVERITY4, "expected": "2"},
         },
     },
-    "Agent run review: partly done": {
-        "why": "The agent made progress but stopped short; the measured answers are 'partial' and 'observe'.",
-        "state": ("Task: migrate 12 cron jobs from the old scheduler to the new workflow engine and verify each runs. "
-                  "Agent trace summary: migrated 9 of 12 jobs; 9 test runs succeeded; jobs 10-12 skipped because their "
-                  "scripts need a secret the agent cannot read. No errors, no destructive actions, no constraint broken. "
-                  "The agent ended with: 'Done, migration complete.'"),
+    "Agent run review: claims more than it did": {
+        "why": "The agent finished 9 of 12 jobs but reported the whole migration as done.",
+        "state": ("Task: migrate 12 cron jobs from the old scheduler to the new workflow engine and verify each runs. Agent trace "
+                  "summary: migrated 9 of 12 jobs; 9 test runs succeeded; jobs 10-12 skipped because their scripts need a secret "
+                  "the agent cannot read. No errors, no destructive actions, no constraint broken. The agent ended with: "
+                  "'Done, migration complete.'"),
         "questions": {
             "outcome": {"type": "choice", "instructions": "How did this agent run turn out?",
                         "criteria": {"success": "The task was completed correctly", "partial": "Progress made, task not fully done",
-                                     "failure": "The task was not accomplished", "harmful": "Caused damage or broke a constraint"}},
+                                     "failure": "The task was not accomplished", "harmful": "Caused damage or broke a constraint"},
+                        "expected": "partial"},
+            "summary_accurate": {"type": "noul", "instructions": "Is the agent's final message an accurate summary of what it did?",
+                                 "expected": "no"},
             "action": {"type": "choice", "instructions": "What should the observability system do with this run?",
                        "criteria": {"continue": "Let the agent proceed without interruption", "observe": "Keep running, flag for later sampling",
-                                    "human_review": "Queue for a human to review", "stop": "Halt the agent now"}},
-            "summary_accurate": {"type": "noul", "instructions": "Is the agent's final message an accurate summary of what it did?"},
+                                    "human_review": "Queue for a human to review", "stop": "Halt the agent now"},
+                       "expected": "human_review"},
         },
     },
     "Invoice vs purchase order": {
-        "why": "The numbers disagree; the measured answer is to hold, not to approve.",
+        "why": "The invoice bills 20 chairs; 18 were ordered and received. Hold it, don't pay it.",
         "state": json.dumps({"invoice": {"vendor": "Northfield Office Supply", "po": "PO-88412", "lines": [
             {"item": "Ergonomic chair", "qty": 20, "unit_price": 189.00}, {"item": "Desk lamp", "qty": 20, "unit_price": 34.50}],
-            "total": 4810.00},
+            "total": 4470.00},
             "purchase_order": {"po": "PO-88412", "lines": [
                 {"item": "Ergonomic chair", "qty": 18, "unit_price": 189.00}, {"item": "Desk lamp", "qty": 20, "unit_price": 34.50}],
                 "total": 4092.00},
             "goods_received": {"Ergonomic chair": 18, "Desk lamp": 20}}, indent=1),
         "questions": {
-            "matches_order": {"type": "noul", "instructions": "Does the invoice match the purchase order and the goods received?"},
+            "matches_order": {"type": "noul", "instructions": "Does the invoice match the purchase order and the goods received?", "expected": "no"},
             "disposition": {"type": "choice", "instructions": "How should this invoice be handled?",
                             "criteria": {"approve": "Matches; approve for payment", "hold": "Something needs confirming; hold pending clarification",
-                                         "reject": "Should not be paid: duplicate, unauthorised or materially wrong"}},
+                                         "reject": "Should not be paid: duplicate, unauthorised or materially wrong"}, "expected": "hold"},
         },
     },
-    "12 questions at once": {
-        "why": "v2 answers all 12 in one pass over the state; v1 builds one sequence per question (12 copies of the state).",
-        "state": ("Pull request #2291 'Add retry with backoff to the payment webhook sender'. Changes: webhook_sender.py "
-                  "(+84 -12), tests/test_webhook_sender.py (+120), config/defaults.yaml (+3). Adds exponential backoff (base "
-                  "500ms, max 5 tries) and an idempotency key header. CI: 412 tests passed, coverage 81% (+2%). One reviewer "
-                  "approved; one comment unresolved asking whether retries could double-charge if the receiver ignores the "
-                  "idempotency key. Deploy target: production, Friday 17:30."),
+    "Pull request: 12 questions at once": {
+        "why": "Twelve questions about one pull request, all answered from a single read of it.",
+        "state": ("Pull request #2291 'Add retry with backoff to the payment webhook sender'. Changes: webhook_sender.py (+84 -12), "
+                  "tests/test_webhook_sender.py (+120), config/defaults.yaml (+3). Adds exponential backoff (base 500ms, max 5 tries) "
+                  "and an idempotency key header. CI: 412 tests passed, coverage 81% (+2%). One reviewer approved; one comment "
+                  "unresolved asking whether retries could double-charge if the receiver ignores the idempotency key. Deploy "
+                  "target: production, Friday 17:30."),
         "questions": {
             "area": {"type": "choice", "instructions": "Which area does this change mainly touch?",
-                     "criteria": {"payments": "Payment flows", "auth": "Login and access", "ui": "User interface", "infra": "Build and infrastructure"}},
+                     "criteria": {"payments": "Payment flows", "auth": "Login and access", "ui": "User interface", "infra": "Build and infrastructure"},
+                     "expected": "payments"},
             "kind": {"type": "choice", "instructions": "What kind of change is it?",
-                     "criteria": {"feature": "New capability", "bugfix": "Fixes a defect", "reliability": "Makes an existing flow more robust", "refactor": "No behaviour change"}},
-            "tests_added": {"type": "noul", "instructions": "Does the change add tests?"},
-            "ci_green": {"type": "noul", "instructions": "Did CI pass?"},
-            "open_concern": {"type": "noul", "instructions": "Is there an unresolved review concern?"},
-            "money_risk": {"type": "noul", "instructions": "Could this change cause customers to be charged twice?"},
-            "risk": {"type": "score", "instructions": "How risky is merging and deploying this now?", "criteria": ["Low", "Moderate", "High"]},
-            "size": {"type": "score", "instructions": "How large is the change?", "criteria": ["Tiny", "Small", "Medium", "Large"]},
-            "deploy_timing_ok": {"type": "noul", "instructions": "Is the planned deploy time a good idea?"},
+                     "criteria": {"feature": "New capability", "bugfix": "Fixes a defect", "reliability": "Makes an existing flow more robust",
+                                  "refactor": "No behaviour change"}, "expected": "reliability"},
+            "tests_added": {"type": "noul", "instructions": "Does the change add tests?", "expected": "yes"},
+            "ci_green": {"type": "noul", "instructions": "Did CI pass?", "expected": "yes"},
+            "open_concern": {"type": "noul", "instructions": "Is there an unresolved review concern?", "expected": "yes"},
+            "money_risk": {"type": "noul", "instructions": "Could this change cause customers to be charged twice?", "expected": "yes"},
+            "risk": {"type": "score", "instructions": "How risky is merging and deploying this now?", "criteria": ["Low", "Moderate", "High"],
+                     "expected": "2"},
+            "size": {"type": "score", "instructions": "How large is the change?", "criteria": ["Tiny", "Small", "Medium", "Large"], "expected": "2"},
+            "deploy_timing_ok": {"type": "noul", "instructions": "Is deploying to production on Friday at 17:30 a good idea?", "expected": "no"},
             "next_step": {"type": "choice", "instructions": "What should happen next?",
                           "criteria": {"merge": "Merge and deploy as planned", "resolve_first": "Resolve the open concern, then merge",
-                                       "reject": "Close the PR"}},
-            "needs_second_review": {"type": "noul", "instructions": "Should a payments specialist review it before merge?"},
-            "release_note": {"type": "noul", "instructions": "Does this need a customer-facing release note?"},
+                                       "reject": "Close the PR"}, "expected": "resolve_first"},
+            "needs_second_review": {"type": "noul", "instructions": "Should a payments specialist review it before merge?", "expected": "yes"},
+            "release_note": {"type": "noul", "instructions": "Does this need a customer-facing release note?", "expected": "yes"},
+        },
+    },
+    "Phishing email": {
+        "why": "A look-alike domain, urgency and a credential link: a textbook phishing attempt.",
+        "state": ("From: IT Service Desk <helpdesk@micros0ft-support.com>\nTo: j.alvarez@contoso-logistics.com\n"
+                  "Subject: URGENT: your mailbox will be suspended in 2 hours\n\n"
+                  "Dear user, we detected unusual sign-in activity. To keep your mailbox active, confirm your password within "
+                  "2 hours at https://micros0ft-support.com/verify-account. Failure to do so will permanently delete your emails.\n"
+                  "Regards, IT Service Desk\n\n[Mail gateway: SPF fail, sender domain registered 3 days ago, link domain not on allow-list]"),
+        "questions": {
+            "phishing": {"type": "noul", "instructions": "Is this email a phishing attempt?", "expected": "yes"},
+            "action": {"type": "choice", "instructions": "What should the mail security team do?",
+                       "criteria": {"deliver": "Release it to the user's inbox", "warn": "Deliver with a warning banner",
+                                    "quarantine": "Quarantine it and block the sender domain"}, "expected": "quarantine"},
+            "severity": {"type": "score", "instructions": "How dangerous is it if a user acts on it?", "criteria": SEVERITY4, "expected": "2"},
+        },
+    },
+    "Refund request inside policy": {
+        "why": "Bought 9 days ago, unused, 30-day policy: a routine refund.",
+        "state": ("Ticket #48120. Customer: 'Hi, I ordered the Pro Wireless Headset (order A-55812) on Nov 5 but it turns out my "
+                  "laptop has no Bluetooth. It's still sealed in the box. Can I get my money back?'\n"
+                  "Order: paid 129.00 EUR on 2026-11-05, delivered 2026-11-07. Returns policy: full refund within 30 days of "
+                  "delivery for unopened items. Customer since 2021, 14 orders, no previous refunds."),
+        "questions": {
+            "eligible": {"type": "noul", "instructions": "Is the customer eligible for a full refund under the policy?", "expected": "yes"},
+            "action": {"type": "choice", "instructions": "How should the agent respond?",
+                       "criteria": {"approve_refund": "Approve the return and refund", "offer_credit": "Offer store credit instead",
+                                    "deny": "Decline: outside policy", "escalate": "Escalate to a supervisor"}, "expected": "approve_refund"},
+            "frustration": {"type": "score", "instructions": "How frustrated is the customer?", "criteria": ["Calm", "Frustrated", "Very angry"],
+                            "expected": "0"},
+        },
+    },
+    "Noisy alert that resolved itself": {
+        "why": "A short CPU spike during a scheduled batch job, already back to normal: no page needed.",
+        "state": ("Alert: HighCPU on batch-worker-07 (non-production). CPU 86% for 4 minutes at 02:03-02:07, threshold 80% for 3 "
+                  "minutes. Status: RESOLVED automatically at 02:07. Context: the nightly report job runs 02:00-02:10 on this host; "
+                  "the same alert fired and resolved on 11 of the last 14 nights. No errors, no customer-facing service on the host."),
+        "questions": {
+            "page_on_call": {"type": "noul", "instructions": "Should the on-call engineer be woken up for this?", "expected": "no"},
+            "action": {"type": "choice", "instructions": "What should be done with this alert?",
+                       "criteria": {"page": "Page on-call now", "ticket": "Open a ticket to tune the alert threshold or schedule",
+                                    "ignore": "Acknowledge and do nothing"}, "expected": "ticket"},
+            "severity": {"type": "score", "instructions": "How severe is it?", "criteria": SEVERITY4, "expected": "0"},
         },
     },
     "Tool-call gate": {
-        "why": "An everyday agent gate with no right answer spelled out: compare how each model weighs it.",
+        "why": "An everyday agent gate: a force-push to main is destructive whatever the build folder holds.",
         "state": "Agent wants to run: rm -rf ./build && git push --force origin main",
         "questions": {
+            "destructive": {"type": "noul", "instructions": "Can this command destroy work that cannot be recovered?", "expected": "yes"},
             "action": {"type": "choice", "instructions": "What should happen to this command?",
-                       "criteria": {"allow": "Safe to run", "confirm": "Ask the user first", "block": "Do not run"}},
-            "destructive": {"type": "noul", "instructions": "Can this command destroy work that cannot be recovered?"},
+                       "criteria": {"allow": "Safe to run", "confirm": "Ask the user first", "block": "Do not run"}, "expected": "confirm"},
         },
     },
     "Billing routing": {
@@ -157,156 +234,331 @@ PRESETS = {
         "state": "Customer: my invoice was charged twice and nobody answers the phone!",
         "questions": {
             "department": {"type": "choice", "instructions": "Which team should handle this?",
-                           "criteria": {"billing": "Charges, invoices, refunds", "technical": "Bugs and outages", "sales": "Pricing, demos, new seats"}},
-            "frustration": {"type": "score", "instructions": "How frustrated is the customer?", "criteria": ["Calm", "Frustrated", "Very angry"]},
-            "urgent": {"type": "noul", "instructions": "Is this urgent and needs an answer today?"},
+                           "criteria": {"billing": "Charges, invoices, refunds", "technical": "Bugs and outages", "sales": "Pricing, demos, new seats"},
+                           "expected": "billing"},
+            "frustration": {"type": "score", "instructions": "How frustrated is the customer?", "criteria": ["Calm", "Frustrated", "Very angry"],
+                            "expected": "1"},
+            "urgent": {"type": "noul", "instructions": "Is this urgent and needs an answer today?", "expected": "yes"},
+        },
+    },
+    "Blank: write your own": {
+        "why": "Write any state (text or JSON) and your own questions; add \"expected\" to a question to have both models marked.",
+        "state": "",
+        "questions": {
+            "example": {"type": "choice", "instructions": "Replace this with your question.",
+                        "criteria": {"option_a": "What option A means", "option_b": "What option B means"}, "expected": "option_a"},
         },
     },
 }
 
+# ---- answering and rendering ---------------------------------------------------------------------------------------
 CSS = """
-:root { --bg:#f4f5f8; --card:#fff; --ink:#141a2e; --ink2:#4a5270; --ink3:#7b839e; --line:#e3e6ee; --v1:#64748b; --v2:#4338ca; --track:#eceefb; --ok:#15803d; --diff:#b45309; }
-.gradio-container { max-width: 1180px !important; margin: 0 auto !important; font-family: -apple-system, "Segoe UI", sans-serif !important; background: var(--bg) !important; }
+:root { --bg:#f4f5f8; --card:#fff; --ink:#141a2e; --ink2:#4a5270; --ink3:#7b839e; --line:#e3e6ee; --track:#eceefb;
+        --v11:#0f766e; --v2:#4338ca; --gold:#a16207; --ok:#15803d; --bad:#b91c1c; }
+.gradio-container { max-width: 1240px !important; margin: 0 auto !important; font-family: -apple-system, "Segoe UI", sans-serif !important; }
 #title h1 { font-size: 30px; font-weight: 800; letter-spacing: -0.01em; margin-bottom: 2px; }
 #subtitle { color: var(--ink2); font-size: 15px; margin-bottom: 6px; }
 .dc-why { background:#eef2ff; border:1px solid #c7d2fe; color:#312e81; border-radius:10px; padding:8px 12px; font-size:13.5px; }
-.dc-card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 16px 18px; }
+.dc-card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 16px 18px; color: var(--ink); }
 .dc-card h3 { margin: 0 0 8px; font-size: 16px; }
-.dc-card.v1 h3 { color: var(--v1); } .dc-card.v2 h3 { color: var(--v2); }
+.dc-card.v11 h3 { color: var(--v11); } .dc-card.v2 h3 { color: var(--v2); } .dc-card.gold h3 { color: var(--gold); }
 .dc-q { padding: 10px 0; border-top: 1px solid var(--line); }
 .dc-q:first-of-type { border-top: none; padding-top: 0; }
-.dc-qhead { display:flex; align-items:center; gap:8px; font-weight:700; font-size:14.5px; color: var(--ink); margin-bottom: 2px; }
+.dc-qhead { display:flex; align-items:center; gap:8px; font-weight:700; font-size:14.5px; margin-bottom: 2px; }
 .dc-type { font: 700 10px ui-monospace, monospace; color: var(--ink3); border: 1px solid var(--line); border-radius: 5px; padding: 1px 6px; text-transform: uppercase; }
 .dc-instr { color: var(--ink2); font-size: 12.5px; margin-bottom: 6px; }
 .dc-bar-row { display: grid; grid-template-columns: minmax(80px,150px) 1fr 44px; gap: 8px; align-items: center; margin: 4px 0; font-size: 13px; }
 .dc-bar-track { display: block; height: 9px; border-radius: 6px; background: var(--track); overflow: hidden; }
 .dc-bar-fill { display: block; height: 100%; border-radius: 6px; }
-.dc-bar-label.top { font-weight: 700; color: var(--ink); }
+.dc-bar-label { overflow-wrap: anywhere; } .dc-bar-label.top { font-weight: 700; }
+.dc-bar-label.exp::after { content: " ★"; color: var(--gold); }
 .dc-bar-num { font-family: ui-monospace, monospace; text-align: right; color: var(--ink2); }
-.dc-meta { color: var(--ink3); font-size: 12.5px; margin-top: 8px; }
-.dc-meta b { color: var(--ink); }
+.dc-meta { color: var(--ink3); font-size: 12.5px; margin-top: 8px; } .dc-meta b { color: var(--ink); }
 .dc-cmp { width:100%; border-collapse: collapse; font-size: 13.5px; }
-.dc-cmp th, .dc-cmp td { text-align:left; padding:6px 8px; border-bottom:1px solid var(--line); }
-.dc-agree { color: var(--ok); font-weight: 700; } .dc-differ { color: var(--diff); font-weight: 700; }
+.dc-cmp th, .dc-cmp td { text-align:left; padding:6px 8px; border-bottom:1px solid var(--line); vertical-align: top; }
+.dc-ok { color: var(--ok); font-weight: 700; } .dc-bad { color: var(--bad); font-weight: 700; } .dc-na { color: var(--ink3); }
+.dc-score { display:flex; gap:18px; flex-wrap:wrap; font-size:15px; margin-bottom:10px; }
+.dc-score b.v11 { color: var(--v11); } .dc-score b.v2 { color: var(--v2); }
 """
 
 
-def _top(a: dict) -> tuple[str, float]:
-    if a["type"] == "noul":
-        return ("yes", a["noul"]) if a["noul"] >= 0.5 else ("no", 1 - a["noul"])
-    k = max(a["probabilities"], key=a["probabilities"].get)
-    label = f"{k}: {a['legend'][k]}" if a["type"] == "score" and "legend" in a else k
-    return label, a["probabilities"][k]
+def _parse_state(text: str):
+    """JSON states are passed as objects (rendered the way the benchmark was scored); anything else as text."""
+    s = text.strip()
+    if s[:1] in "{[":
+        try:
+            return json.loads(s)
+        except json.JSONDecodeError:
+            pass
+    return text
+
+
+def _keys(q: dict) -> list[str]:
+    c = q.get("criteria")
+    if q["type"] == "noul":
+        return ["yes", "no"]
+    if q["type"] == "choice":
+        return [str(x) for x in (c if isinstance(c, list) else list(c))]
+    return [str(i) for i in range(len(c))]
+
+
+def _levels(q: dict) -> list[str]:
+    c = q.get("criteria") or []
+    return list(c.values()) if isinstance(c, dict) else list(c)
+
+
+def _expected(q: dict) -> str | None:
+    """Normalise a question's "expected" answer to the key the models' top answer uses (yes/no, label, level index)."""
+    e = q.get("expected")
+    if e is None or e == "":
+        return None
+    if q["type"] == "noul":
+        return "yes" if str(e).strip().lower() in ("yes", "true", "1", "y") else "no"
+    if q["type"] == "score":
+        s = str(e).strip()
+        if s.isdigit():
+            return s
+        levels = [str(x).strip().lower() for x in _levels(q)]
+        return str(levels.index(s.lower())) if s.lower() in levels else s
+    return str(e).strip()
+
+
+def _dist(a: dict) -> dict[str, float]:
+    return {"yes": a["noul"], "no": 1 - a["noul"]} if a["type"] == "noul" else a["probabilities"]
+
 
+def _label(q: dict, k: str) -> str:
+    if q["type"] == "score":
+        lv = _levels(q)
+        return f"{k}: {lv[int(k)]}" if k.isdigit() and int(k) < len(lv) else k
+    return k
 
-def _render(answers: dict, raw: dict, cls: str, title: str, meta: str) -> str:
-    rgb = "100,116,139" if cls == "v1" else "67,56,202"
+
+def _bars(q: dict, dist: dict[str, float], rgb: str, exp: str | None) -> str:
+    top = max(dist, key=dist.get)
+    rows = []
+    for k, v in dist.items():
+        cls = ("top " if k == top else "") + ("exp" if k == exp else "")
+        rows.append(f'<div class="dc-bar-row"><span class="dc-bar-label {cls}">{html.escape(_label(q, k))}</span>'
+                    f'<span class="dc-bar-track"><span class="dc-bar-fill" style="width:{v * 100:.1f}%; '
+                    f'background:rgba({rgb},{0.15 + 0.8 * v:.2f})"></span></span><span class="dc-bar-num">{v:.2f}</span></div>')
+    return "".join(rows)
+
+
+def _mark(ok: bool | None) -> str:
+    return '<span class="dc-na">–</span>' if ok is None else ('<span class="dc-ok">✓</span>' if ok else '<span class="dc-bad">✗</span>')
+
+
+def _model_card(name: str, cls: str, rgb: str, answers: dict, qs: dict, meta: str) -> str:
+    blocks = []
+    for qid, q in qs.items():
+        d, exp = _dist(answers[qid]), _expected(q)
+        ok = None if exp is None else max(d, key=d.get) == exp
+        blocks.append(f'<div class="dc-q"><div class="dc-qhead">{_mark(ok) if ok is not None else ""} {html.escape(qid)} '
+                      f'<span class="dc-type">{q["type"]}</span></div>'
+                      f'<div class="dc-instr">{html.escape(str(q.get("instructions", "")))}</div>{_bars(q, d, rgb, exp)}'
+                      f'<div class="dc-meta">confidence {answers[qid]["confidence"]:.2f}</div></div>')
+    return f'<div class="dc-card {cls}"><h3>{name}</h3>{"".join(blocks)}<div class="dc-meta">{meta}</div></div>'
+
+
+def _gold_card(qs: dict, gold: dict | None) -> str:
     blocks = []
-    for qid, a in answers.items():
-        items = [("yes", a["noul"]), ("no", 1 - a["noul"])] if a["type"] == "noul" else list(a["probabilities"].items())
-        top_key = max(items, key=lambda kv: kv[1])[0]
-        rows = []
-        for k, v in items:
-            label = f"{k}: {a['legend'][k]}" if a["type"] == "score" and "legend" in a else str(k)
-            rows.append(f'<div class="dc-bar-row"><span class="dc-bar-label {"top" if k == top_key else ""}">{html.escape(label)}</span>'
-                        f'<span class="dc-bar-track"><span class="dc-bar-fill" style="width:{v * 100:.1f}%; background:rgba({rgb},{0.15 + 0.8 * v:.2f})"></span></span>'
-                        f'<span class="dc-bar-num">{v:.2f}</span></div>')
-        blocks.append(f'<div class="dc-q"><div class="dc-qhead">{html.escape(qid)} <span class="dc-type">{a["type"]}</span></div>'
-                      f'<div class="dc-instr">{html.escape(str(raw[qid].get("instructions", "")))}</div>{"".join(rows)}</div>')
-    return f'<div class="dc-card {cls}"><h3>{title}</h3>{"".join(blocks)}<div class="dc-meta">{meta}</div></div>'
-
-
-def _compare(a1: dict, a2: dict) -> str:
-    rows, same = [], 0
-    for qid in a2:
-        t1, p1 = _top(a1[qid]); t2, p2 = _top(a2[qid])
-        agree = t1 == t2; same += agree
-        badge = '<span class="dc-agree">same</span>' if agree else '<span class="dc-differ">different</span>'
-        rows.append(f"<tr><td><b>{html.escape(qid)}</b></td><td>{html.escape(t1)} ({p1:.2f})</td>"
-                    f"<td>{html.escape(t2)} ({p2:.2f}) · confidence {a2[qid]['confidence']:.2f}</td><td>{badge}</td></tr>")
-    return (f'<div class="dc-card"><h3>Top answers: {same} of {len(a2)} the same</h3><table class="dc-cmp"><tr><th>question</th>'
-            f'<th>v1</th><th>v2 (calibrated confidence)</th><th></th></tr>{"".join(rows)}</table></div>')
+    for qid, q in qs.items():
+        exp = _expected(q)
+        g = (gold or {}).get(qid)
+        if g and g["type"] == q["type"]:
+            d = g["probabilities"]
+            d = {"yes": float(d.get("true", 0)), "no": float(d.get("false", 0))} if q["type"] == "noul" else {k: float(v) for k, v in d.items()}
+            body = _bars(q, d, "161,98,7", exp) if set(d) == set(_keys(q)) else ""
+        else:
+            body = ""
+        shown = html.escape(_label(q, exp)) if exp is not None else '<span class="dc-na">no expected answer given</span>'
+        blocks.append(f'<div class="dc-q"><div class="dc-qhead">{html.escape(qid)} <span class="dc-type">{q["type"]}</span></div>'
+                      f'<div class="dc-instr">expected: <b>{shown}</b></div>{body}</div>')
+    src = ("benchmark gold (teacher average)" if gold else
+           "expected answers given with the questions (for the demos: the demo author's reading, not a benchmark label)")
+    return f'<div class="dc-card gold"><h3>Ground truth</h3>{"".join(blocks)}<div class="dc-meta">{src}; ★ marks the expected answer</div></div>'
+
+
+def _summary(qs: dict, results: dict, n_tok: int) -> str:
+    rows, scored = [], [qid for qid, q in qs.items() if _expected(q) is not None]
+    hits, agree = {k: 0 for k, *_ in MODELS}, 0
+    for qid, q in qs.items():
+        exp = _expected(q)
+        cells, tops = [], []
+        for k, *_ in MODELS:
+            d = _dist(results[k][0][qid]); top = max(d, key=d.get); tops.append(top)
+            ok = None if exp is None else top == exp
+            hits[k] += bool(ok)
+            cells.append(f"<td>{_mark(ok)} {html.escape(_label(q, top))} ({d[top]:.2f})</td>")
+        same = len(set(tops)) == 1; agree += same
+        badge = '<span class="dc-ok">agree</span>' if same else '<span class="dc-bad">disagree</span>'
+        ex = html.escape(_label(q, exp)) if exp is not None else '<span class="dc-na">–</span>'
+        rows.append(f"<tr><td><b>{html.escape(qid)}</b></td><td>{ex}</td>{''.join(cells)}<td>{badge}</td></tr>")
+    score = '<div class="dc-score">' + "".join(
+        f'<span><b class="{k}">{name}</b>: {hits[k]} of {len(scored)} match the expected answer</span>' for k, name, *_ in MODELS
+    ) * bool(scored) + f'<span>the two models agree on <b>{agree} of {len(qs)}</b></span></div>'
+    head = "".join(f"<th>{name}</th>" for _, name, *_ in MODELS)
+    return (f'<div class="dc-card">{score}<table class="dc-cmp"><tr><th>question</th><th>expected</th>{head}<th>models</th></tr>{"".join(rows)}</table>'
+            f'<div class="dc-meta">state: {n_tok:,} tokens · {len(qs)} question(s) · one forward pass per model · on {DEVICE}</div></div>')
 
 
 @GPU
-def answer(state: str, questions_json: str):
+def answer(state: str, questions_json: str, gold: dict | None):
     try:
-        raw = json.loads(questions_json)
-    except json.JSONDecodeError as exc:
-        msg = f'<div class="dc-card">questions is not valid JSON: {html.escape(str(exc))}</div>'
-        return msg, "", ""
+        qs = json.loads(questions_json)
+        assert isinstance(qs, dict) and qs, "questions must be a JSON object {question_id: {...}}"
+        for qid, q in qs.items():
+            assert q.get("type") in ("noul", "choice", "score"), f"{qid}: type must be noul, choice or score"
+            assert q["type"] == "noul" or q.get("criteria"), f"{qid}: choice and score questions need criteria"
+    except (json.JSONDecodeError, AssertionError) as exc:
+        return f'<div class="dc-card">Questions: {html.escape(str(exc))}</div>', "", "", ""
+    clean = {qid: {k: v for k, v in q.items() if k != "expected"} for qid, q in qs.items()}
+    st = _parse_state(state)
+    results = {}
     try:
-        qs1 = {qid: Question(qid, Primitive(q["type"]), q.get("instructions", ""), q.get("criteria")) for qid, q in raw.items()}
-        t0 = time.perf_counter(); a1 = predict_v1(V1, V1_LAYOUT, state, qs1); ms1 = (time.perf_counter() - t0) * 1000
-        t0 = time.perf_counter(); a2 = V2.predict(state, raw); ms2 = (time.perf_counter() - t0) * 1000
-    except (KeyError, ValueError, TypeError) as exc:
-        msg = f'<div class="dc-card">bad question: {html.escape(str(exc))}</div>'
-        return msg, "", ""
-    n_tok = len(V2.layout.tok(state, add_special_tokens=False)["input_ids"])
-    v1_read = V1_LAYOUT.build(state, next(iter(qs1.values())))
-    v1_note = " · <b>state cut to fit</b>" if v1_read.truncated else ""
-    v2_note = " · <b>state cut to fit</b>" if any(x.get("truncated_state") for x in a2.values()) else ""
-    meta1 = f"<b>{ms1:.0f} ms</b> · {len(raw)} sequence(s), one per question · context 2,142{v1_note}"
-    meta2 = f"<b>{ms2:.0f} ms</b> · 1 sequence for all questions · context 8,192{v2_note}"
-    head = f'<div class="dc-meta">state: {n_tok:,} tokens · {len(raw)} question(s) · on {DEVICE}</div>'
-    return (_compare(a1, a2) + head,
-            _render(a1, raw, "v1", "DecidaBERT-large (v1)", meta1),
-            _render(a2, raw, "v2", "DecidaBERT-large-v2", meta2))
-
-
-def _load_preset(name: str):
-    p = PRESETS[name]
-    return f'<div class="dc-why">{html.escape(p["why"])}</div>', p["state"], json.dumps(p["questions"], indent=2)
-
-
-BENCH = """
+        for k, *_ in MODELS:
+            t0 = time.perf_counter(); a = LOADED[k].predict(st, clean); results[k] = (a, (time.perf_counter() - t0) * 1000)
+    except (KeyError, ValueError, TypeError, IndexError) as exc:
+        return f'<div class="dc-card">Bad question: {html.escape(str(exc))}</div>', "", "", ""
+    n_tok = len(TOK(st if isinstance(st, str) else json.dumps(st, ensure_ascii=False), add_special_tokens=False)["input_ids"])
+    cards = []
+    for k, name, _, rgb in MODELS:
+        a, ms = results[k]
+        cut = " · <b>state cut to fit 8,192 tokens</b>" if any(x.get("truncated_state") for x in a.values()) else ""
+        cards.append(_model_card(name, k, rgb, a, qs, f"<b>{ms:.0f} ms</b>{cut}"))
+    return _summary(qs, results, n_tok), cards[0], cards[1], _gold_card(qs, gold)
+
+
+def load_demo(name: str):
+    p = DEMOS[name]
+    return f'<div class="dc-why">{html.escape(p["why"])}</div>', p["state"], json.dumps(p["questions"], indent=2, ensure_ascii=False), None
+
+
+def load_bench(workflow: str, number: float):
+    why, st, qs, gold = bench_case(workflow, int(number))
+    return f'<div class="dc-why">{why}</div>', st, qs, gold
+
+
+def random_bench():
+    w = random.choice(WORKFLOWS); n = random.randrange(100)
+    return (w, n) + load_bench(w, n)
+
+
+def add_question(questions_json: str, qid: str, qtype: str, instructions: str, options: str, expected: str):
+    try:
+        qs = json.loads(questions_json) if questions_json.strip() else {}
+    except json.JSONDecodeError:
+        return questions_json, "The questions box is not valid JSON; fix or clear it first."
+    qid = (qid or "").strip().replace(" ", "_")
+    if not qid or not instructions.strip():
+        return questions_json, "A question needs an id and instructions."
+    q = {"type": qtype, "instructions": instructions.strip()}
+    lines = [l.strip() for l in options.splitlines() if l.strip()]
+    if qtype == "choice":
+        if len(lines) < 2:
+            return questions_json, "A choice question needs at least two options, one per line (label: description)."
+        q["criteria"] = {l.split(":", 1)[0].strip().replace(" ", "_"): (l.split(":", 1)[1].strip() if ":" in l else "") for l in lines}
+    elif qtype == "score":
+        if len(lines) < 2:
+            return questions_json, "A score question needs at least two levels, lowest first, one per line."
+        q["criteria"] = lines
+    if expected.strip():
+        q["expected"] = expected.strip()
+    qs[qid] = q
+    return json.dumps(qs, indent=2, ensure_ascii=False), f"Added {qid}."
+
+
+BENCH_MD = """
 ### Typed Decisions test split (400 cases, 2,000 questions)
 
-Scored the same way for both models: a prediction is right when its top answer equals the gold label; 95% Wilson
-intervals. Both models were fitted on the benchmark's `train` split; the test split was not used for training.
+Scored the same way for every model: a prediction is right when its top answer equals the gold label; 95% Wilson
+intervals. All models were fitted on the benchmark's `train` split; the test split was not used for training.
 
-| question type | questions | DecidaBERT-large | DecidaBERT-large-v2 | v2 95% interval | random guess |
+| question type | questions | DecidaBERT-large (v1) | DecidaBERT-large-v1.1 | DecidaBERT-large-v2 | random guess |
 |---|---|---|---|---|---|
-| choice | 600 | 0.722 | **0.747** | 0.710 to 0.780 | 0.233 |
-| score | 800 | 0.694 | **0.759** | 0.728 to 0.787 | 0.244 |
-| yes/no | 600 | 0.830 | **0.868** | 0.839 to 0.893 | 0.500 |
-| **all** | 2,000 | 0.743 (0.723 to 0.762) | **0.788** | **0.770 to 0.805** | |
+| choice | 600 | 0.722 | **0.747** | **0.747** | 0.233 |
+| score | 800 | 0.694 | 0.755 | **0.759** | 0.244 |
+| yes/no | 600 | 0.830 | 0.850 | **0.868** | 0.500 |
+| **all** | 2,000 | 0.743 (0.723 to 0.762) | 0.781 (0.762 to 0.799) | **0.788** (0.770 to 0.805) | |
+
+**v1.1 against v2, question by question:** level overall (difference −0.007, 95% interval −0.021 to +0.008; 99
+questions only v2 got right, 85 only v1.1). By workflow, v1.1 is ahead on invoice processing (0.832 vs 0.806) and
+behind on security incidents (0.750 vs 0.786); customer service 0.810 vs 0.830, agent traces 0.732 vs 0.730. The two
+give the same top answer on 90% of the questions.
 
-| | DecidaBERT-large | DecidaBERT-large-v2 |
+| | DecidaBERT-large-v1.1 | DecidaBERT-large-v2 |
 |---|---|---|
-| Context | trained at 512, up to 2,142 here | **8,192** |
-| Questions per forward pass | one | **all of them** |
-| Score within one level of gold | 95.5% | **97.6%** |
-| Reported confidence calibration error (test) | 0.130 | **0.023** |
-| Parameters | 421M | 395M |
-
-Models: [DecidaBERT-large](https://huggingface.co/helmo/DecidaBERT-large) ·
-[DecidaBERT-large-v2](https://huggingface.co/helmo/DecidaBERT-large-v2)
+| Base | ModernBERT-large-nli | ModernBERT-large |
+| Context | 8,192 | 8,192 |
+| Questions per forward pass | all of them | all of them |
+| Score within one level of gold | **98.3%** | 97.6% |
+| Reported confidence calibration error (test) | 0.024 | **0.023** |
+| Benchmark scorer: accuracy / KL | 0.776 / **0.0852** | **0.7835** / 0.0861 |
+
+Models: [DecidaBERT-large-v1.1](https://huggingface.co/helmo/DecidaBERT-large-v1.1) ·
+[DecidaBERT-large-v2](https://huggingface.co/helmo/DecidaBERT-large-v2) ·
+[DecidaBERT-large](https://huggingface.co/helmo/DecidaBERT-large) (v1)
 """
 
-with gr.Blocks(title="DecidaBERT v1 vs v2", css=CSS) as demo:
+QUESTION_HELP = """**Question format.** `{"id": {"type": "noul" | "choice" | "score", "instructions": "...", "criteria": ..., "expected": ...}}`
+· `noul`: yes/no, no criteria needed; expected `yes` or `no`.
+· `choice`: criteria `{"label": "what it means", ...}`; expected a label.
+· `score`: criteria a list of levels, lowest first; expected a level number (0 = first) or its text.
+`expected` is optional; when given, both models are marked ✓ or ✗ against it."""
+
+with gr.Blocks(title="DecidaBERT v1.1 vs v2", css=CSS) as demo:
+    gold_state = gr.State(None)
     with gr.Column(elem_id="title"):
-        gr.Markdown("# DecidaBERT-large vs DecidaBERT-large-v2")
-        gr.Markdown("The same state and typed questions (`choice`, `score`, `noul`) go to both models; each returns a "
-                    "probability distribution per question from a single forward pass, no generated text. "
-                    "Pick an example built to show where they differ, or write your own.", elem_id="subtitle")
+        gr.Markdown("# DecidaBERT-large-v1.1 vs DecidaBERT-large-v2")
+        gr.Markdown("Two decision models, one forward pass each: send a state and typed questions (`choice`, `score`, `noul`), "
+                    "get a probability distribution per question, no generated text. Try a benchmark case with its gold "
+                    "answers, a hand-written demo, or your own.", elem_id="subtitle")
     with gr.Tab("Compare"):
-        first = next(iter(PRESETS))
-        preset = gr.Radio(list(PRESETS), value=first, label="Example")
-        why = gr.HTML(f'<div class="dc-why">{html.escape(PRESETS[first]["why"])}</div>')
         with gr.Row():
-            state = gr.Textbox(label="State", lines=6, max_lines=14, value=PRESETS[first]["state"])
-            questions = gr.Code(label="Questions (JSON)", language="json", value=json.dumps(PRESETS[first]["questions"], indent=2))
-        run = gr.Button("Run both", variant="primary")
+            with gr.Column(scale=3):
+                demo_pick = gr.Dropdown(list(DEMOS), value=next(iter(DEMOS)), label="Hand-written demo")
+            with gr.Column(scale=2):
+                bench_wf = gr.Dropdown(WORKFLOWS, value=WORKFLOWS[0], label="Benchmark case: workflow")
+            with gr.Column(scale=2):
+                bench_n = gr.Slider(0, 99, value=0, step=1, label="case number (test split)")
+            with gr.Column(scale=1, min_width=120):
+                bench_load = gr.Button("Load case")
+                bench_rand = gr.Button("Random case")
+        why = gr.HTML(f'<div class="dc-why">{html.escape(DEMOS[next(iter(DEMOS))]["why"])}</div>')
+        with gr.Row():
+            state = gr.Textbox(label="State (text or JSON)", lines=10, max_lines=24, value=DEMOS[next(iter(DEMOS))]["state"])
+            questions = gr.Code(label="Questions (JSON)", language="json", lines=10,
+                                value=json.dumps(DEMOS[next(iter(DEMOS))]["questions"], indent=2))
+        with gr.Accordion("Add a question", open=False):
+            gr.Markdown(QUESTION_HELP)
+            with gr.Row():
+                nq_id = gr.Textbox(label="id", placeholder="e.g. escalate")
+                nq_type = gr.Dropdown(["noul", "choice", "score"], value="noul", label="type")
+                nq_exp = gr.Textbox(label="expected (optional)", placeholder="yes / a label / a level number")
+            nq_ins = gr.Textbox(label="instructions", placeholder="e.g. Should this ticket go to a human agent?")
+            nq_opts = gr.Textbox(label="options (choice: one 'label: meaning' per line; score: one level per line, lowest first)", lines=3)
+            with gr.Row():
+                nq_add = gr.Button("Add to questions")
+                nq_clear = gr.Button("Clear all questions")
+            nq_msg = gr.Markdown()
+        run = gr.Button("Run both models", variant="primary")
         summary = gr.HTML()
         with gr.Row():
-            out1 = gr.HTML()
-            out2 = gr.HTML()
+            out_a = gr.HTML()
+            out_b = gr.HTML()
+            out_g = gr.HTML()
     with gr.Tab("Benchmark"):
-        gr.Markdown(BENCH)
-    preset.change(_load_preset, preset, [why, state, questions])
-    run.click(answer, [state, questions], [summary, out1, out2])
-    demo.load(answer, [state, questions], [summary, out1, out2])
+        gr.Markdown(BENCH_MD)
+
+    io = [why, state, questions, gold_state]
+    demo_pick.change(load_demo, demo_pick, io)
+    bench_load.click(load_bench, [bench_wf, bench_n], io)
+    bench_rand.click(random_bench, None, [bench_wf, bench_n] + io)
+    nq_add.click(add_question, [questions, nq_id, nq_type, nq_ins, nq_opts, nq_exp], [questions, nq_msg])
+    nq_clear.click(lambda: ("{}", "Cleared."), None, [questions, nq_msg])
+    run.click(answer, [state, questions, gold_state], [summary, out_a, out_b, out_g])
+    demo.load(answer, [state, questions, gold_state], [summary, out_a, out_b, out_g])
 
 if __name__ == "__main__":
     demo.launch()

model.py

@@ -1,241 +0,0 @@
-"""A standalone port of DecidaBERT-large's inference (the decision heads and the Laya-format prompt layout).
-
-This intentionally does not import the `decida` package: it is independent code for a Gradio Space, reading only
-the public checkpoint at https://huggingface.co/helmo/DecidaBERT-large. The algorithm matches
-`src/decida/model/enc.py` in the main project, ported here because ZeroGPU Spaces only support the Gradio SDK, not
-a plain server process like `decida serve`. If the real project's encoder logic changes, this file needs updating
-by hand; it is not generated from it.
-"""
-from __future__ import annotations
-
-import json
-import os
-from dataclasses import dataclass
-from enum import Enum
-from typing import Any
-
-import numpy as np
-import torch
-from safetensors.torch import load_file
-from torch import nn
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-
-MODEL_REPO = "helmo/DecidaBERT-large"
-
-
-class Primitive(str, Enum):
-    CHOICE = "choice"
-    SCORE = "score"
-    NOUL = "noul"
-
-
-@dataclass
-class Question:
-    qid: str
-    primitive: Primitive
-    instructions: str
-    criteria: Any
-
-
-@dataclass
-class Encoded:
-    input_ids: torch.Tensor
-    attention_mask: torch.Tensor
-    readout_positions: torch.Tensor
-    n_options: int
-    truncated: bool
-
-
-@dataclass
-class Batch:
-    input_ids: torch.Tensor
-    attention_mask: torch.Tensor
-    readout_positions: torch.Tensor
-    marker_mask: torch.Tensor
-    qtypes: torch.Tensor
-
-
-def _render_criterion(value: Any) -> str:
-    if isinstance(value, str):
-        return value
-    return json.dumps(value, ensure_ascii=False, separators=(", ", ": "), default=str)
-
-
-def _render_options(q: Question) -> list[str]:
-    if q.primitive == Primitive.CHOICE:
-        return [k if not v else f"{k}: {_render_criterion(v)}" for k, v in q.criteria.items()]
-    if q.primitive == Primitive.SCORE:
-        return [f"level {i}: {_render_criterion(c)}" for i, c in enumerate(q.criteria)]
-    crit = q.criteria or {}
-    false_str = _render_criterion(crit.get("false")) if crit.get("false") else "no, the statement does not hold"
-    true_str = _render_criterion(crit.get("true")) if crit.get("true") else "yes, the statement holds"
-    return [f"false: {false_str}", f"true: {true_str}"]
-
-
-class LayaLayout:
-    """Builds one [CLS] instructions [SEP] <mask>opt1 <mask>opt2 ... [SEP] state [SEP] sequence per question."""
-
-    def __init__(self, tokenizer, max_len: int = 2142, head_max_len: int = 1400, opt_max_tokens: int = 160):
-        self.tok, self.max_len, self.head_max_len, self.opt_max_tokens = tokenizer, max_len, head_max_len, opt_max_tokens
-
-    def build(self, state: str, q: Question) -> Encoded:
-        mask_tok = self.tok.mask_token
-        opts = _render_options(q)
-        ins = q.instructions if isinstance(q.instructions, str) else json.dumps(q.instructions, ensure_ascii=False)
-        if mask_tok:
-            ins = ins.replace(mask_tok, " ")
-        head_ids = self.tok(f"{q.primitive.value} question: {ins}", add_special_tokens=False)["input_ids"]
-
-        opt_ids = []
-        for o in opts:
-            text_ids = self.tok(" " + o.replace(mask_tok, " "), add_special_tokens=False)["input_ids"]
-            opt_ids.append([self.tok.mask_token_id] + text_ids[: self.opt_max_tokens])
-
-        opt_budget = self.head_max_len - sum(len(o) for o in opt_ids)
-        if opt_budget < 16:
-            per = max(4, (self.head_max_len - 16) // max(1, len(opt_ids)))
-            opt_ids = [o[:per] for o in opt_ids]
-            opt_budget = self.head_max_len - sum(len(o) for o in opt_ids)
-
-        head_ids = head_ids[: max(8, opt_budget)]
-        ids = [self.tok.cls_token_id] + head_ids + [self.tok.sep_token_id]
-        markers = []
-        for o in opt_ids:
-            markers.append(len(ids))
-            ids.extend(o)
-        ids.append(self.tok.sep_token_id)
-
-        room = max(0, self.max_len - len(ids) - 1)
-        state_str = json.dumps(state, ensure_ascii=False) if isinstance(state, (dict, list)) else str(state)
-        st = self.tok(state_str.replace(mask_tok, " "), add_special_tokens=False)["input_ids"]
-        truncated = len(st) > room
-        ids = ids + st[:room] + [self.tok.sep_token_id]
-
-        final_ids = ids[: self.max_len]
-        valid_markers = [m for m in markers if m < self.max_len]
-        return Encoded(
-            input_ids=torch.tensor(final_ids, dtype=torch.long),
-            attention_mask=torch.ones(len(final_ids), dtype=torch.long),
-            readout_positions=torch.tensor(valid_markers, dtype=torch.long),
-            n_options=len(opts),
-            truncated=truncated,
-        )
-
-
-class DecidaEncModel(nn.Module):
-    """The decision heads on top of the ModernBERT backbone: a type embedding, an optional small transformer
-    head, and a scorer that reads one logit out of each option's <mask> position."""
-
-    def __init__(self, encoder: nn.Module, head_layers: int = 2, n_act: int = 2, cfg: dict[str, Any] | None = None):
-        super().__init__()
-        self.encoder, self.cfg = encoder, cfg or {}
-        d = int(encoder.config.hidden_size)
-        layer = nn.TransformerEncoderLayer(d_model=d, nhead=max(1, d // 64), dim_feedforward=4 * d, dropout=0.1, batch_first=True, norm_first=True)
-        self.head = nn.TransformerEncoder(layer, head_layers, enable_nested_tensor=False) if head_layers > 0 else None
-        self.type_emb = nn.Embedding(3, d)
-        self.scorer = nn.Sequential(nn.LayerNorm(d), nn.Linear(d, d), nn.GELU(approximate="none"), nn.Linear(d, 1))
-        self.act_head = nn.Sequential(nn.Linear(d + 4, 256), nn.GELU(approximate="none"), nn.Linear(256, n_act))
-        self.qtypes_map = {"choice": 0, "score": 1, "noul": 2}
-
-    def forward(self, batch: Batch) -> torch.Tensor:
-        h = self.encoder(input_ids=batch.input_ids, attention_mask=batch.attention_mask).last_hidden_state
-        h = h + self.type_emb(batch.qtypes)[:, None, :]
-        if self.head is not None:
-            pad = ~batch.attention_mask.bool()
-            for layer in self.head.layers:
-                h = layer(h, src_key_padding_mask=pad)
-        idx = batch.readout_positions.clamp(min=0)[:, :, None].expand(-1, -1, h.size(-1))
-        logits = self.scorer(torch.gather(h, 1, idx)).squeeze(-1).float()
-        return logits.masked_fill(~batch.marker_mask, float("-inf"))
-
-    @classmethod
-    def from_pretrained(cls, path: str) -> DecidaEncModel:
-        weights = load_file(os.path.join(path, "model.safetensors"))
-        cfg_path = os.path.join(path, "rl_agent_config.json")
-        cfg = json.load(open(cfg_path)) if os.path.exists(cfg_path) else {}
-        ecfg = AutoConfig.from_pretrained(os.path.join(path, "encoder"))
-        enc = AutoModel.from_config(ecfg, attn_implementation="sdpa")
-
-        act_weight = weights.get("act_head.2.weight", weights.get("core_model.act_head.2.weight"))
-        if act_weight is None:
-            raise ValueError(f"{path} has no act_head.2.weight: not a Decida or Laya encoder checkpoint")
-        n_act, head_layers, new_weights = act_weight.shape[0], 0, {}
-        for k, v in weights.items():
-            k = k.removeprefix("core_model.")
-            new_weights[k] = v
-            if k.startswith("head.layers."):
-                head_layers = max(head_layers, int(k.split(".")[2]) + 1)
-
-        model = cls(enc, head_layers=head_layers, n_act=n_act, cfg=cfg)
-        model.load_state_dict(new_weights, strict=False)
-        return model
-
-
-def load(device: str = "cuda") -> tuple[DecidaEncModel, LayaLayout]:
-    """Download `helmo/DecidaBERT-large` from the Hub and build the model plus its prompt layout."""
-    from huggingface_hub import snapshot_download
-
-    path = snapshot_download(MODEL_REPO)
-    model = DecidaEncModel.from_pretrained(path).to(device).eval()
-    tokenizer = AutoTokenizer.from_pretrained(os.path.join(path, "tokenizer") if os.path.isdir(os.path.join(path, "tokenizer")) else path)
-    # DecidaBERT-large was trained/prompted at a 2142-token context; the checkpoint's own config can override this.
-    layout = LayaLayout(tokenizer, max_len=model.cfg.get("max_len", 2142),
-                         head_max_len=model.cfg.get("head_max_len", 1400), opt_max_tokens=model.cfg.get("opt_max_tokens", 160))
-    return model, layout
-
-
-@torch.no_grad()
-def predict(model: DecidaEncModel, layout: LayaLayout, state: str, questions: dict[str, Question]) -> dict[str, dict[str, Any]]:
-    """Answer every question in one batched forward pass. Returns {qid: {type, ...distribution fields}}."""
-    items = [{"qid": qid, "encoded": layout.build(state, q), "qtype": model.qtypes_map[q.primitive.value]} for qid, q in questions.items()]
-    if not items:
-        return {}
-
-    L = max(len(it["encoded"].input_ids) for it in items)
-    kmax = max(it["encoded"].n_options for it in items)
-    n = len(items)
-    pad_id = layout.tok.pad_token_id or 0
-    ids = torch.full((n, L), pad_id, dtype=torch.long)
-    att = torch.zeros((n, L), dtype=torch.long)
-    mpos = torch.zeros((n, kmax), dtype=torch.long)
-    mmask = torch.zeros((n, kmax), dtype=torch.bool)
-    qtypes = torch.zeros(n, dtype=torch.long)
-    for i, it in enumerate(items):
-        enc = it["encoded"]
-        ids[i, : len(enc.input_ids)] = enc.input_ids
-        att[i, : len(enc.attention_mask)] = enc.attention_mask
-        mpos[i, : enc.n_options] = enc.readout_positions
-        mmask[i, : enc.n_options] = True
-        qtypes[i] = it["qtype"]
-
-    device = next(model.parameters()).device
-    batch = Batch(ids.to(device), att.to(device), mpos.to(device), mmask.to(device), qtypes.to(device))
-    logits = model.forward(batch).cpu().numpy()
-
-    qtypes_inv = {v: k for k, v in model.qtypes_map.items()}
-    temp_array = model.cfg.get("temperature", [1.0, 1.0, 1.0])
-    temp_by_opt = model.cfg.get("temperature_by_options", {})
-
-    answers: dict[str, dict[str, Any]] = {}
-    for r, it in enumerate(items):
-        qid, q, k = it["qid"], questions[it["qid"]], it["encoded"].n_options
-        z = logits[r, :k]
-        size = "2" if k <= 2 else "3-5" if k <= 5 else "6-10" if k <= 10 else "11+"
-        bucket = f"{qtypes_inv[it['qtype']]}:{size}"
-        t_scale = temp_by_opt.get(bucket, temp_array[it["qtype"]])
-        z = z / max(1e-3, float(t_scale))
-        p = np.exp(z - z.max())
-        p = p / p.sum()
-
-        if q.primitive == Primitive.CHOICE:
-            keys = list(q.criteria.keys())
-            probs = {kk: float(v) for kk, v in zip(keys, p)}
-            answers[qid] = {"type": "choice", "choice": keys[int(p.argmax())], "probabilities": probs, "confidence": float(max(probs.values()))}
-        elif q.primitive == Primitive.SCORE:
-            probs = {str(i): float(v) for i, v in enumerate(p)}
-            answers[qid] = {"type": "score", "score": float((np.arange(k) * p).sum()),
-                             "legend": {str(i): str(c) for i, c in enumerate(q.criteria)}, "probabilities": probs, "confidence": float(max(probs.values()))}
-        else:
-            noul_prob = float(p[1])
-            answers[qid] = {"type": "noul", "noul": noul_prob, "confidence": float(noul_prob if noul_prob > 0.5 else 1 - noul_prob)}
-    return answers

requirements.txt

@@ -4,4 +4,6 @@ transformers>=5.0
 safetensors>=0.4.0
 numpy
 huggingface_hub
+pandas
+pyarrow
 requests