| { |
| "version": 1, |
| "context_length": 2048, |
| "examples": [ |
| { |
| "id": "customer_service_000072", |
| "source": { |
| "dataset": "LocalLLaMA/typed-decisions", |
| "revision": "d0e2f0c42fef86cc15d1688d25a19f5ba7c85b18", |
| "config": "all", |
| "split": "test", |
| "row_id": "customer_service_000072", |
| "question_ids": [ |
| "action", |
| "category", |
| "needs_human", |
| "urgency" |
| ], |
| "selection": "selected jointly correct native multiquestion subset, not aggregate evaluation" |
| }, |
| "input": { |
| "state": { |
| "account": { |
| "lifetime_value_usd": 4732, |
| "prior_tickets_90d": 3, |
| "seats": 1, |
| "tenure_months": 60, |
| "tier": "free" |
| }, |
| "orders": [ |
| { |
| "amount_usd": 87, |
| "date": "2026-01-01", |
| "id": "A-96646", |
| "status": "settled" |
| } |
| ], |
| "thread": [ |
| { |
| "role": "customer", |
| "text": "I need a copy of the invoice from my account ASAP – I've been a free‑plan user for five years, paid about $87, and I've already opened three tickets about this that never got answered. If this isn’t fixed immediately, I’m done." |
| }, |
| { |
| "role": "agent", |
| "text": "I’m sorry you’ve had to wait. Could you let me know the invoice date or the reference number you’re looking for?" |
| }, |
| { |
| "role": "customer", |
| "text": "It’s the one from March 2024; I’ve asked for it three times already and got nothing. Get it to me now or I’m leaving." |
| } |
| ] |
| }, |
| "questions": [ |
| { |
| "type": "choice", |
| "instructions": "What should the assistant do next with this conversation?", |
| "options": [ |
| "answer_directly: The assistant can resolve this itself with information it already has.", |
| "close_no_action: No further action is warranted; the matter is settled.", |
| "escalate_to_human: Hand off to a human agent with the appropriate authority.", |
| "execute_refund: Issue the refund or credit the customer is owed.", |
| "request_information: More detail is needed from the customer before anything can be done." |
| ] |
| }, |
| { |
| "type": "choice", |
| "instructions": "What is this customer conversation primarily about?", |
| "options": [ |
| "account: Login, plan changes, profile or access management.", |
| "billing: A charge, invoice, subscription or payment problem.", |
| "delivery: Shipping, fulfilment or delivery of a physical item.", |
| "refund: The customer is explicitly asking for money back.", |
| "technical: The product or service is not working as expected." |
| ] |
| }, |
| { |
| "type": "noul", |
| "instructions": "This conversation requires a human agent rather than automated handling. (false: Automation can carry this to resolution.; true: A human must take over: judgement, authority or empathy is required.)", |
| "options": [] |
| }, |
| { |
| "type": "score", |
| "instructions": "How time-sensitive is this conversation?", |
| "options": [ |
| "No time pressure; can wait indefinitely.", |
| "Routine; handle within the normal queue.", |
| "Elevated; should be handled within the same week.", |
| "Critical; requires action within the same day." |
| ] |
| } |
| ] |
| }, |
| "gold": [ |
| { |
| "qid": "action", |
| "type": "choice", |
| "label": "escalate_to_human", |
| "labels": [ |
| "answer_directly", |
| "close_no_action", |
| "escalate_to_human", |
| "execute_refund", |
| "request_information" |
| ], |
| "probabilities": { |
| "answer_directly": 0.216667, |
| "close_no_action": 0.033333, |
| "escalate_to_human": 0.466667, |
| "execute_refund": 0.0, |
| "request_information": 0.283333 |
| } |
| }, |
| { |
| "qid": "category", |
| "type": "choice", |
| "label": "billing", |
| "labels": [ |
| "account", |
| "billing", |
| "delivery", |
| "refund", |
| "technical" |
| ], |
| "probabilities": { |
| "account": 0.01, |
| "billing": 0.976667, |
| "delivery": 0.003333, |
| "refund": 0.003333, |
| "technical": 0.006667 |
| } |
| }, |
| { |
| "qid": "needs_human", |
| "type": "noul", |
| "label": "true", |
| "labels": [ |
| "false", |
| "true" |
| ], |
| "probabilities": { |
| "false": 0.223333, |
| "true": 0.776667 |
| } |
| }, |
| { |
| "qid": "urgency", |
| "type": "score", |
| "label": "3", |
| "labels": [ |
| "0", |
| "1", |
| "2", |
| "3" |
| ], |
| "probabilities": { |
| "0": 0.0, |
| "1": 0.05, |
| "2": 0.333333, |
| "3": 0.616667 |
| } |
| } |
| ], |
| "outputs": { |
| "sys1-nano": [ |
| { |
| "type": "choice", |
| "probs": [ |
| 0.23520175316958344, |
| 0.2532956643533352, |
| 0.27248106580225123, |
| 0.058664757941379944, |
| 0.18035675873345017 |
| ], |
| "argmax": 2, |
| "confidence": 0.09060133225281403 |
| }, |
| { |
| "type": "choice", |
| "probs": [ |
| 0.3029430517445156, |
| 0.5075163294962111, |
| 0.054784355974919735, |
| 0.09262472149882246, |
| 0.042131541285531295 |
| ], |
| "argmax": 1, |
| "confidence": 0.38439541187026377 |
| }, |
| { |
| "type": "noul", |
| "p": 0.5053835439453084, |
| "logit": 0.024785444140434265 |
| }, |
| { |
| "type": "score", |
| "probs": [ |
| 0.19241074857351476, |
| 0.19781303315947876, |
| 0.22827297021271375, |
| 0.38150324805429275 |
| ], |
| "argmax": 3, |
| "value": 1.7988687177477845 |
| } |
| ], |
| "sys1-micro": [ |
| { |
| "type": "choice", |
| "probs": [ |
| 0.2005481482047673, |
| 0.2667642078012643, |
| 0.27163273613918687, |
| 0.049486456372734924, |
| 0.21156845148204648 |
| ], |
| "argmax": 2, |
| "confidence": 0.08954092017398357 |
| }, |
| { |
| "type": "choice", |
| "probs": [ |
| 0.11180067612102801, |
| 0.7409786051189173, |
| 0.030379424329880025, |
| 0.03480601929837383, |
| 0.08203527513180081 |
| ], |
| "argmax": 1, |
| "confidence": 0.6762232563986467 |
| }, |
| { |
| "type": "noul", |
| "p": 0.6315588777255923, |
| "logit": 0.7081672549247742 |
| }, |
| { |
| "type": "score", |
| "probs": [ |
| 0.085208882098541, |
| 0.17460809874550334, |
| 0.31188678591110985, |
| 0.42829623324484584 |
| ], |
| "argmax": 3, |
| "value": 2.0832703703022606 |
| } |
| ] |
| }, |
| "correct_by_model": { |
| "sys1-nano": [ |
| true, |
| true, |
| true, |
| true |
| ], |
| "sys1-micro": [ |
| true, |
| true, |
| true, |
| true |
| ] |
| } |
| }, |
| { |
| "id": "security_incidents_000030", |
| "source": { |
| "dataset": "LocalLLaMA/typed-decisions", |
| "revision": "d0e2f0c42fef86cc15d1688d25a19f5ba7c85b18", |
| "config": "all", |
| "split": "test", |
| "row_id": "security_incidents_000030", |
| "question_ids": [ |
| "credential_compromise", |
| "disposition", |
| "urgency" |
| ], |
| "selection": "selected jointly correct native multiquestion subset, not aggregate evaluation" |
| }, |
| "input": { |
| "state": { |
| "alert": { |
| "description": "an account granted itself or received elevated permissions", |
| "evidence": "On 2026-09-14 22:07 UTC, contractor account \"csmith\" used PowerShell on DC01 (host 10.1.8.4) to add itself to the \"Domain Admins\" group. The action originated from an unrecognized source IP 172.16.254.9. MFA was active for this account, and its credentials were last rotated 90 days earlier.", |
| "rule": "privilege_escalation" |
| }, |
| "context": { |
| "asset_criticality": "medium", |
| "change_window_active": false, |
| "source_on_allowlist": false |
| }, |
| "history": { |
| "credential_rotation_days_ago": 90, |
| "distinct_countries_30d": 3, |
| "logins_30d": 1946, |
| "prior_alerts_90d": 1 |
| }, |
| "principal": { |
| "mfa_enrolled": true, |
| "name": "ci-infra-171", |
| "privileges": [ |
| "deploy:production", |
| "secrets:read" |
| ], |
| "type": "contractor" |
| } |
| }, |
| "questions": [ |
| { |
| "type": "noul", |
| "instructions": "The evidence indicates a credential or account has been compromised.", |
| "options": [] |
| }, |
| { |
| "type": "choice", |
| "instructions": "How should this security alert be dispositioned?", |
| "options": [ |
| "close_benign: Expected, explainable activity; close without analyst time.", |
| "contain: Contain the host or account immediately; do not wait for triage.", |
| "investigate: Warrants an analyst opening an investigation.", |
| "monitor: Not clearly malicious, but worth watching for recurrence." |
| ] |
| }, |
| { |
| "type": "score", |
| "instructions": "How quickly must a responder act on this alert?", |
| "options": [ |
| "No time pressure; can wait indefinitely.", |
| "Routine; handle within the normal queue.", |
| "Elevated; should be handled within the same week.", |
| "Critical; requires action within the same day." |
| ] |
| } |
| ] |
| }, |
| "gold": [ |
| { |
| "qid": "credential_compromise", |
| "type": "noul", |
| "label": "true", |
| "labels": [ |
| "false", |
| "true" |
| ], |
| "probabilities": { |
| "false": 0.406667, |
| "true": 0.593333 |
| } |
| }, |
| { |
| "qid": "disposition", |
| "type": "choice", |
| "label": "contain", |
| "labels": [ |
| "close_benign", |
| "contain", |
| "investigate", |
| "monitor" |
| ], |
| "probabilities": { |
| "close_benign": 0.003333, |
| "contain": 0.56, |
| "investigate": 0.183333, |
| "monitor": 0.253333 |
| } |
| }, |
| { |
| "qid": "urgency", |
| "type": "score", |
| "label": "3", |
| "labels": [ |
| "0", |
| "1", |
| "2", |
| "3" |
| ], |
| "probabilities": { |
| "0": 0.0, |
| "1": 0.083333, |
| "2": 0.3, |
| "3": 0.616667 |
| } |
| } |
| ], |
| "outputs": { |
| "sys1-nano": [ |
| { |
| "type": "noul", |
| "p": 0.9627577511226095, |
| "logit": 3.743260145187378 |
| }, |
| { |
| "type": "choice", |
| "probs": [ |
| 0.15567752502477153, |
| 0.36718717228324227, |
| 0.3255476682569624, |
| 0.15158763443502388 |
| ], |
| "argmax": 1, |
| "confidence": 0.15624956304432303 |
| }, |
| { |
| "type": "score", |
| "probs": [ |
| 0.0505407421177171, |
| 0.1841486483744628, |
| 0.2518935849612855, |
| 0.5134170245465346 |
| ], |
| "argmax": 3, |
| "value": 2.2281868919366374 |
| } |
| ], |
| "sys1-micro": [ |
| { |
| "type": "noul", |
| "p": 0.5973868393245925, |
| "logit": 0.5185182094573975 |
| }, |
| { |
| "type": "choice", |
| "probs": [ |
| 0.2707563139405646, |
| 0.3422287384455644, |
| 0.11215031731337624, |
| 0.27486463030049474 |
| ], |
| "argmax": 1, |
| "confidence": 0.12297165126075253 |
| }, |
| { |
| "type": "score", |
| "probs": [ |
| 0.0993003655186528, |
| 0.2947025513291329, |
| 0.21563770934925663, |
| 0.39035937380295754 |
| ], |
| "argmax": 3, |
| "value": 1.8970560914365189 |
| } |
| ] |
| }, |
| "correct_by_model": { |
| "sys1-nano": [ |
| true, |
| true, |
| true |
| ], |
| "sys1-micro": [ |
| true, |
| true, |
| true |
| ] |
| } |
| }, |
| { |
| "id": "security_incidents_000069", |
| "source": { |
| "dataset": "LocalLLaMA/typed-decisions", |
| "revision": "d0e2f0c42fef86cc15d1688d25a19f5ba7c85b18", |
| "config": "all", |
| "split": "test", |
| "row_id": "security_incidents_000069", |
| "question_ids": [ |
| "credential_compromise", |
| "severity", |
| "true_positive", |
| "urgency" |
| ], |
| "selection": "selected jointly correct native multiquestion subset, not aggregate evaluation" |
| }, |
| "input": { |
| "state": { |
| "alert": { |
| "description": "an endpoint agent matched a known signature", |
| "evidence": "On 2026-09-16 08:27:53 UTC the CI runner \"gitlab-runner-12\" on host build-01 triggered an endpoint‑agent alert for file \"C:\\temp\\malicious.exe\" matching the known signature \"Trojan:Win32/Emotet.B\". The process \"malicious.exe\" was executed under the unprivileged runner account with MFA enabled; the runner's credentials were last rotated three days prior.", |
| "rule": "malware_signature" |
| }, |
| "context": { |
| "asset_criticality": "crown_jewel", |
| "change_window_active": false, |
| "source_on_allowlist": false |
| }, |
| "history": { |
| "credential_rotation_days_ago": 3, |
| "distinct_countries_30d": 3, |
| "logins_30d": 1404, |
| "prior_alerts_90d": 8 |
| }, |
| "principal": { |
| "mfa_enrolled": true, |
| "name": "svc-billing-746", |
| "privileges": [ |
| "repo:read" |
| ], |
| "type": "ci_runner" |
| } |
| }, |
| "questions": [ |
| { |
| "type": "noul", |
| "instructions": "The evidence indicates a credential or account has been compromised.", |
| "options": [] |
| }, |
| { |
| "type": "score", |
| "instructions": "How severe is the potential impact if this alert is real?", |
| "options": [ |
| "Negligible: no access to anything sensitive.", |
| "Low: limited access, easily reversed.", |
| "Moderate: access to internal systems or non-public data.", |
| "High: access to production, secrets or customer data.", |
| "Critical: active compromise of crown-jewel systems." |
| ] |
| }, |
| { |
| "type": "noul", |
| "instructions": "This alert reflects genuinely malicious or unauthorised activity. (false: Benign activity, a misconfiguration, or a known false positive.; true: The underlying behaviour is malicious or unauthorised.)", |
| "options": [] |
| }, |
| { |
| "type": "score", |
| "instructions": "How quickly must a responder act on this alert?", |
| "options": [ |
| "No time pressure; can wait indefinitely.", |
| "Routine; handle within the normal queue.", |
| "Elevated; should be handled within the same week.", |
| "Critical; requires action within the same day." |
| ] |
| } |
| ] |
| }, |
| "gold": [ |
| { |
| "qid": "credential_compromise", |
| "type": "noul", |
| "label": "true", |
| "labels": [ |
| "false", |
| "true" |
| ], |
| "probabilities": { |
| "false": 0.433333, |
| "true": 0.566667 |
| } |
| }, |
| { |
| "qid": "severity", |
| "type": "score", |
| "label": "4", |
| "labels": [ |
| "0", |
| "1", |
| "2", |
| "3", |
| "4" |
| ], |
| "probabilities": { |
| "0": 0.003333, |
| "1": 0.03, |
| "2": 0.116667, |
| "3": 0.4, |
| "4": 0.45 |
| } |
| }, |
| { |
| "qid": "true_positive", |
| "type": "noul", |
| "label": "true", |
| "labels": [ |
| "false", |
| "true" |
| ], |
| "probabilities": { |
| "false": 0.083333, |
| "true": 0.916667 |
| } |
| }, |
| { |
| "qid": "urgency", |
| "type": "score", |
| "label": "3", |
| "labels": [ |
| "0", |
| "1", |
| "2", |
| "3" |
| ], |
| "probabilities": { |
| "0": 0.003333, |
| "1": 0.063333, |
| "2": 0.2, |
| "3": 0.733333 |
| } |
| } |
| ], |
| "outputs": { |
| "sys1-nano": [ |
| { |
| "type": "noul", |
| "p": 0.6907134024710205, |
| "logit": 0.9247282147407532 |
| }, |
| { |
| "type": "score", |
| "probs": [ |
| 0.010825525386043254, |
| 0.043146320719381966, |
| 0.03647054584189759, |
| 0.1122763436922054, |
| 0.7972812643604718 |
| ], |
| "argmax": 4, |
| "value": 3.6420415009216804 |
| }, |
| { |
| "type": "noul", |
| "p": 0.8073136767536342, |
| "logit": 1.6488888263702393 |
| }, |
| { |
| "type": "score", |
| "probs": [ |
| 0.06038556540706689, |
| 0.2182734730314863, |
| 0.3018875423411267, |
| 0.4194534192203201 |
| ], |
| "argmax": 3, |
| "value": 2.0804088153747 |
| } |
| ], |
| "sys1-micro": [ |
| { |
| "type": "noul", |
| "p": 0.5621988523952844, |
| "logit": 0.3286375105381012 |
| }, |
| { |
| "type": "score", |
| "probs": [ |
| 0.011758020589272063, |
| 0.045895041089217596, |
| 0.19454460729903852, |
| 0.2713845595832922, |
| 0.47641777143917946 |
| ], |
| "argmax": 4, |
| "value": 3.1548090201938894 |
| }, |
| { |
| "type": "noul", |
| "p": 0.725215110808429, |
| "logit": 1.2752809524536133 |
| }, |
| { |
| "type": "score", |
| "probs": [ |
| 0.09492541576483614, |
| 0.2532985791381369, |
| 0.20916129067028402, |
| 0.44261471442674294 |
| ], |
| "argmax": 3, |
| "value": 1.9994653037589338 |
| } |
| ] |
| }, |
| "correct_by_model": { |
| "sys1-nano": [ |
| true, |
| true, |
| true, |
| true |
| ], |
| "sys1-micro": [ |
| true, |
| true, |
| true, |
| true |
| ] |
| } |
| }, |
| { |
| "id": "security_incidents_000046", |
| "source": { |
| "dataset": "LocalLLaMA/typed-decisions", |
| "revision": "d0e2f0c42fef86cc15d1688d25a19f5ba7c85b18", |
| "config": "all", |
| "split": "test", |
| "row_id": "security_incidents_000046", |
| "question_ids": [ |
| "credential_compromise", |
| "severity", |
| "true_positive", |
| "urgency" |
| ], |
| "selection": "selected jointly correct native multiquestion subset, not aggregate evaluation" |
| }, |
| "input": { |
| "state": { |
| "alert": { |
| "description": "an unrecognised binary executed on a server", |
| "evidence": "At 2026-09-15T22:13:58Z the admin account \"admin_jane\" launched an executable named \"c37b9e.exe\" from C:\\Windows\\Temp\\ on server \"prod-db-01\". The binary is not recognized in the software catalog, MFA was disabled for the account, and its password had not been changed for 412 days.", |
| "rule": "suspicious_process" |
| }, |
| "context": { |
| "asset_criticality": "medium", |
| "change_window_active": false, |
| "source_on_allowlist": false |
| }, |
| "history": { |
| "credential_rotation_days_ago": 412, |
| "distinct_countries_30d": 4, |
| "logins_30d": 556, |
| "prior_alerts_90d": 3 |
| }, |
| "principal": { |
| "mfa_enrolled": false, |
| "name": "usr-support-766", |
| "privileges": [ |
| "deploy:production", |
| "secrets:read" |
| ], |
| "type": "admin" |
| } |
| }, |
| "questions": [ |
| { |
| "type": "noul", |
| "instructions": "The evidence indicates a credential or account has been compromised.", |
| "options": [] |
| }, |
| { |
| "type": "score", |
| "instructions": "How severe is the potential impact if this alert is real?", |
| "options": [ |
| "Negligible: no access to anything sensitive.", |
| "Low: limited access, easily reversed.", |
| "Moderate: access to internal systems or non-public data.", |
| "High: access to production, secrets or customer data.", |
| "Critical: active compromise of crown-jewel systems." |
| ] |
| }, |
| { |
| "type": "noul", |
| "instructions": "This alert reflects genuinely malicious or unauthorised activity. (false: Benign activity, a misconfiguration, or a known false positive.; true: The underlying behaviour is malicious or unauthorised.)", |
| "options": [] |
| }, |
| { |
| "type": "score", |
| "instructions": "How quickly must a responder act on this alert?", |
| "options": [ |
| "No time pressure; can wait indefinitely.", |
| "Routine; handle within the normal queue.", |
| "Elevated; should be handled within the same week.", |
| "Critical; requires action within the same day." |
| ] |
| } |
| ] |
| }, |
| "gold": [ |
| { |
| "qid": "credential_compromise", |
| "type": "noul", |
| "label": "true", |
| "labels": [ |
| "false", |
| "true" |
| ], |
| "probabilities": { |
| "false": 0.35, |
| "true": 0.65 |
| } |
| }, |
| { |
| "qid": "severity", |
| "type": "score", |
| "label": "3", |
| "labels": [ |
| "0", |
| "1", |
| "2", |
| "3", |
| "4" |
| ], |
| "probabilities": { |
| "0": 0.013333, |
| "1": 0.036667, |
| "2": 0.266667, |
| "3": 0.566667, |
| "4": 0.116667 |
| } |
| }, |
| { |
| "qid": "true_positive", |
| "type": "noul", |
| "label": "true", |
| "labels": [ |
| "false", |
| "true" |
| ], |
| "probabilities": { |
| "false": 0.223333, |
| "true": 0.776667 |
| } |
| }, |
| { |
| "qid": "urgency", |
| "type": "score", |
| "label": "3", |
| "labels": [ |
| "0", |
| "1", |
| "2", |
| "3" |
| ], |
| "probabilities": { |
| "0": 0.033333, |
| "1": 0.1, |
| "2": 0.283333, |
| "3": 0.583333 |
| } |
| } |
| ], |
| "outputs": { |
| "sys1-nano": [ |
| { |
| "type": "noul", |
| "p": 0.8260771007373111, |
| "logit": 1.7932478189468384 |
| }, |
| { |
| "type": "score", |
| "probs": [ |
| 0.11585200749573464, |
| 0.09276030468497909, |
| 0.20493821087897482, |
| 0.3446104562895984, |
| 0.24183902065071305 |
| ], |
| "argmax": 3, |
| "value": 2.5038241779145762 |
| }, |
| { |
| "type": "noul", |
| "p": 0.978391229239724, |
| "logit": 4.3883056640625 |
| }, |
| { |
| "type": "score", |
| "probs": [ |
| 0.08819562852658062, |
| 0.14345711516084683, |
| 0.2291628879471023, |
| 0.5391843683654702 |
| ], |
| "argmax": 3, |
| "value": 2.2193359961514623 |
| } |
| ], |
| "sys1-micro": [ |
| { |
| "type": "noul", |
| "p": 0.6620399516430859, |
| "logit": 0.8835801482200623 |
| }, |
| { |
| "type": "score", |
| "probs": [ |
| 0.003880996398078084, |
| 0.013101634222409604, |
| 0.18421949035653218, |
| 0.7772363867968501, |
| 0.021561492226129894 |
| ], |
| "argmax": 3, |
| "value": 2.799495744230544 |
| }, |
| { |
| "type": "noul", |
| "p": 0.6179108879400087, |
| "logit": 0.631662130355835 |
| }, |
| { |
| "type": "score", |
| "probs": [ |
| 0.0717564910077619, |
| 0.19664117732863817, |
| 0.16292774178173414, |
| 0.5686745898818657 |
| ], |
| "argmax": 3, |
| "value": 2.2285204305377038 |
| } |
| ] |
| }, |
| "correct_by_model": { |
| "sys1-nano": [ |
| true, |
| true, |
| true, |
| true |
| ], |
| "sys1-micro": [ |
| true, |
| true, |
| true, |
| true |
| ] |
| } |
| } |
| ] |
| } |
|
|
