| { |
| "model": "OpenJev-4B", |
| "comparison_models": { |
| "openjev": "OpenJev-4B", |
| "jev": "Jev", |
| "decider": "Decider 4B", |
| "jevk5": "JevK5 4B", |
| "intern": "Intern-Decision-4B" |
| }, |
| "versions": { |
| "jev": "1.13.0", |
| "decider": "2.1", |
| "jevk5": "0.3" |
| }, |
| "snapshot_date": "2026-09-29", |
| "rows": [ |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Answer preference", |
| "n": 185, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 90.81, |
| "brier": 0.1236 |
| }, |
| "jev": { |
| "accuracy_percent": 88.11, |
| "brier": 0.1794 |
| }, |
| "decider": { |
| "accuracy_percent": 85.95, |
| "brier": 0.2263 |
| }, |
| "jevk5": { |
| "accuracy_percent": 77.84, |
| "brier": 0.3276 |
| }, |
| "intern": { |
| "accuracy_percent": 77.84, |
| "brier": 0.3142 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Code / database", |
| "n": 122, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 84.43, |
| "brier": 0.2128 |
| }, |
| "jev": { |
| "accuracy_percent": 83.61, |
| "brier": 0.2189 |
| }, |
| "decider": { |
| "accuracy_percent": 77.05, |
| "brier": 0.3293 |
| }, |
| "jevk5": { |
| "accuracy_percent": 72.95, |
| "brier": 0.3546 |
| }, |
| "intern": { |
| "accuracy_percent": 68.03, |
| "brier": 0.3364 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Commonsense context", |
| "n": 355, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 93.8, |
| "brier": 0.0854 |
| }, |
| "jev": { |
| "accuracy_percent": 96.9, |
| "brier": 0.0409 |
| }, |
| "decider": { |
| "accuracy_percent": 95.21, |
| "brier": 0.0687 |
| }, |
| "jevk5": { |
| "accuracy_percent": 93.8, |
| "brier": 0.1073 |
| }, |
| "intern": { |
| "accuracy_percent": 87.04, |
| "brier": 0.2096 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Constraint planning", |
| "n": 334, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 76.05, |
| "brier": 0.2944 |
| }, |
| "jev": { |
| "accuracy_percent": 75.75, |
| "brier": 0.2859 |
| }, |
| "decider": { |
| "accuracy_percent": 71.26, |
| "brier": 0.3685 |
| }, |
| "jevk5": { |
| "accuracy_percent": 64.37, |
| "brier": 0.4816 |
| }, |
| "intern": { |
| "accuracy_percent": 58.38, |
| "brier": 0.5222 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Deductive reasoning", |
| "n": 619, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 90.47, |
| "brier": 0.1389 |
| }, |
| "jev": { |
| "accuracy_percent": 94.51, |
| "brier": 0.0781 |
| }, |
| "decider": { |
| "accuracy_percent": 86.27, |
| "brier": 0.1861 |
| }, |
| "jevk5": { |
| "accuracy_percent": 79.97, |
| "brier": 0.2809 |
| }, |
| "intern": { |
| "accuracy_percent": 68.66, |
| "brier": 0.3848 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Emotion recognition", |
| "n": 9, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 100.0, |
| "brier": 0.0243 |
| }, |
| "jev": { |
| "accuracy_percent": 77.78, |
| "brier": 0.2353 |
| }, |
| "decider": { |
| "accuracy_percent": 100.0, |
| "brier": 0.0684 |
| }, |
| "jevk5": { |
| "accuracy_percent": 77.78, |
| "brier": 0.3633 |
| }, |
| "intern": { |
| "accuracy_percent": 77.78, |
| "brier": 0.369 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Evidence validation", |
| "n": 177, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 91.53, |
| "brier": 0.1221 |
| }, |
| "jev": { |
| "accuracy_percent": 90.96, |
| "brier": 0.1211 |
| }, |
| "decider": { |
| "accuracy_percent": 85.31, |
| "brier": 0.2103 |
| }, |
| "jevk5": { |
| "accuracy_percent": 88.14, |
| "brier": 0.196 |
| }, |
| "intern": { |
| "accuracy_percent": 84.18, |
| "brier": 0.2301 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Intent routing", |
| "n": 253, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 94.07, |
| "brier": 0.1045 |
| }, |
| "jev": { |
| "accuracy_percent": 93.28, |
| "brier": 0.1047 |
| }, |
| "decider": { |
| "accuracy_percent": 92.49, |
| "brier": 0.1223 |
| }, |
| "jevk5": { |
| "accuracy_percent": 87.75, |
| "brier": 0.1688 |
| }, |
| "intern": { |
| "accuracy_percent": 64.82, |
| "brier": 0.2352 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Knowledge QA", |
| "n": 572, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 95.63, |
| "brier": 0.0693 |
| }, |
| "jev": { |
| "accuracy_percent": 98.95, |
| "brier": 0.0202 |
| }, |
| "decider": { |
| "accuracy_percent": 96.85, |
| "brier": 0.0527 |
| }, |
| "jevk5": { |
| "accuracy_percent": 94.76, |
| "brier": 0.0847 |
| }, |
| "intern": { |
| "accuracy_percent": 88.99, |
| "brier": 0.1722 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Multi-hop reasoning", |
| "n": 211, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 92.42, |
| "brier": 0.0992 |
| }, |
| "jev": { |
| "accuracy_percent": 89.57, |
| "brier": 0.1274 |
| }, |
| "decider": { |
| "accuracy_percent": 73.93, |
| "brier": 0.3978 |
| }, |
| "jevk5": { |
| "accuracy_percent": 74.41, |
| "brier": 0.3046 |
| }, |
| "intern": { |
| "accuracy_percent": 64.45, |
| "brier": 0.4399 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Option logic", |
| "n": 287, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 51.92, |
| "brier": 0.5694 |
| }, |
| "jev": { |
| "accuracy_percent": 41.11, |
| "brier": 0.6551 |
| }, |
| "decider": { |
| "accuracy_percent": 34.84, |
| "brier": 0.7715 |
| }, |
| "jevk5": { |
| "accuracy_percent": 35.19, |
| "brier": 0.7176 |
| }, |
| "intern": { |
| "accuracy_percent": 28.22, |
| "brier": 0.7979 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Policy exceptions", |
| "n": 576, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 95.31, |
| "brier": 0.0688 |
| }, |
| "jev": { |
| "accuracy_percent": 90.97, |
| "brier": 0.1301 |
| }, |
| "decider": { |
| "accuracy_percent": 82.81, |
| "brier": 0.262 |
| }, |
| "jevk5": { |
| "accuracy_percent": 72.74, |
| "brier": 0.3858 |
| }, |
| "intern": { |
| "accuracy_percent": 63.89, |
| "brier": 0.463 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Quantitative reasoning", |
| "n": 330, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 69.09, |
| "brier": 0.3752 |
| }, |
| "jev": { |
| "accuracy_percent": 77.88, |
| "brier": 0.2841 |
| }, |
| "decider": { |
| "accuracy_percent": 51.21, |
| "brier": 0.6098 |
| }, |
| "jevk5": { |
| "accuracy_percent": 46.36, |
| "brier": 0.6636 |
| }, |
| "intern": { |
| "accuracy_percent": 37.58, |
| "brier": 0.7533 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Reading comprehension", |
| "n": 48, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 95.83, |
| "brier": 0.0523 |
| }, |
| "jev": { |
| "accuracy_percent": 100.0, |
| "brier": 0.0091 |
| }, |
| "decider": { |
| "accuracy_percent": 95.83, |
| "brier": 0.064 |
| }, |
| "jevk5": { |
| "accuracy_percent": 91.67, |
| "brier": 0.1074 |
| }, |
| "intern": { |
| "accuracy_percent": 89.58, |
| "brier": 0.1868 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Score rubrics", |
| "n": 135, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 78.52, |
| "brier": 0.2939 |
| }, |
| "jev": { |
| "accuracy_percent": 85.93, |
| "brier": 0.2055 |
| }, |
| "decider": { |
| "accuracy_percent": 72.59, |
| "brier": 0.3487 |
| }, |
| "jevk5": { |
| "accuracy_percent": 73.33, |
| "brier": 0.319 |
| }, |
| "intern": { |
| "accuracy_percent": 69.63, |
| "brier": 0.3827 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Temporal / numeric", |
| "n": 138, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 55.07, |
| "brier": 0.4695 |
| }, |
| "jev": { |
| "accuracy_percent": 64.49, |
| "brier": 0.4235 |
| }, |
| "decider": { |
| "accuracy_percent": 54.35, |
| "brier": 0.5197 |
| }, |
| "jevk5": { |
| "accuracy_percent": 53.62, |
| "brier": 0.5379 |
| }, |
| "intern": { |
| "accuracy_percent": 59.42, |
| "brier": 0.5422 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Hard", |
| "task": "Total", |
| "n": 4351, |
| "models": { |
| "openjev": { |
| "accuracy_percent": 85.57, |
| "brier": 0.1864 |
| }, |
| "jev": { |
| "accuracy_percent": 86.37, |
| "brier": 0.1733 |
| }, |
| "decider": { |
| "accuracy_percent": 78.88, |
| "brier": 0.2832 |
| }, |
| "jevk5": { |
| "accuracy_percent": 74.7, |
| "brier": 0.333 |
| }, |
| "intern": { |
| "accuracy_percent": 66.95, |
| "brier": 0.4069 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Soft", |
| "task": "Constraint planning", |
| "n": 1, |
| "models": { |
| "openjev": { |
| "accuracy_percent": null, |
| "brier": 0.007 |
| }, |
| "jev": { |
| "accuracy_percent": null, |
| "brier": 0.1412 |
| }, |
| "decider": { |
| "accuracy_percent": null, |
| "brier": 0.1338 |
| }, |
| "jevk5": { |
| "accuracy_percent": null, |
| "brier": 0.0251 |
| }, |
| "intern": { |
| "accuracy_percent": null, |
| "brier": 0.0258 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Soft", |
| "task": "Evidence validation", |
| "n": 22, |
| "models": { |
| "openjev": { |
| "accuracy_percent": null, |
| "brier": 0.1066 |
| }, |
| "jev": { |
| "accuracy_percent": null, |
| "brier": 0.0493 |
| }, |
| "decider": { |
| "accuracy_percent": null, |
| "brier": 0.1517 |
| }, |
| "jevk5": { |
| "accuracy_percent": null, |
| "brier": 0.0773 |
| }, |
| "intern": { |
| "accuracy_percent": null, |
| "brier": 0.0914 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Soft", |
| "task": "Intent routing", |
| "n": 22, |
| "models": { |
| "openjev": { |
| "accuracy_percent": null, |
| "brier": 0.0681 |
| }, |
| "jev": { |
| "accuracy_percent": null, |
| "brier": 0.157 |
| }, |
| "decider": { |
| "accuracy_percent": null, |
| "brier": 0.1366 |
| }, |
| "jevk5": { |
| "accuracy_percent": null, |
| "brier": 0.1304 |
| }, |
| "intern": { |
| "accuracy_percent": null, |
| "brier": 0.0838 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Soft", |
| "task": "Policy exceptions", |
| "n": 18, |
| "models": { |
| "openjev": { |
| "accuracy_percent": null, |
| "brier": 0.0423 |
| }, |
| "jev": { |
| "accuracy_percent": null, |
| "brier": 0.0606 |
| }, |
| "decider": { |
| "accuracy_percent": null, |
| "brier": 0.1128 |
| }, |
| "jevk5": { |
| "accuracy_percent": null, |
| "brier": 0.0916 |
| }, |
| "intern": { |
| "accuracy_percent": null, |
| "brier": 0.0674 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Soft", |
| "task": "Probabilistic reasoning", |
| "n": 505, |
| "models": { |
| "openjev": { |
| "accuracy_percent": null, |
| "brier": 0.0278 |
| }, |
| "jev": { |
| "accuracy_percent": null, |
| "brier": 0.1164 |
| }, |
| "decider": { |
| "accuracy_percent": null, |
| "brier": 0.1387 |
| }, |
| "jevk5": { |
| "accuracy_percent": null, |
| "brier": 0.1092 |
| }, |
| "intern": { |
| "accuracy_percent": null, |
| "brier": 0.1242 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Soft", |
| "task": "Score rubrics", |
| "n": 30, |
| "models": { |
| "openjev": { |
| "accuracy_percent": null, |
| "brier": 0.0864 |
| }, |
| "jev": { |
| "accuracy_percent": null, |
| "brier": 0.0895 |
| }, |
| "decider": { |
| "accuracy_percent": null, |
| "brier": 0.1653 |
| }, |
| "jevk5": { |
| "accuracy_percent": null, |
| "brier": 0.1242 |
| }, |
| "intern": { |
| "accuracy_percent": null, |
| "brier": 0.1746 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Soft", |
| "task": "Temporal / numeric", |
| "n": 12, |
| "models": { |
| "openjev": { |
| "accuracy_percent": null, |
| "brier": 0.2168 |
| }, |
| "jev": { |
| "accuracy_percent": null, |
| "brier": 0.0822 |
| }, |
| "decider": { |
| "accuracy_percent": null, |
| "brier": 0.11 |
| }, |
| "jevk5": { |
| "accuracy_percent": null, |
| "brier": 0.1283 |
| }, |
| "intern": { |
| "accuracy_percent": null, |
| "brier": 0.1747 |
| } |
| } |
| }, |
| { |
| "group": "MMDM \u00b7 Soft", |
| "task": "Total", |
| "n": 610, |
| "models": { |
| "openjev": { |
| "accuracy_percent": null, |
| "brier": 0.0391 |
| }, |
| "jev": { |
| "accuracy_percent": null, |
| "brier": 0.1119 |
| }, |
| "decider": { |
| "accuracy_percent": null, |
| "brier": 0.1391 |
| }, |
| "jevk5": { |
| "accuracy_percent": null, |
| "brier": 0.1092 |
| }, |
| "intern": { |
| "accuracy_percent": null, |
| "brier": 0.1232 |
| } |
| } |
| }, |
| { |
| "group": "JevBench \u00b7 Public", |
| "task": "Original", |
| "models": { |
| "openjev": { |
| "accuracy_percent": 100.0, |
| "brier": 0.031173049329996292 |
| }, |
| "jev": { |
| "accuracy_percent": 98.61111111111111, |
| "brier": 0.028252777777777776 |
| }, |
| "decider": { |
| "accuracy_percent": 98.61111111111111, |
| "brier": 0.057440274164038074 |
| }, |
| "jevk5": { |
| "accuracy_percent": 94.44444444444444, |
| "brier": 0.10753610290390724 |
| }, |
| "intern": { |
| "accuracy_percent": 98.61111111111111, |
| "brier": 0.034003747946694876 |
| } |
| }, |
| "n": 72 |
| }, |
| { |
| "group": "JevBench \u00b7 Public", |
| "task": "Easy", |
| "models": { |
| "openjev": { |
| "accuracy_percent": 100.0, |
| "brier": 1.0634828094991897e-05 |
| }, |
| "jev": { |
| "accuracy_percent": 100.0, |
| "brier": 0.00019583333333333356 |
| }, |
| "decider": { |
| "accuracy_percent": 100.0, |
| "brier": 4.472370693105332e-05 |
| }, |
| "jevk5": { |
| "accuracy_percent": 100.0, |
| "brier": 0.0025092999623402795 |
| }, |
| "intern": { |
| "accuracy_percent": 100.0, |
| "brier": 0.0011188014255948035 |
| } |
| }, |
| "n": 48 |
| }, |
| { |
| "group": "JevBench \u00b7 Public", |
| "task": "Hard", |
| "models": { |
| "openjev": { |
| "accuracy_percent": 75.67567567567568, |
| "brier": 0.3469152429142525 |
| }, |
| "jev": { |
| "accuracy_percent": 72.07207207207207, |
| "brier": 0.35944144144144147 |
| }, |
| "decider": { |
| "accuracy_percent": 64.86486486486487, |
| "brier": 0.4685710696134996 |
| }, |
| "jevk5": { |
| "accuracy_percent": 77.47747747747748, |
| "brier": 0.31178986168730566 |
| }, |
| "intern": { |
| "accuracy_percent": 73.87387387387388, |
| "brier": 0.34391459157831894 |
| } |
| }, |
| "n": 111 |
| }, |
| { |
| "group": "JevBench \u00b7 Public", |
| "task": "Total", |
| "models": { |
| "openjev": { |
| "accuracy_percent": 88.31168831168831, |
| "brier": 0.17641801725969836 |
| }, |
| "jev": { |
| "accuracy_percent": 86.14718614718615, |
| "brier": 0.18156536796536796 |
| }, |
| "decider": { |
| "accuracy_percent": 82.68398268398268, |
| "brier": 0.2430702822720428 |
| }, |
| "jevk5": { |
| "accuracy_percent": 87.44588744588745, |
| "brier": 0.1838602617080718 |
| }, |
| "intern": { |
| "accuracy_percent": 87.01298701298701, |
| "brier": 0.17608870989516875 |
| } |
| }, |
| "n": 231 |
| } |
| ], |
| "mmdm_hard_brier_n": 4294, |
| "mmdm_soft_brier_n": 610, |
| "mmdm_source_revision": "4af5dba7284e7f72f7ffeb2357c391520813d987", |
| "public_suite": "original72 + easy48 + hard111", |
| "notes": "MMDM cells retain dataset-card rounding; Public values retain stored precision. Accuracy is expressed as a percentage." |
| } |
|
|
