| { |
| "count": 3564, |
| "accuracy": 0.995230078563412, |
| "nll": 0.02104884277840532, |
| "ece": 0.0031197859783365185, |
| "per_decision": { |
| "role": { |
| "count": 1188, |
| "accuracy": 0.9949494949494949, |
| "nll": 0.023152108122646918, |
| "ece": 0.0036363091934409454 |
| }, |
| "action": { |
| "count": 1188, |
| "accuracy": 0.9932659932659933, |
| "nll": 0.026944733532553087, |
| "ece": 0.004396045850182216 |
| }, |
| "risk": { |
| "count": 1188, |
| "accuracy": 0.9974747474747475, |
| "nll": 0.013049686680015953, |
| "ece": 0.002284457286198937 |
| } |
| }, |
| "confidence_policy": [ |
| { |
| "threshold": 0.5, |
| "coverage": 0.9988776655443322, |
| "accuracy": 0.9960674157303371, |
| "count": 3560 |
| }, |
| { |
| "threshold": 0.6, |
| "coverage": 0.9983164983164983, |
| "accuracy": 0.9963462619449128, |
| "count": 3558 |
| }, |
| { |
| "threshold": 0.7, |
| "coverage": 0.9977553310886644, |
| "accuracy": 0.9966254218222722, |
| "count": 3556 |
| }, |
| { |
| "threshold": 0.75, |
| "coverage": 0.9971941638608305, |
| "accuracy": 0.9971862689926843, |
| "count": 3554 |
| }, |
| { |
| "threshold": 0.8, |
| "coverage": 0.9966329966329966, |
| "accuracy": 0.9974662162162162, |
| "count": 3552 |
| }, |
| { |
| "threshold": 0.85, |
| "coverage": 0.9957912457912458, |
| "accuracy": 0.9980276134122288, |
| "count": 3549 |
| }, |
| { |
| "threshold": 0.9, |
| "coverage": 0.994668911335578, |
| "accuracy": 0.9983074753173484, |
| "count": 3545 |
| }, |
| { |
| "threshold": 0.95, |
| "coverage": 0.9943883277216611, |
| "accuracy": 0.9983069977426636, |
| "count": 3544 |
| } |
| ], |
| "task_count": 1188, |
| "task_exact_match": 0.9865319865319865, |
| "task_head_accuracy": { |
| "role": 0.9949494949494949, |
| "action": 0.9932659932659933, |
| "risk": 0.9974747474747475 |
| } |
| } |
|
|
