{
  "count": 3564,
  "accuracy": 0.995230078563412,
  "nll": 0.02104884277840532,
  "ece": 0.0031197859783365185,
  "per_decision": {
    "role": {
      "count": 1188,
      "accuracy": 0.9949494949494949,
      "nll": 0.023152108122646918,
      "ece": 0.0036363091934409454
    },
    "action": {
      "count": 1188,
      "accuracy": 0.9932659932659933,
      "nll": 0.026944733532553087,
      "ece": 0.004396045850182216
    },
    "risk": {
      "count": 1188,
      "accuracy": 0.9974747474747475,
      "nll": 0.013049686680015953,
      "ece": 0.002284457286198937
    }
  },
  "confidence_policy": [
    {
      "threshold": 0.5,
      "coverage": 0.9988776655443322,
      "accuracy": 0.9960674157303371,
      "count": 3560
    },
    {
      "threshold": 0.6,
      "coverage": 0.9983164983164983,
      "accuracy": 0.9963462619449128,
      "count": 3558
    },
    {
      "threshold": 0.7,
      "coverage": 0.9977553310886644,
      "accuracy": 0.9966254218222722,
      "count": 3556
    },
    {
      "threshold": 0.75,
      "coverage": 0.9971941638608305,
      "accuracy": 0.9971862689926843,
      "count": 3554
    },
    {
      "threshold": 0.8,
      "coverage": 0.9966329966329966,
      "accuracy": 0.9974662162162162,
      "count": 3552
    },
    {
      "threshold": 0.85,
      "coverage": 0.9957912457912458,
      "accuracy": 0.9980276134122288,
      "count": 3549
    },
    {
      "threshold": 0.9,
      "coverage": 0.994668911335578,
      "accuracy": 0.9983074753173484,
      "count": 3545
    },
    {
      "threshold": 0.95,
      "coverage": 0.9943883277216611,
      "accuracy": 0.9983069977426636,
      "count": 3544
    }
  ],
  "task_count": 1188,
  "task_exact_match": 0.9865319865319865,
  "task_head_accuracy": {
    "role": 0.9949494949494949,
    "action": 0.9932659932659933,
    "risk": 0.9974747474747475
  }
}