eval.json

{
  "n": 1217,
  "n_user": 1066,
  "kind_acc": 0.9975,
  "durable_f1": 0.9966,
  "category_acc_on_durable": 1.0,
  "ece_durable": 0.0171,
  "temperature": 1.8,
  "encoder_user_durable_f1": 0.9949,
  "bayes_same_data_user_durable_f1": 0.6873,
  "bayes_mcode_seed_user_durable_f1": 0.7338,
  "beats_same_data_bayes": true,
  "beats_mcode_bayes": true,
  "gold": {
    "n": 65,
    "kind_acc": 0.9692,
    "preference_acc": 0.9846,
    "category_acc_on_durable": 0.9706,
    "no_diff_acc": 1.0
  },
  "ood": {
    "n": 78,
    "kind_acc": 0.9872,
    "preference_acc": 0.9872,
    "category_acc_on_durable": 1.0,
    "no_diff_acc": null,
    "pass": true,
    "failures": [
      {
        "user": "keep the old name",
        "expected_kind": "correction",
        "expected_preference": false,
        "got": [
          {
            "kind": "durable",
            "category": "language",
            "preference": true,
            "text": "keep the old name"
          }
        ]
      }
    ]
  },
  "encoder_trained": true
}