{
  "task": "regulated_advice",
  "threshold": 0.88,
  "per_language": {
    "az": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 1.0,
      "n": 52
    },
    "bg": {
      "support": 22,
      "precision": 1.0,
      "recall": 0.9545,
      "f1": 0.9767,
      "fpr": 0.0,
      "accuracy": 0.9615,
      "n": 52
    },
    "cs": {
      "support": 22,
      "precision": 1.0,
      "recall": 0.9545,
      "f1": 0.9767,
      "fpr": 0.0,
      "accuracy": 0.9615,
      "n": 52
    },
    "da": {
      "support": 22,
      "precision": 1.0,
      "recall": 0.9545,
      "f1": 0.9767,
      "fpr": 0.0,
      "accuracy": 0.9423,
      "n": 52
    },
    "de": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9808,
      "n": 52
    },
    "el": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9808,
      "n": 52
    },
    "en": {
      "support": 22,
      "precision": 1.0,
      "recall": 0.9545,
      "f1": 0.9767,
      "fpr": 0.0,
      "accuracy": 0.9423,
      "n": 52
    },
    "es": {
      "support": 22,
      "precision": 1.0,
      "recall": 0.9545,
      "f1": 0.9767,
      "fpr": 0.0,
      "accuracy": 0.9808,
      "n": 52
    },
    "et": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9615,
      "n": 52
    },
    "fi": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9423,
      "n": 52
    },
    "fr": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9231,
      "n": 52
    },
    "ga": {
      "support": 22,
      "precision": 0.913,
      "recall": 0.9545,
      "f1": 0.9333,
      "fpr": 0.0667,
      "accuracy": 0.8269,
      "n": 52
    },
    "hr": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9231,
      "n": 52
    },
    "hu": {
      "support": 22,
      "precision": 1.0,
      "recall": 0.9545,
      "f1": 0.9767,
      "fpr": 0.0,
      "accuracy": 0.9038,
      "n": 52
    },
    "it": {
      "support": 22,
      "precision": 1.0,
      "recall": 0.9545,
      "f1": 0.9767,
      "fpr": 0.0,
      "accuracy": 0.9231,
      "n": 52
    },
    "lt": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9808,
      "n": 52
    },
    "lv": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9808,
      "n": 52
    },
    "mt": {
      "support": 22,
      "precision": 1.0,
      "recall": 0.8182,
      "f1": 0.9,
      "fpr": 0.0,
      "accuracy": 0.8269,
      "n": 52
    },
    "nl": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 1.0,
      "n": 52
    },
    "pl": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9038,
      "n": 52
    },
    "pt": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9808,
      "n": 52
    },
    "ro": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9808,
      "n": 52
    },
    "sk": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9808,
      "n": 52
    },
    "sl": {
      "support": 22,
      "precision": 1.0,
      "recall": 0.9545,
      "f1": 0.9767,
      "fpr": 0.0,
      "accuracy": 0.9231,
      "n": 52
    },
    "sv": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9808,
      "n": 52
    },
    "tr": {
      "support": 22,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9615,
      "n": 52
    }
  },
  "per_register": {
    "definition": {
      "support": 0,
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "fpr": 0.0,
      "accuracy": 1.0,
      "n": 123
    },
    "directive": {
      "support": 133,
      "precision": 1.0,
      "recall": 0.9699,
      "f1": 0.9847,
      "fpr": 0.0,
      "accuracy": 0.8947,
      "n": 133
    },
    "general_risk": {
      "support": 0,
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "fpr": 0.0,
      "accuracy": 1.0,
      "n": 111
    },
    "historical_fact": {
      "support": 0,
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "fpr": 0.0,
      "accuracy": 1.0,
      "n": 107
    },
    "hypothetical": {
      "support": 0,
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "fpr": 0.0,
      "accuracy": 1.0,
      "n": 98
    },
    "mundane_account_access": {
      "support": 0,
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "fpr": 0.0179,
      "accuracy": 0.9821,
      "n": 56
    },
    "mundane_informational": {
      "support": 0,
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "fpr": 0.0,
      "accuracy": 1.0,
      "n": 48
    },
    "mundane_operational": {
      "support": 0,
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "fpr": 0.0,
      "accuracy": 1.0,
      "n": 50
    },
    "mundane_transactional": {
      "support": 0,
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "fpr": 0.0,
      "accuracy": 1.0,
      "n": 54
    },
    "personal_recommendation": {
      "support": 328,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0,
      "fpr": 0.0,
      "accuracy": 0.9695,
      "n": 328
    },
    "process_description": {
      "support": 0,
      "precision": 0.0,
      "recall": 0.0,
      "f1": 0.0,
      "fpr": 0.0075,
      "accuracy": 0.9925,
      "n": 133
    },
    "suitability_claim": {
      "support": 111,
      "precision": 1.0,
      "recall": 0.9189,
      "f1": 0.9577,
      "fpr": 0.0,
      "accuracy": 0.6036,
      "n": 111
    }
  },
  "per_label": {
    "financial_advice": {
      "support": 260,
      "precision": 0.8745,
      "recall": 0.9115,
      "f1": 0.8927,
      "fpr": 0.0311,
      "accuracy": 0.0,
      "n": 0
    },
    "legal_advice": {
      "support": 156,
      "precision": 0.8933,
      "recall": 0.859,
      "f1": 0.8758,
      "fpr": 0.0134,
      "accuracy": 0.0,
      "n": 0
    },
    "medical_advice": {
      "support": 156,
      "precision": 0.9504,
      "recall": 0.859,
      "f1": 0.9024,
      "fpr": 0.0059,
      "accuracy": 0.0,
      "n": 0
    }
  },
  "worst_languages": [
    [
      "mt",
      0.9
    ],
    [
      "ga",
      0.9333333333333332
    ],
    [
      "bg",
      0.9767441860465117
    ],
    [
      "cs",
      0.9767441860465117
    ],
    [
      "da",
      0.9767441860465117
    ]
  ],
  "calibration": {
    "threshold": 0.88,
    "objective": "macro_f1",
    "value_on_val": 0.8921,
    "default_threshold": 0.5,
    "default_value_on_val": 0.8606,
    "improvement": 0.0315,
    "chosen_on": "validation split, never test",
    "curve": [
      {
        "threshold": 0.02,
        "macro_f1": 0.4851,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0577
      },
      {
        "threshold": 0.03,
        "macro_f1": 0.5211,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0385
      },
      {
        "threshold": 0.04,
        "macro_f1": 0.5858,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0295
      },
      {
        "threshold": 0.05,
        "macro_f1": 0.6416,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0269
      },
      {
        "threshold": 0.06,
        "macro_f1": 0.6811,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0244
      },
      {
        "threshold": 0.07,
        "macro_f1": 0.7085,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.08,
        "macro_f1": 0.7321,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.09,
        "macro_f1": 0.7437,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.1,
        "macro_f1": 0.7512,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.11,
        "macro_f1": 0.764,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.12,
        "macro_f1": 0.7664,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.13,
        "macro_f1": 0.7717,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.14,
        "macro_f1": 0.774,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.15,
        "macro_f1": 0.7775,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.16,
        "macro_f1": 0.7822,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.17,
        "macro_f1": 0.7838,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.18,
        "macro_f1": 0.7844,
        "any_label_recall": 1.0,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.19,
        "macro_f1": 0.7867,
        "any_label_recall": 0.9983,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.2,
        "macro_f1": 0.7905,
        "any_label_recall": 0.9983,
        "any_label_fpr": 0.0192
      },
      {
        "threshold": 0.21,
        "macro_f1": 0.795,
        "any_label_recall": 0.9983,
        "any_label_fpr": 0.0167
      },
      {
        "threshold": 0.22,
        "macro_f1": 0.7968,
        "any_label_recall": 0.9983,
        "any_label_fpr": 0.0167
      },
      {
        "threshold": 0.23,
        "macro_f1": 0.7978,
        "any_label_recall": 0.9983,
        "any_label_fpr": 0.0167
      },
      {
        "threshold": 0.24,
        "macro_f1": 0.8015,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0167
      },
      {
        "threshold": 0.25,
        "macro_f1": 0.8038,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0167
      },
      {
        "threshold": 0.26,
        "macro_f1": 0.8078,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0167
      },
      {
        "threshold": 0.27,
        "macro_f1": 0.8091,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0167
      },
      {
        "threshold": 0.28,
        "macro_f1": 0.8135,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0154
      },
      {
        "threshold": 0.29,
        "macro_f1": 0.8156,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0154
      },
      {
        "threshold": 0.3,
        "macro_f1": 0.816,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0154
      },
      {
        "threshold": 0.31,
        "macro_f1": 0.8186,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0154
      },
      {
        "threshold": 0.32,
        "macro_f1": 0.8246,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0154
      },
      {
        "threshold": 0.33,
        "macro_f1": 0.8259,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0154
      },
      {
        "threshold": 0.34,
        "macro_f1": 0.8273,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0154
      },
      {
        "threshold": 0.35,
        "macro_f1": 0.8274,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0141
      },
      {
        "threshold": 0.36,
        "macro_f1": 0.8293,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0141
      },
      {
        "threshold": 0.37,
        "macro_f1": 0.8334,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0141
      },
      {
        "threshold": 0.38,
        "macro_f1": 0.8356,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0141
      },
      {
        "threshold": 0.39,
        "macro_f1": 0.837,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0141
      },
      {
        "threshold": 0.4,
        "macro_f1": 0.8385,
        "any_label_recall": 0.9965,
        "any_label_fpr": 0.0141
      },
      {
        "threshold": 0.41,
        "macro_f1": 0.8433,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0128
      },
      {
        "threshold": 0.42,
        "macro_f1": 0.8429,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0128
      },
      {
        "threshold": 0.43,
        "macro_f1": 0.845,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0128
      },
      {
        "threshold": 0.44,
        "macro_f1": 0.8458,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0128
      },
      {
        "threshold": 0.45,
        "macro_f1": 0.8458,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0128
      },
      {
        "threshold": 0.46,
        "macro_f1": 0.8474,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0128
      },
      {
        "threshold": 0.47,
        "macro_f1": 0.8512,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0115
      },
      {
        "threshold": 0.48,
        "macro_f1": 0.8528,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0115
      },
      {
        "threshold": 0.49,
        "macro_f1": 0.8558,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0115
      },
      {
        "threshold": 0.5,
        "macro_f1": 0.8606,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0115
      },
      {
        "threshold": 0.51,
        "macro_f1": 0.8604,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0115
      },
      {
        "threshold": 0.52,
        "macro_f1": 0.8627,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0115
      },
      {
        "threshold": 0.53,
        "macro_f1": 0.8641,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0103
      },
      {
        "threshold": 0.54,
        "macro_f1": 0.8641,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0103
      },
      {
        "threshold": 0.55,
        "macro_f1": 0.8635,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0103
      },
      {
        "threshold": 0.56,
        "macro_f1": 0.8643,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0103
      },
      {
        "threshold": 0.57,
        "macro_f1": 0.8661,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0103
      },
      {
        "threshold": 0.58,
        "macro_f1": 0.8661,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0103
      },
      {
        "threshold": 0.59,
        "macro_f1": 0.8683,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0103
      },
      {
        "threshold": 0.6,
        "macro_f1": 0.8683,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0103
      },
      {
        "threshold": 0.61,
        "macro_f1": 0.8699,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0103
      },
      {
        "threshold": 0.62,
        "macro_f1": 0.8707,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0103
      },
      {
        "threshold": 0.63,
        "macro_f1": 0.8686,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0103
      },
      {
        "threshold": 0.64,
        "macro_f1": 0.8717,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.009
      },
      {
        "threshold": 0.65,
        "macro_f1": 0.8715,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.66,
        "macro_f1": 0.8748,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.67,
        "macro_f1": 0.8782,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.68,
        "macro_f1": 0.8782,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.69,
        "macro_f1": 0.877,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.7,
        "macro_f1": 0.8779,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.71,
        "macro_f1": 0.8779,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.72,
        "macro_f1": 0.8833,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.73,
        "macro_f1": 0.8839,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.74,
        "macro_f1": 0.8882,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.75,
        "macro_f1": 0.8899,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.76,
        "macro_f1": 0.8896,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.77,
        "macro_f1": 0.8888,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.78,
        "macro_f1": 0.8876,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.79,
        "macro_f1": 0.8885,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.8,
        "macro_f1": 0.885,
        "any_label_recall": 0.9948,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.81,
        "macro_f1": 0.885,
        "any_label_recall": 0.993,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.82,
        "macro_f1": 0.8847,
        "any_label_recall": 0.993,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.83,
        "macro_f1": 0.8837,
        "any_label_recall": 0.9913,
        "any_label_fpr": 0.0077
      },
      {
        "threshold": 0.84,
        "macro_f1": 0.8893,
        "any_label_recall": 0.9913,
        "any_label_fpr": 0.0064
      },
      {
        "threshold": 0.85,
        "macro_f1": 0.8886,
        "any_label_recall": 0.9913,
        "any_label_fpr": 0.0064
      },
      {
        "threshold": 0.86,
        "macro_f1": 0.89,
        "any_label_recall": 0.9843,
        "any_label_fpr": 0.0064
      },
      {
        "threshold": 0.87,
        "macro_f1": 0.8912,
        "any_label_recall": 0.9825,
        "any_label_fpr": 0.0064
      },
      {
        "threshold": 0.88,
        "macro_f1": 0.8921,
        "any_label_recall": 0.9738,
        "any_label_fpr": 0.0064
      },
      {
        "threshold": 0.89,
        "macro_f1": 0.89,
        "any_label_recall": 0.9685,
        "any_label_fpr": 0.0064
      },
      {
        "threshold": 0.9,
        "macro_f1": 0.8904,
        "any_label_recall": 0.965,
        "any_label_fpr": 0.0064
      },
      {
        "threshold": 0.91,
        "macro_f1": 0.8902,
        "any_label_recall": 0.9458,
        "any_label_fpr": 0.0064
      },
      {
        "threshold": 0.92,
        "macro_f1": 0.8882,
        "any_label_recall": 0.9266,
        "any_label_fpr": 0.0064
      },
      {
        "threshold": 0.93,
        "macro_f1": 0.8887,
        "any_label_recall": 0.9003,
        "any_label_fpr": 0.0064
      },
      {
        "threshold": 0.94,
        "macro_f1": 0.8878,
        "any_label_recall": 0.8881,
        "any_label_fpr": 0.0064
      },
      {
        "threshold": 0.95,
        "macro_f1": 0.8808,
        "any_label_recall": 0.8601,
        "any_label_fpr": 0.0064
      }
    ],
    "notes": []
  },
  "pair_accuracy": 0.8794
}