{
  "artifacts": "/home/simon/Projects/mverify/artifacts",
  "eval": {
    "n": 500,
    "accuracy": 0.86,
    "pass": {
      "precision": 0.4848,
      "recall": 0.9697,
      "f1": 0.6465,
      "tp": 64,
      "fp": 68,
      "fn": 2
    },
    "fail": {
      "precision": 0.9946,
      "recall": 0.8433,
      "f1": 0.9127,
      "tp": 366,
      "fp": 2,
      "fn": 68
    },
    "fpr_good": 0.0303,
    "blatant_fail_recall": 0.9563,
    "blatant_n": 183,
    "per_category": {
      "pass": {
        "n": 66,
        "accuracy": 0.9697,
        "fail_recall": null
      },
      "off_topic": {
        "n": 62,
        "accuracy": 1.0,
        "fail_recall": 1.0
      },
      "empty": {
        "n": 62,
        "accuracy": 1.0,
        "fail_recall": 1.0
      },
      "truncated": {
        "n": 63,
        "accuracy": 0.9524,
        "fail_recall": 0.9524
      },
      "ignored_constraint": {
        "n": 59,
        "accuracy": 0.8644,
        "fail_recall": 0.8644
      },
      "wrong_format": {
        "n": 62,
        "accuracy": 0.8387,
        "fail_recall": 0.8387
      },
      "different_question": {
        "n": 63,
        "accuracy": 0.254,
        "fail_recall": 0.254
      },
      "confident_refusal": {
        "n": 63,
        "accuracy": 1.0,
        "fail_recall": 1.0
      }
    },
    "latency_ms": {
      "mean": 0.127,
      "p95": 0.217,
      "max": 2.349
    },
    "targets": {
      "fpr_good_lt": 0.05,
      "blatant_fail_recall_gt": 0.95,
      "p95_ms_lt": 200.0
    },
    "pass_targets": true
  },
  "eval_bayes": {
    "n": 500,
    "accuracy": 0.794,
    "pass": {
      "precision": 0.2319,
      "recall": 0.2424,
      "f1": 0.237,
      "tp": 16,
      "fp": 53,
      "fn": 50
    },
    "fail": {
      "precision": 0.884,
      "recall": 0.8779,
      "f1": 0.8809,
      "tp": 381,
      "fp": 50,
      "fn": 53
    },
    "fpr_good": 0.7576,
    "blatant_fail_recall": 0.918,
    "blatant_n": 183,
    "per_category": {
      "pass": {
        "n": 66,
        "accuracy": 0.2424,
        "fail_recall": null
      },
      "off_topic": {
        "n": 62,
        "accuracy": 1.0,
        "fail_recall": 1.0
      },
      "empty": {
        "n": 62,
        "accuracy": 0.9355,
        "fail_recall": 0.9355
      },
      "truncated": {
        "n": 63,
        "accuracy": 0.8254,
        "fail_recall": 0.8254
      },
      "ignored_constraint": {
        "n": 59,
        "accuracy": 0.8136,
        "fail_recall": 0.8136
      },
      "wrong_format": {
        "n": 62,
        "accuracy": 0.8065,
        "fail_recall": 0.8065
      },
      "different_question": {
        "n": 63,
        "accuracy": 0.7619,
        "fail_recall": 0.7619
      },
      "confident_refusal": {
        "n": 63,
        "accuracy": 1.0,
        "fail_recall": 1.0
      }
    },
    "latency_ms": {
      "mean": 0.0,
      "p95": 0.0,
      "max": 0.0
    },
    "targets": {
      "fpr_good_lt": 0.05,
      "blatant_fail_recall_gt": 0.95,
      "p95_ms_lt": 200.0
    },
    "pass_targets": false
  },
  "test": {
    "n": 151,
    "accuracy": 0.8874,
    "pass": {
      "precision": 0.5294,
      "recall": 0.9474,
      "f1": 0.6792,
      "tp": 18,
      "fp": 16,
      "fn": 1
    },
    "fail": {
      "precision": 0.9915,
      "recall": 0.8788,
      "f1": 0.9317,
      "tp": 116,
      "fp": 1,
      "fn": 16
    },
    "fpr_good": 0.0526,
    "blatant_fail_recall": 0.9643,
    "blatant_n": 56,
    "per_category": {
      "pass": {
        "n": 19,
        "accuracy": 0.9474,
        "fail_recall": null
      },
      "off_topic": {
        "n": 19,
        "accuracy": 0.9474,
        "fail_recall": 0.9474
      },
      "empty": {
        "n": 19,
        "accuracy": 1.0,
        "fail_recall": 1.0
      },
      "truncated": {
        "n": 19,
        "accuracy": 0.9474,
        "fail_recall": 0.9474
      },
      "ignored_constraint": {
        "n": 18,
        "accuracy": 0.9444,
        "fail_recall": 0.9444
      },
      "wrong_format": {
        "n": 19,
        "accuracy": 0.8421,
        "fail_recall": 0.8421
      },
      "different_question": {
        "n": 19,
        "accuracy": 0.4737,
        "fail_recall": 0.4737
      },
      "confident_refusal": {
        "n": 19,
        "accuracy": 1.0,
        "fail_recall": 1.0
      }
    },
    "latency_ms": {
      "mean": 0.087,
      "p95": 0.195,
      "max": 0.235
    },
    "targets": {
      "fpr_good_lt": 0.05,
      "blatant_fail_recall_gt": 0.95,
      "p95_ms_lt": 200.0
    },
    "pass_targets": false
  },
  "gold": {
    "n": 48,
    "accuracy": 0.8333,
    "pass": {
      "precision": 0.6667,
      "recall": 0.7692,
      "f1": 0.7143,
      "tp": 10,
      "fp": 5,
      "fn": 3
    },
    "fail": {
      "precision": 0.9091,
      "recall": 0.8571,
      "f1": 0.8824,
      "tp": 30,
      "fp": 3,
      "fn": 5
    },
    "fpr_good": 0.2308,
    "blatant_fail_recall": 0.875,
    "blatant_n": 16,
    "per_category": {
      "pass": {
        "n": 13,
        "accuracy": 0.7692,
        "fail_recall": null
      },
      "off_topic": {
        "n": 5,
        "accuracy": 0.8,
        "fail_recall": 0.8
      },
      "empty": {
        "n": 4,
        "accuracy": 1.0,
        "fail_recall": 1.0
      },
      "truncated": {
        "n": 4,
        "accuracy": 1.0,
        "fail_recall": 1.0
      },
      "ignored_constraint": {
        "n": 7,
        "accuracy": 0.8571,
        "fail_recall": 0.8571
      },
      "wrong_format": {
        "n": 5,
        "accuracy": 0.8,
        "fail_recall": 0.8
      },
      "different_question": {
        "n": 4,
        "accuracy": 0.5,
        "fail_recall": 0.5
      },
      "confident_refusal": {
        "n": 6,
        "accuracy": 1.0,
        "fail_recall": 1.0
      }
    },
    "latency_ms": {
      "mean": 0.083,
      "p95": 0.155,
      "max": 0.196
    },
    "targets": {
      "fpr_good_lt": 0.05,
      "blatant_fail_recall_gt": 0.95,
      "p95_ms_lt": 200.0
    },
    "pass_targets": false
  }
}