{
  "threshold": 0.3292989730834961,
  "review_threshold": 0.052187759429216385,
  "threshold_source": "RSNA validation patients",
  "test_sets": {
    "cq500": {
      "threshold": 0.3292989730834961,
      "scans": 471,
      "bleeds": 196,
      "tp": 179,
      "fn": 17,
      "fp": 25,
      "tn": 250,
      "recall": 0.9133,
      "recall_ci": [
        0.8655,
        0.9451
      ],
      "specificity": 0.9091,
      "specificity_ci": [
        0.8692,
        0.9377
      ],
      "precision": 0.8775,
      "npv": 0.9363,
      "accuracy": 0.9108,
      "balanced_accuracy": 0.9112,
      "f1": 0.895,
      "mcc": 0.8181,
      "auc": 0.9691,
      "auc_ci": [
        0.9554,
        0.982
      ],
      "average_precision": 0.9614,
      "roc": {
        "fpr": [
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0036,
          0.0036,
          0.0073,
          0.0073,
          0.0109,
          0.0109,
          0.0145,
          0.0145,
          0.0218,
          0.0218,
          0.0255,
          0.0255,
          0.0327,
          0.0327,
          0.0364,
          0.0364,
          0.04,
          0.04,
          0.0473,
          0.0473,
          0.0545,
          0.0545,
          0.0582,
          0.0582,
          0.0618,
          0.0618,
          0.0691,
          0.0691,
          0.0764,
          0.0764,
          0.08,
          0.08,
          0.0836,
          0.0836,
          0.0945,
          0.0945,
          0.1018,
          0.1018,
          0.1091,
          0.1091,
          0.1127,
          0.1127,
          0.1273,
          0.1273,
          0.1418,
          0.1418,
          0.1636,
          0.1636,
          0.1745,
          0.1745,
          0.1818,
          0.1818,
          0.2364,
          0.2364,
          0.2509,
          0.2509,
          0.2727,
          0.2727,
          0.3564,
          0.3564,
          0.3891,
          0.3891,
          0.4473,
          0.4473,
          0.5273,
          0.5273,
          1.0
        ],
        "tpr": [
          0.0,
          0.0051,
          0.0306,
          0.0408,
          0.4949,
          0.4949,
          0.5816,
          0.5816,
          0.6582,
          0.6582,
          0.6735,
          0.6735,
          0.7551,
          0.7551,
          0.7653,
          0.7653,
          0.7755,
          0.7755,
          0.7806,
          0.7806,
          0.801,
          0.801,
          0.8061,
          0.8061,
          0.8112,
          0.8112,
          0.8214,
          0.8214,
          0.8316,
          0.8316,
          0.8367,
          0.8367,
          0.852,
          0.852,
          0.8571,
          0.8571,
          0.8776,
          0.8776,
          0.9133,
          0.9133,
          0.9235,
          0.9235,
          0.9286,
          0.9286,
          0.9337,
          0.9337,
          0.9388,
          0.9388,
          0.9439,
          0.9439,
          0.949,
          0.949,
          0.9541,
          0.9541,
          0.9592,
          0.9592,
          0.9643,
          0.9643,
          0.9694,
          0.9694,
          0.9745,
          0.9745,
          0.9796,
          0.9796,
          0.9847,
          0.9847,
          0.9898,
          0.9898,
          0.9949,
          0.9949,
          1.0,
          1.0
        ]
      },
      "by_bleed_type": {
        "epidural": {
          "bleeds": 11,
          "caught": 11,
          "recall": 1.0
        },
        "intraparenchymal": {
          "bleeds": 129,
          "caught": 116,
          "recall": 0.8992
        },
        "intraventricular": {
          "bleeds": 26,
          "caught": 26,
          "recall": 1.0
        },
        "subarachnoid": {
          "bleeds": 57,
          "caught": 55,
          "recall": 0.9649
        },
        "subdural": {
          "bleeds": 48,
          "caught": 47,
          "recall": 0.9792
        }
      },
      "either_tier": {
        "threshold": 0.052187759429216385,
        "scans": 471,
        "bleeds": 196,
        "tp": 194,
        "fn": 2,
        "fp": 110,
        "tn": 165,
        "recall": 0.9898,
        "recall_ci": [
          0.9636,
          0.9972
        ],
        "specificity": 0.6,
        "specificity_ci": [
          0.5411,
          0.6561
        ],
        "precision": 0.6382,
        "npv": 0.988,
        "accuracy": 0.7622,
        "balanced_accuracy": 0.7949,
        "f1": 0.776,
        "mcc": 0.6077,
        "auc": 0.9691,
        "auc_ci": [
          0.9554,
          0.982
        ],
        "average_precision": 0.9614
      }
    },
    "rsna_test": {
      "threshold": 0.3292989730834961,
      "scans": 2132,
      "bleeds": 880,
      "tp": 832,
      "fn": 48,
      "fp": 72,
      "tn": 1180,
      "recall": 0.9455,
      "recall_ci": [
        0.9284,
        0.9586
      ],
      "specificity": 0.9425,
      "specificity_ci": [
        0.9282,
        0.9541
      ],
      "precision": 0.9204,
      "npv": 0.9609,
      "accuracy": 0.9437,
      "balanced_accuracy": 0.944,
      "f1": 0.9327,
      "mcc": 0.8846,
      "auc": 0.9821,
      "auc_ci": [
        0.9763,
        0.9871
      ],
      "average_precision": 0.9795,
      "roc": {
        "fpr": [
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0,
          0.0008,
          0.0008,
          0.0016,
          0.0016,
          0.0016,
          0.0016,
          0.0024,
          0.0024,
          0.0032,
          0.0032,
          0.004,
          0.004,
          0.0048,
          0.0048,
          0.0056,
          0.0056,
          0.0064,
          0.0064,
          0.0072,
          0.0072,
          0.008,
          0.008,
          0.0088,
          0.0088,
          0.0104,
          0.0104,
          0.0112,
          0.0112,
          0.012,
          0.012,
          0.0128,
          0.0128,
          0.0136,
          0.0136,
          0.0144,
          0.0144,
          0.0152,
          0.0152,
          0.016,
          0.016,
          0.0168,
          0.0168,
          0.0176,
          0.0176,
          0.0184,
          0.0184,
          0.0192,
          0.0192,
          0.02,
          0.02,
          0.0216,
          0.0216,
          0.0224,
          0.0224,
          0.0232,
          0.0232,
          0.024,
          0.024,
          0.0264,
          0.0264,
          0.0272,
          0.0272,
          0.0288,
          0.0288,
          0.0296,
          0.0296,
          0.0319,
          0.0319,
          0.0327,
          0.0327,
          0.0335,
          0.0335,
          0.0343,
          0.0343,
          0.0359,
          0.0359,
          0.0367,
          0.0367,
          0.0375,
          0.0375,
          0.0383,
          0.0383,
          0.0407,
          0.0407,
          0.0415,
          0.0415,
          0.0423,
          0.0423,
          0.0431,
          0.0431,
          0.0447,
          0.0447,
          0.0455,
          0.0455,
          0.0463,
          0.0463,
          0.0471,
          0.0471,
          0.0479,
          0.0479,
          0.0527,
          0.0527,
          0.0535,
          0.0535,
          0.0543,
          0.0543,
          0.0551,
          0.0551,
          0.0559,
          0.0559,
          0.0575,
          0.0575,
          0.0583,
          0.0583,
          0.0591,
          0.0591,
          0.0599,
          0.0599,
          0.0607,
          0.0607,
          0.0623,
          0.0623,
          0.0639,
          0.0639,
          0.0687,
          0.0687,
          0.0703,
          0.0703,
          0.0767,
          0.0767,
          0.0775,
          0.0775,
          0.0783,
          0.0783,
          0.0791,
          0.0791,
          0.0823,
          0.0823,
          0.0839,
          0.0839,
          0.0871,
          0.0871,
          0.0966,
          0.0966,
          0.0974,
          0.0974,
          0.107,
          0.107,
          0.1078,
          0.1078,
          0.1134,
          0.1134,
          0.1182,
          0.1182,
          0.135,
          0.135,
          0.1438,
          0.1438,
          0.1581,
          0.1581,
          0.1845,
          0.1845,
          0.25,
          0.25,
          0.3011,
          0.3011,
          0.3099,
          0.3099,
          0.3251,
          0.3251,
          0.3522,
          0.3522,
          0.4417,
          0.4417,
          0.4433,
          0.4433,
          0.4904,
          0.4904,
          0.4928,
          0.4928,
          0.5064,
          0.5064,
          0.5216,
          0.5216,
          0.5304,
          0.5304,
          0.5695,
          0.5695,
          0.603,
          0.603,
          0.6853,
          0.6853,
          0.7636,
          0.7636,
          0.8227,
          0.8227,
          0.9433,
          0.9449,
          1.0
        ],
        "tpr": [
          0.0,
          0.0011,
          0.05,
          0.0523,
          0.108,
          0.1102,
          0.142,
          0.1443,
          0.1614,
          0.1636,
          0.4205,
          0.4205,
          0.4841,
          0.4841,
          0.5136,
          0.5159,
          0.5693,
          0.5693,
          0.642,
          0.642,
          0.6625,
          0.6625,
          0.7193,
          0.7193,
          0.7273,
          0.7273,
          0.7511,
          0.7511,
          0.7568,
          0.7568,
          0.7659,
          0.7659,
          0.7818,
          0.7818,
          0.7875,
          0.7875,
          0.8023,
          0.8023,
          0.8034,
          0.8034,
          0.8068,
          0.8068,
          0.8102,
          0.8102,
          0.817,
          0.817,
          0.8193,
          0.8193,
          0.8261,
          0.8261,
          0.8386,
          0.8386,
          0.8409,
          0.8409,
          0.8545,
          0.8545,
          0.858,
          0.858,
          0.8602,
          0.8602,
          0.8614,
          0.8614,
          0.8659,
          0.8659,
          0.867,
          0.867,
          0.8693,
          0.8693,
          0.8716,
          0.8716,
          0.8761,
          0.8761,
          0.8841,
          0.8841,
          0.8875,
          0.8875,
          0.8909,
          0.8909,
          0.8966,
          0.8966,
          0.8989,
          0.8989,
          0.9,
          0.9,
          0.9011,
          0.9011,
          0.9023,
          0.9023,
          0.9091,
          0.9091,
          0.9114,
          0.9114,
          0.9159,
          0.9159,
          0.9193,
          0.9193,
          0.9216,
          0.9216,
          0.9239,
          0.9239,
          0.9261,
          0.9261,
          0.9284,
          0.9284,
          0.9295,
          0.9295,
          0.9318,
          0.9318,
          0.933,
          0.933,
          0.9375,
          0.9375,
          0.9386,
          0.9386,
          0.9398,
          0.9398,
          0.942,
          0.942,
          0.9432,
          0.9432,
          0.9443,
          0.9443,
          0.9466,
          0.9466,
          0.9489,
          0.9489,
          0.95,
          0.95,
          0.9511,
          0.9511,
          0.9523,
          0.9523,
          0.9534,
          0.9534,
          0.9545,
          0.9545,
          0.9568,
          0.9568,
          0.958,
          0.958,
          0.9602,
          0.9602,
          0.9614,
          0.9614,
          0.9625,
          0.9625,
          0.9648,
          0.9648,
          0.9659,
          0.9659,
          0.967,
          0.967,
          0.9693,
          0.9693,
          0.9705,
          0.9705,
          0.9716,
          0.9716,
          0.9727,
          0.9727,
          0.9739,
          0.9739,
          0.975,
          0.975,
          0.9761,
          0.9761,
          0.9773,
          0.9773,
          0.9784,
          0.9784,
          0.9795,
          0.9795,
          0.9807,
          0.9807,
          0.9818,
          0.9818,
          0.983,
          0.983,
          0.9841,
          0.9841,
          0.9852,
          0.9852,
          0.9864,
          0.9864,
          0.9875,
          0.9875,
          0.9886,
          0.9886,
          0.9898,
          0.9898,
          0.9909,
          0.9909,
          0.992,
          0.992,
          0.9932,
          0.9932,
          0.9943,
          0.9943,
          0.9955,
          0.9955,
          0.9966,
          0.9966,
          0.9977,
          0.9977,
          0.9989,
          0.9989,
          1.0,
          1.0,
          1.0,
          1.0
        ]
      },
      "by_bleed_type": {
        "epidural": {
          "bleeds": 26,
          "caught": 26,
          "recall": 1.0
        },
        "intraparenchymal": {
          "bleeds": 552,
          "caught": 528,
          "recall": 0.9565
        },
        "intraventricular": {
          "bleeds": 369,
          "caught": 366,
          "recall": 0.9919
        },
        "subarachnoid": {
          "bleeds": 400,
          "caught": 388,
          "recall": 0.97
        },
        "subdural": {
          "bleeds": 407,
          "caught": 391,
          "recall": 0.9607
        }
      },
      "either_tier": {
        "threshold": 0.052187759429216385,
        "scans": 2132,
        "bleeds": 880,
        "tp": 863,
        "fn": 17,
        "fp": 306,
        "tn": 946,
        "recall": 0.9807,
        "recall_ci": [
          0.9693,
          0.9879
        ],
        "specificity": 0.7556,
        "specificity_ci": [
          0.731,
          0.7786
        ],
        "precision": 0.7382,
        "npv": 0.9823,
        "accuracy": 0.8485,
        "balanced_accuracy": 0.8681,
        "f1": 0.8424,
        "mcc": 0.7284,
        "auc": 0.9821,
        "auc_ci": [
          0.9763,
          0.9871
        ],
        "average_precision": 0.9795
      }
    },
    "kaggle": {
      "threshold": 0.3292989730834961,
      "scans": 43,
      "bleeds": 17,
      "tp": 17,
      "fn": 0,
      "fp": 16,
      "tn": 10,
      "recall": 1.0,
      "recall_ci": [
        0.8157,
        1.0
      ],
      "specificity": 0.3846,
      "specificity_ci": [
        0.2243,
        0.5747
      ],
      "precision": 0.5152,
      "npv": 1.0,
      "accuracy": 0.6279,
      "balanced_accuracy": 0.6923,
      "f1": 0.68,
      "mcc": 0.4451,
      "auc": 0.9344,
      "auc_ci": [
        0.8565,
        0.9861
      ],
      "average_precision": 0.9175,
      "roc": {
        "fpr": [
          0.0,
          0.0,
          0.0,
          0.0385,
          0.0385,
          0.1538,
          0.1538,
          0.1923,
          0.1923,
          0.2308,
          0.2308,
          0.3462,
          0.3462,
          1.0
        ],
        "tpr": [
          0.0,
          0.0588,
          0.6471,
          0.6471,
          0.7059,
          0.7059,
          0.8235,
          0.8235,
          0.8824,
          0.8824,
          0.9412,
          0.9412,
          1.0,
          1.0
        ]
      },
      "by_bleed_type": {},
      "either_tier": {
        "threshold": 0.052187759429216385,
        "scans": 43,
        "bleeds": 17,
        "tp": 17,
        "fn": 0,
        "fp": 26,
        "tn": 0,
        "recall": 1.0,
        "recall_ci": [
          0.8157,
          1.0
        ],
        "specificity": 0.0,
        "specificity_ci": [
          0.0,
          0.1287
        ],
        "precision": 0.3953,
        "npv": NaN,
        "accuracy": 0.3953,
        "balanced_accuracy": 0.5,
        "f1": 0.5667,
        "mcc": NaN,
        "auc": 0.9344,
        "auc_ci": [
          0.8565,
          0.9861
        ],
        "average_precision": 0.9175
      }
    }
  },
  "cpu_seconds_per_scan": 1.94
}