{
  "model": "V2C1",
  "threshold": 0.4684000015258789,
  "artifact": {
    "bytes": 40200,
    "sha256": "e2e4e4b0e49d50f0d3c0b2129240564ec84b93b4c252651ef10b82560b9d72ae",
    "size_kb": 39.2578125
  },
  "test_binary_metrics": {
    "threshold": 0.4684000015258789,
    "tp": 189,
    "fn": 9,
    "tn": 1407,
    "fp": 16,
    "n": 1621,
    "n_drone": 198,
    "n_nondrone": 1423,
    "recall_tpr": 0.9545454545454546,
    "specificity": 0.988756148981026,
    "fpr": 0.011243851018973999,
    "precision": 0.9219512195121952,
    "f1": 0.9379652605459058,
    "npv": 0.9936440677966102
  },
  "test_roc_auc": 0.9970825613833345,
  "test_pr_auc": 0.983962221893969,
  "evidence_units": {
    "segments": 1621,
    "recordings": 296,
    "sessions": null,
    "session_note": "corpus carries no session metadata; recording is the independence unit"
  },
  "subgroup_far": [
    {
      "subgroup": "chainsaw",
      "segments": 35,
      "recordings": 3,
      "fp": 7,
      "far": 0.2
    },
    {
      "subgroup": "toilet_flush",
      "segments": 25,
      "recordings": 5,
      "fp": 4,
      "far": 0.16
    },
    {
      "subgroup": "glass_breaking",
      "segments": 14,
      "recordings": 1,
      "fp": 1,
      "far": 0.07142857142857142
    },
    {
      "subgroup": "pig",
      "segments": 22,
      "recordings": 4,
      "fp": 1,
      "far": 0.045454545454545456
    },
    {
      "subgroup": "engine",
      "segments": 23,
      "recordings": 4,
      "fp": 1,
      "far": 0.043478260869565216
    },
    {
      "subgroup": "airplane",
      "segments": 25,
      "recordings": 4,
      "fp": 1,
      "far": 0.04
    },
    {
      "subgroup": "washing_machine",
      "segments": 30,
      "recordings": 6,
      "fp": 1,
      "far": 0.03333333333333333
    },
    {
      "subgroup": "breathing",
      "segments": 24,
      "recordings": 7,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "brushing_teeth",
      "segments": 46,
      "recordings": 8,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "can_opening",
      "segments": 26,
      "recordings": 8,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "car_horn",
      "segments": 37,
      "recordings": 9,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "cat",
      "segments": 18,
      "recordings": 4,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "chirping_birds",
      "segments": 55,
      "recordings": 7,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "church_bells",
      "segments": 25,
      "recordings": 5,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "clapping",
      "segments": 59,
      "recordings": 10,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "clock_alarm",
      "segments": 25,
      "recordings": 4,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "clock_tick",
      "segments": 45,
      "recordings": 9,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "coughing",
      "segments": 23,
      "recordings": 7,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "cow",
      "segments": 31,
      "recordings": 5,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "crackling_fire",
      "segments": 41,
      "recordings": 7,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "crickets",
      "segments": 15,
      "recordings": 3,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "crow",
      "segments": 15,
      "recordings": 2,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "crying_baby",
      "segments": 68,
      "recordings": 4,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "dog",
      "segments": 38,
      "recordings": 9,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "door_wood_creaks",
      "segments": 26,
      "recordings": 5,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "door_wood_knock",
      "segments": 28,
      "recordings": 6,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "drinking_sipping",
      "segments": 25,
      "recordings": 5,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "footsteps",
      "segments": 34,
      "recordings": 6,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "frog",
      "segments": 55,
      "recordings": 5,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "hand_saw",
      "segments": 5,
      "recordings": 1,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "helicopter",
      "segments": 28,
      "recordings": 3,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "hen",
      "segments": 52,
      "recordings": 7,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "insects",
      "segments": 5,
      "recordings": 1,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "keyboard_typing",
      "segments": 40,
      "recordings": 7,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "laughing",
      "segments": 19,
      "recordings": 5,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "mouse_click",
      "segments": 3,
      "recordings": 1,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "other:unknown",
      "segments": 41,
      "recordings": 10,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "pouring_water",
      "segments": 15,
      "recordings": 2,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "rain",
      "segments": 25,
      "recordings": 5,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "rooster",
      "segments": 18,
      "recordings": 4,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "sea_waves",
      "segments": 20,
      "recordings": 3,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "sheep",
      "segments": 30,
      "recordings": 3,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "siren",
      "segments": 35,
      "recordings": 5,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "sneezing",
      "segments": 19,
      "recordings": 6,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "snoring",
      "segments": 38,
      "recordings": 6,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "thunderstorm",
      "segments": 20,
      "recordings": 2,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "train",
      "segments": 25,
      "recordings": 5,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "vacuum_cleaner",
      "segments": 20,
      "recordings": 4,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "water_drops",
      "segments": 12,
      "recordings": 3,
      "fp": 0,
      "far": 0.0
    },
    {
      "subgroup": "wind",
      "segments": 20,
      "recordings": 3,
      "fp": 0,
      "far": 0.0
    }
  ],
  "leakage": {
    "recordings_spanning_splits": 0
  },
  "targets": {
    "Drone recall": {
      "required": ">=0.975",
      "measured": 0.9545454545454546,
      "pass": false
    },
    "Specificity": {
      "required": ">=0.992",
      "measured": 0.988756148981026,
      "pass": false
    },
    "False-positive rate": {
      "required": "<=0.008",
      "measured": 0.011243851018973999,
      "pass": false
    },
    "ROC AUC": {
      "required": ">=0.99",
      "measured": 0.9970825613833345,
      "pass": true
    },
    "INT8 size (KB)": {
      "required": "<46.0",
      "measured": 39.2578125,
      "pass": true
    }
  },
  "targets_passed": 2,
  "targets_total": 5,
  "robustness": {
    "gain": {
      "note": "log-mel round trip, exact",
      "rows": [
        {
          "gain_db": 12,
          "recall": 0.9545454545454546,
          "mean_p": 0.8830366730690002
        },
        {
          "gain_db": 6,
          "recall": 0.9545454545454546,
          "mean_p": 0.8720455169677734
        },
        {
          "gain_db": 0,
          "recall": 0.9545454545454546,
          "mean_p": 0.8539071679115295
        },
        {
          "gain_db": -6,
          "recall": 0.9545454545454546,
          "mean_p": 0.8455276489257812
        },
        {
          "gain_db": -12,
          "recall": 0.9494949494949495,
          "mean_p": 0.841806173324585
        },
        {
          "gain_db": -18,
          "recall": 0.9292929292929293,
          "mean_p": 0.8254163265228271
        },
        {
          "gain_db": -24,
          "recall": 0.8939393939393939,
          "mean_p": 0.7971206307411194
        }
      ],
      "clean_recall": 0.9545454545454546
    },
    "noise": {
      "note": "power-domain mixing, approximate",
      "rows": [
        {
          "snr_db": 20,
          "recall": 0.9444444444444444,
          "mean_p": 0.847468376159668
        },
        {
          "snr_db": 15,
          "recall": 0.9242424242424242,
          "mean_p": 0.8327803015708923
        },
        {
          "snr_db": 10,
          "recall": 0.9090909090909091,
          "mean_p": 0.8135850429534912
        },
        {
          "snr_db": 5,
          "recall": 0.8333333333333334,
          "mean_p": 0.7600786685943604
        },
        {
          "snr_db": 0,
          "recall": 0.803030303030303,
          "mean_p": 0.7059253454208374
        },
        {
          "snr_db": -5,
          "recall": 0.6868686868686869,
          "mean_p": 0.6087281703948975
        }
      ],
      "clean_recall": 0.9545454545454546
    },
    "quiet": {
      "n_recordings": 56,
      "n_segments": 56,
      "native_recall": 0.9821428571428571,
      "native_mean_p": 0.962945282459259,
      "level_gap_db": 25.768593283507233,
      "gain_matched_recall": 1.0
    }
  },
  "hard_negative_overall": {
    "n": 1543,
    "false_alarms": 53,
    "far": 0.034348671419313026
  },
  "hard_negative_by_category": [
    {
      "category": "vacuum_cleaner",
      "n": 200,
      "false_alarms": 26,
      "false_alarm_rate": 0.13,
      "mean_p": 0.26350951194763184,
      "max_p": 0.7690551280975342,
      "n_test": 20,
      "false_alarms_test": 0,
      "false_alarm_rate_test": 0.0
    },
    {
      "category": "engine",
      "n": 192,
      "false_alarms": 1,
      "false_alarm_rate": 0.005208333333333333,
      "mean_p": 0.12046986073255539,
      "max_p": 0.4928717017173767,
      "n_test": 23,
      "false_alarms_test": 1,
      "false_alarm_rate_test": 0.043478260869565216
    },
    {
      "category": "airplane",
      "n": 200,
      "false_alarms": 1,
      "false_alarm_rate": 0.005,
      "mean_p": 0.09766843169927597,
      "max_p": 0.47673359513282776,
      "n_test": 25,
      "false_alarms_test": 1,
      "false_alarm_rate_test": 0.04
    },
    {
      "category": "helicopter",
      "n": 196,
      "false_alarms": 0,
      "false_alarm_rate": 0.0,
      "mean_p": 0.08340717852115631,
      "max_p": 0.33890360593795776,
      "n_test": 28,
      "false_alarms_test": 0,
      "false_alarm_rate_test": 0.0
    },
    {
      "category": "chainsaw",
      "n": 200,
      "false_alarms": 11,
      "false_alarm_rate": 0.055,
      "mean_p": 0.21956782042980194,
      "max_p": 0.7645877599716187,
      "n_test": 35,
      "false_alarms_test": 6,
      "false_alarm_rate_test": 0.17142857142857143
    },
    {
      "category": "train",
      "n": 200,
      "false_alarms": 3,
      "false_alarm_rate": 0.015,
      "mean_p": 0.14352451264858246,
      "max_p": 0.5502827167510986,
      "n_test": 25,
      "false_alarms_test": 0,
      "false_alarm_rate_test": 0.0
    },
    {
      "category": "siren",
      "n": 195,
      "false_alarms": 4,
      "false_alarm_rate": 0.020512820512820513,
      "mean_p": 0.15756839513778687,
      "max_p": 0.6565051674842834,
      "n_test": 35,
      "false_alarms_test": 0,
      "false_alarm_rate_test": 0.0
    },
    {
      "category": "car_horn",
      "n": 160,
      "false_alarms": 7,
      "false_alarm_rate": 0.04375,
      "mean_p": 0.15969011187553406,
      "max_p": 0.7667993307113647,
      "n_test": 37,
      "false_alarms_test": 0,
      "false_alarm_rate_test": 0.0
    }
  ],
  "confusable_far_fp32_vs_int8": {
    "set": "v2_robustness.CONFUSABLE, all splits (n=1543, 226 recordings)",
    "leakage_free": false,
    "note": "this set is mostly TRAIN data; see the test-subset column for the clean view",
    "fp32": {
      "false_alarms": 53,
      "n": 1543,
      "far": 0.034348671419313026
    },
    "int8": {
      "false_alarms": 59,
      "n": 1543,
      "far": 0.03823720025923526,
      "deployed": true
    },
    "delta_from_quantisation": 6,
    "test_subset": {
      "n": 228,
      "fp32_fa": 8,
      "int8_fa": 9
    }
  }
}