| { | |
| "model": "V2C1", | |
| "threshold": 0.4684000015258789, | |
| "artifact": { | |
| "bytes": 40200, | |
| "sha256": "e2e4e4b0e49d50f0d3c0b2129240564ec84b93b4c252651ef10b82560b9d72ae", | |
| "size_kb": 39.2578125 | |
| }, | |
| "test_binary_metrics": { | |
| "threshold": 0.4684000015258789, | |
| "tp": 189, | |
| "fn": 9, | |
| "tn": 1407, | |
| "fp": 16, | |
| "n": 1621, | |
| "n_drone": 198, | |
| "n_nondrone": 1423, | |
| "recall_tpr": 0.9545454545454546, | |
| "specificity": 0.988756148981026, | |
| "fpr": 0.011243851018973999, | |
| "precision": 0.9219512195121952, | |
| "f1": 0.9379652605459058, | |
| "npv": 0.9936440677966102 | |
| }, | |
| "test_roc_auc": 0.9970825613833345, | |
| "test_pr_auc": 0.983962221893969, | |
| "evidence_units": { | |
| "segments": 1621, | |
| "recordings": 296, | |
| "sessions": null, | |
| "session_note": "corpus carries no session metadata; recording is the independence unit" | |
| }, | |
| "subgroup_far": [ | |
| { | |
| "subgroup": "chainsaw", | |
| "segments": 35, | |
| "recordings": 3, | |
| "fp": 7, | |
| "far": 0.2 | |
| }, | |
| { | |
| "subgroup": "toilet_flush", | |
| "segments": 25, | |
| "recordings": 5, | |
| "fp": 4, | |
| "far": 0.16 | |
| }, | |
| { | |
| "subgroup": "glass_breaking", | |
| "segments": 14, | |
| "recordings": 1, | |
| "fp": 1, | |
| "far": 0.07142857142857142 | |
| }, | |
| { | |
| "subgroup": "pig", | |
| "segments": 22, | |
| "recordings": 4, | |
| "fp": 1, | |
| "far": 0.045454545454545456 | |
| }, | |
| { | |
| "subgroup": "engine", | |
| "segments": 23, | |
| "recordings": 4, | |
| "fp": 1, | |
| "far": 0.043478260869565216 | |
| }, | |
| { | |
| "subgroup": "airplane", | |
| "segments": 25, | |
| "recordings": 4, | |
| "fp": 1, | |
| "far": 0.04 | |
| }, | |
| { | |
| "subgroup": "washing_machine", | |
| "segments": 30, | |
| "recordings": 6, | |
| "fp": 1, | |
| "far": 0.03333333333333333 | |
| }, | |
| { | |
| "subgroup": "breathing", | |
| "segments": 24, | |
| "recordings": 7, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "brushing_teeth", | |
| "segments": 46, | |
| "recordings": 8, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "can_opening", | |
| "segments": 26, | |
| "recordings": 8, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "car_horn", | |
| "segments": 37, | |
| "recordings": 9, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "cat", | |
| "segments": 18, | |
| "recordings": 4, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "chirping_birds", | |
| "segments": 55, | |
| "recordings": 7, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "church_bells", | |
| "segments": 25, | |
| "recordings": 5, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "clapping", | |
| "segments": 59, | |
| "recordings": 10, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "clock_alarm", | |
| "segments": 25, | |
| "recordings": 4, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "clock_tick", | |
| "segments": 45, | |
| "recordings": 9, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "coughing", | |
| "segments": 23, | |
| "recordings": 7, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "cow", | |
| "segments": 31, | |
| "recordings": 5, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "crackling_fire", | |
| "segments": 41, | |
| "recordings": 7, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "crickets", | |
| "segments": 15, | |
| "recordings": 3, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "crow", | |
| "segments": 15, | |
| "recordings": 2, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "crying_baby", | |
| "segments": 68, | |
| "recordings": 4, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "dog", | |
| "segments": 38, | |
| "recordings": 9, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "door_wood_creaks", | |
| "segments": 26, | |
| "recordings": 5, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "door_wood_knock", | |
| "segments": 28, | |
| "recordings": 6, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "drinking_sipping", | |
| "segments": 25, | |
| "recordings": 5, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "footsteps", | |
| "segments": 34, | |
| "recordings": 6, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "frog", | |
| "segments": 55, | |
| "recordings": 5, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "hand_saw", | |
| "segments": 5, | |
| "recordings": 1, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "helicopter", | |
| "segments": 28, | |
| "recordings": 3, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "hen", | |
| "segments": 52, | |
| "recordings": 7, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "insects", | |
| "segments": 5, | |
| "recordings": 1, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "keyboard_typing", | |
| "segments": 40, | |
| "recordings": 7, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "laughing", | |
| "segments": 19, | |
| "recordings": 5, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "mouse_click", | |
| "segments": 3, | |
| "recordings": 1, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "other:unknown", | |
| "segments": 41, | |
| "recordings": 10, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "pouring_water", | |
| "segments": 15, | |
| "recordings": 2, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "rain", | |
| "segments": 25, | |
| "recordings": 5, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "rooster", | |
| "segments": 18, | |
| "recordings": 4, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "sea_waves", | |
| "segments": 20, | |
| "recordings": 3, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "sheep", | |
| "segments": 30, | |
| "recordings": 3, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "siren", | |
| "segments": 35, | |
| "recordings": 5, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "sneezing", | |
| "segments": 19, | |
| "recordings": 6, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "snoring", | |
| "segments": 38, | |
| "recordings": 6, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "thunderstorm", | |
| "segments": 20, | |
| "recordings": 2, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "train", | |
| "segments": 25, | |
| "recordings": 5, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "vacuum_cleaner", | |
| "segments": 20, | |
| "recordings": 4, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "water_drops", | |
| "segments": 12, | |
| "recordings": 3, | |
| "fp": 0, | |
| "far": 0.0 | |
| }, | |
| { | |
| "subgroup": "wind", | |
| "segments": 20, | |
| "recordings": 3, | |
| "fp": 0, | |
| "far": 0.0 | |
| } | |
| ], | |
| "leakage": { | |
| "recordings_spanning_splits": 0 | |
| }, | |
| "targets": { | |
| "Drone recall": { | |
| "required": ">=0.975", | |
| "measured": 0.9545454545454546, | |
| "pass": false | |
| }, | |
| "Specificity": { | |
| "required": ">=0.992", | |
| "measured": 0.988756148981026, | |
| "pass": false | |
| }, | |
| "False-positive rate": { | |
| "required": "<=0.008", | |
| "measured": 0.011243851018973999, | |
| "pass": false | |
| }, | |
| "ROC AUC": { | |
| "required": ">=0.99", | |
| "measured": 0.9970825613833345, | |
| "pass": true | |
| }, | |
| "INT8 size (KB)": { | |
| "required": "<46.0", | |
| "measured": 39.2578125, | |
| "pass": true | |
| } | |
| }, | |
| "targets_passed": 2, | |
| "targets_total": 5, | |
| "robustness": { | |
| "gain": { | |
| "note": "log-mel round trip, exact", | |
| "rows": [ | |
| { | |
| "gain_db": 12, | |
| "recall": 0.9545454545454546, | |
| "mean_p": 0.8830366730690002 | |
| }, | |
| { | |
| "gain_db": 6, | |
| "recall": 0.9545454545454546, | |
| "mean_p": 0.8720455169677734 | |
| }, | |
| { | |
| "gain_db": 0, | |
| "recall": 0.9545454545454546, | |
| "mean_p": 0.8539071679115295 | |
| }, | |
| { | |
| "gain_db": -6, | |
| "recall": 0.9545454545454546, | |
| "mean_p": 0.8455276489257812 | |
| }, | |
| { | |
| "gain_db": -12, | |
| "recall": 0.9494949494949495, | |
| "mean_p": 0.841806173324585 | |
| }, | |
| { | |
| "gain_db": -18, | |
| "recall": 0.9292929292929293, | |
| "mean_p": 0.8254163265228271 | |
| }, | |
| { | |
| "gain_db": -24, | |
| "recall": 0.8939393939393939, | |
| "mean_p": 0.7971206307411194 | |
| } | |
| ], | |
| "clean_recall": 0.9545454545454546 | |
| }, | |
| "noise": { | |
| "note": "power-domain mixing, approximate", | |
| "rows": [ | |
| { | |
| "snr_db": 20, | |
| "recall": 0.9444444444444444, | |
| "mean_p": 0.847468376159668 | |
| }, | |
| { | |
| "snr_db": 15, | |
| "recall": 0.9242424242424242, | |
| "mean_p": 0.8327803015708923 | |
| }, | |
| { | |
| "snr_db": 10, | |
| "recall": 0.9090909090909091, | |
| "mean_p": 0.8135850429534912 | |
| }, | |
| { | |
| "snr_db": 5, | |
| "recall": 0.8333333333333334, | |
| "mean_p": 0.7600786685943604 | |
| }, | |
| { | |
| "snr_db": 0, | |
| "recall": 0.803030303030303, | |
| "mean_p": 0.7059253454208374 | |
| }, | |
| { | |
| "snr_db": -5, | |
| "recall": 0.6868686868686869, | |
| "mean_p": 0.6087281703948975 | |
| } | |
| ], | |
| "clean_recall": 0.9545454545454546 | |
| }, | |
| "quiet": { | |
| "n_recordings": 56, | |
| "n_segments": 56, | |
| "native_recall": 0.9821428571428571, | |
| "native_mean_p": 0.962945282459259, | |
| "level_gap_db": 25.768593283507233, | |
| "gain_matched_recall": 1.0 | |
| } | |
| }, | |
| "hard_negative_overall": { | |
| "n": 1543, | |
| "false_alarms": 53, | |
| "far": 0.034348671419313026 | |
| }, | |
| "hard_negative_by_category": [ | |
| { | |
| "category": "vacuum_cleaner", | |
| "n": 200, | |
| "false_alarms": 26, | |
| "false_alarm_rate": 0.13, | |
| "mean_p": 0.26350951194763184, | |
| "max_p": 0.7690551280975342, | |
| "n_test": 20, | |
| "false_alarms_test": 0, | |
| "false_alarm_rate_test": 0.0 | |
| }, | |
| { | |
| "category": "engine", | |
| "n": 192, | |
| "false_alarms": 1, | |
| "false_alarm_rate": 0.005208333333333333, | |
| "mean_p": 0.12046986073255539, | |
| "max_p": 0.4928717017173767, | |
| "n_test": 23, | |
| "false_alarms_test": 1, | |
| "false_alarm_rate_test": 0.043478260869565216 | |
| }, | |
| { | |
| "category": "airplane", | |
| "n": 200, | |
| "false_alarms": 1, | |
| "false_alarm_rate": 0.005, | |
| "mean_p": 0.09766843169927597, | |
| "max_p": 0.47673359513282776, | |
| "n_test": 25, | |
| "false_alarms_test": 1, | |
| "false_alarm_rate_test": 0.04 | |
| }, | |
| { | |
| "category": "helicopter", | |
| "n": 196, | |
| "false_alarms": 0, | |
| "false_alarm_rate": 0.0, | |
| "mean_p": 0.08340717852115631, | |
| "max_p": 0.33890360593795776, | |
| "n_test": 28, | |
| "false_alarms_test": 0, | |
| "false_alarm_rate_test": 0.0 | |
| }, | |
| { | |
| "category": "chainsaw", | |
| "n": 200, | |
| "false_alarms": 11, | |
| "false_alarm_rate": 0.055, | |
| "mean_p": 0.21956782042980194, | |
| "max_p": 0.7645877599716187, | |
| "n_test": 35, | |
| "false_alarms_test": 6, | |
| "false_alarm_rate_test": 0.17142857142857143 | |
| }, | |
| { | |
| "category": "train", | |
| "n": 200, | |
| "false_alarms": 3, | |
| "false_alarm_rate": 0.015, | |
| "mean_p": 0.14352451264858246, | |
| "max_p": 0.5502827167510986, | |
| "n_test": 25, | |
| "false_alarms_test": 0, | |
| "false_alarm_rate_test": 0.0 | |
| }, | |
| { | |
| "category": "siren", | |
| "n": 195, | |
| "false_alarms": 4, | |
| "false_alarm_rate": 0.020512820512820513, | |
| "mean_p": 0.15756839513778687, | |
| "max_p": 0.6565051674842834, | |
| "n_test": 35, | |
| "false_alarms_test": 0, | |
| "false_alarm_rate_test": 0.0 | |
| }, | |
| { | |
| "category": "car_horn", | |
| "n": 160, | |
| "false_alarms": 7, | |
| "false_alarm_rate": 0.04375, | |
| "mean_p": 0.15969011187553406, | |
| "max_p": 0.7667993307113647, | |
| "n_test": 37, | |
| "false_alarms_test": 0, | |
| "false_alarm_rate_test": 0.0 | |
| } | |
| ], | |
| "confusable_far_fp32_vs_int8": { | |
| "set": "v2_robustness.CONFUSABLE, all splits (n=1543, 226 recordings)", | |
| "leakage_free": false, | |
| "note": "this set is mostly TRAIN data; see the test-subset column for the clean view", | |
| "fp32": { | |
| "false_alarms": 53, | |
| "n": 1543, | |
| "far": 0.034348671419313026 | |
| }, | |
| "int8": { | |
| "false_alarms": 59, | |
| "n": 1543, | |
| "far": 0.03823720025923526, | |
| "deployed": true | |
| }, | |
| "delta_from_quantisation": 6, | |
| "test_subset": { | |
| "n": 228, | |
| "fp32_fa": 8, | |
| "int8_fa": 9 | |
| } | |
| } | |
| } | |