{
  "operating_points": {
    "external_commands_1pct_fpr": 0.519063,
    "block_0_1pct_fpr": 2.697449
  },
  "default_policy": "external_commands_1pct_fpr",
  "overall_test": {
    "external_commands_1pct_fpr": {
      "n": 7203,
      "pos": 1092,
      "tp": 1021,
      "fp": 44,
      "fn": 71,
      "tn": 6067,
      "recall": 0.935,
      "fpr": 0.0072,
      "fpr_wilson_upper": 0.0097,
      "ap": 0.9879,
      "auc": 0.9975
    },
    "block_0_1pct_fpr": {
      "n": 7203,
      "pos": 1092,
      "tp": 840,
      "fp": 4,
      "fn": 252,
      "tn": 6107,
      "recall": 0.7692,
      "fpr": 0.0007,
      "fpr_wilson_upper": 0.0017,
      "ap": 0.9879,
      "auc": 0.9975
    }
  },
  "original_bash_test": {
    "external_commands_1pct_fpr": {
      "n": 4194,
      "pos": 193,
      "tp": 168,
      "fp": 11,
      "fn": 25,
      "tn": 3990,
      "recall": 0.8705,
      "fpr": 0.0027,
      "fpr_wilson_upper": 0.0049,
      "ap": 0.9632,
      "auc": 0.9964
    },
    "block_0_1pct_fpr": {
      "n": 4194,
      "pos": 193,
      "tp": 127,
      "fp": 1,
      "fn": 66,
      "tn": 4000,
      "recall": 0.658,
      "fpr": 0.0002,
      "fpr_wilson_upper": 0.0014,
      "ap": 0.9632,
      "auc": 0.9964
    }
  },
  "per_family": {
    "shellsafety_powershell": {
      "external_commands_1pct_fpr": {
        "n": 119,
        "pos": 58,
        "tp": 58,
        "fp": 0,
        "fn": 0,
        "tn": 61,
        "recall": 1.0,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.0592,
        "ap": 1.0,
        "auc": 1.0
      },
      "block_0_1pct_fpr": {
        "n": 119,
        "pos": 58,
        "tp": 44,
        "fp": 0,
        "fn": 14,
        "tn": 61,
        "recall": 0.7586,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.0592,
        "ap": 1.0,
        "auc": 1.0
      }
    },
    "shellsafety_posix": {
      "external_commands_1pct_fpr": {
        "n": 735,
        "pos": 292,
        "tp": 265,
        "fp": 14,
        "fn": 27,
        "tn": 429,
        "recall": 0.9075,
        "fpr": 0.0316,
        "fpr_wilson_upper": 0.0523,
        "ap": 0.9857,
        "auc": 0.9892
      },
      "block_0_1pct_fpr": {
        "n": 735,
        "pos": 292,
        "tp": 211,
        "fp": 0,
        "fn": 81,
        "tn": 443,
        "recall": 0.7226,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.0086,
        "ap": 0.9857,
        "auc": 0.9892
      }
    },
    "shellsafety_cmd": {
      "external_commands_1pct_fpr": {
        "n": 64,
        "pos": 24,
        "tp": 24,
        "fp": 0,
        "fn": 0,
        "tn": 40,
        "recall": 1.0,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.0876,
        "ap": 1.0,
        "auc": 1.0
      },
      "block_0_1pct_fpr": {
        "n": 64,
        "pos": 24,
        "tp": 17,
        "fp": 0,
        "fn": 7,
        "tn": 40,
        "recall": 0.7083,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.0876,
        "ap": 1.0,
        "auc": 1.0
      }
    },
    "original_bash": {
      "external_commands_1pct_fpr": {
        "n": 4194,
        "pos": 193,
        "tp": 168,
        "fp": 11,
        "fn": 25,
        "tn": 3990,
        "recall": 0.8705,
        "fpr": 0.0027,
        "fpr_wilson_upper": 0.0049,
        "ap": 0.9632,
        "auc": 0.9964
      },
      "block_0_1pct_fpr": {
        "n": 4194,
        "pos": 193,
        "tp": 127,
        "fp": 1,
        "fn": 66,
        "tn": 4000,
        "recall": 0.658,
        "fpr": 0.0002,
        "fpr_wilson_upper": 0.0014,
        "ap": 0.9632,
        "auc": 0.9964
      }
    },
    "exfil_pipe": {
      "external_commands_1pct_fpr": {
        "n": 160,
        "pos": 160,
        "tp": 154,
        "fp": 0,
        "fn": 6,
        "tn": 0,
        "recall": 0.9625,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 160,
        "pos": 160,
        "tp": 145,
        "fp": 0,
        "fn": 15,
        "tn": 0,
        "recall": 0.9062,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "credential_store": {
      "external_commands_1pct_fpr": {
        "n": 80,
        "pos": 80,
        "tp": 80,
        "fp": 0,
        "fn": 0,
        "tn": 0,
        "recall": 1.0,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 80,
        "pos": 80,
        "tp": 66,
        "fp": 0,
        "fn": 14,
        "tn": 0,
        "recall": 0.825,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "reverse_shell": {
      "external_commands_1pct_fpr": {
        "n": 7,
        "pos": 7,
        "tp": 6,
        "fp": 0,
        "fn": 1,
        "tn": 0,
        "recall": 0.8571,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 7,
        "pos": 7,
        "tp": 3,
        "fp": 0,
        "fn": 4,
        "tn": 0,
        "recall": 0.4286,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "remote_download_exec": {
      "external_commands_1pct_fpr": {
        "n": 147,
        "pos": 147,
        "tp": 144,
        "fp": 0,
        "fn": 3,
        "tn": 0,
        "recall": 0.9796,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 147,
        "pos": 147,
        "tp": 136,
        "fp": 0,
        "fn": 11,
        "tn": 0,
        "recall": 0.9252,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "privilege_escalation": {
      "external_commands_1pct_fpr": {
        "n": 8,
        "pos": 8,
        "tp": 8,
        "fp": 0,
        "fn": 0,
        "tn": 0,
        "recall": 1.0,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 8,
        "pos": 8,
        "tp": 6,
        "fp": 0,
        "fn": 2,
        "tn": 0,
        "recall": 0.75,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "defense_evasion": {
      "external_commands_1pct_fpr": {
        "n": 26,
        "pos": 26,
        "tp": 25,
        "fp": 0,
        "fn": 1,
        "tn": 0,
        "recall": 0.9615,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 26,
        "pos": 26,
        "tp": 19,
        "fp": 0,
        "fn": 7,
        "tn": 0,
        "recall": 0.7308,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "disk_destruction": {
      "external_commands_1pct_fpr": {
        "n": 28,
        "pos": 28,
        "tp": 25,
        "fp": 0,
        "fn": 3,
        "tn": 0,
        "recall": 0.8929,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 28,
        "pos": 28,
        "tp": 16,
        "fp": 0,
        "fn": 12,
        "tn": 0,
        "recall": 0.5714,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "resource_dos": {
      "external_commands_1pct_fpr": {
        "n": 4,
        "pos": 4,
        "tp": 3,
        "fp": 0,
        "fn": 1,
        "tn": 0,
        "recall": 0.75,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 4,
        "pos": 4,
        "tp": 0,
        "fp": 0,
        "fn": 4,
        "tn": 0,
        "recall": 0.0,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "payload_decode_exec": {
      "external_commands_1pct_fpr": {
        "n": 6,
        "pos": 6,
        "tp": 6,
        "fp": 0,
        "fn": 0,
        "tn": 0,
        "recall": 1.0,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 6,
        "pos": 6,
        "tp": 5,
        "fp": 0,
        "fn": 1,
        "tn": 0,
        "recall": 0.8333,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "agent_log": {
      "external_commands_1pct_fpr": {
        "n": 307,
        "pos": 0,
        "tp": 0,
        "fp": 14,
        "fn": 0,
        "tn": 293,
        "recall": null,
        "fpr": 0.0456,
        "fpr_wilson_upper": 0.0751,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 307,
        "pos": 0,
        "tp": 0,
        "fp": 2,
        "fn": 0,
        "tn": 305,
        "recall": null,
        "fpr": 0.0065,
        "fpr_wilson_upper": 0.0234,
        "ap": null,
        "auc": null
      }
    },
    "issue_1": {
      "external_commands_1pct_fpr": {
        "n": 10,
        "pos": 10,
        "tp": 7,
        "fp": 0,
        "fn": 3,
        "tn": 0,
        "recall": 0.7,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 10,
        "pos": 10,
        "tp": 0,
        "fp": 0,
        "fn": 10,
        "tn": 0,
        "recall": 0.0,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "issue_2": {
      "external_commands_1pct_fpr": {
        "n": 10,
        "pos": 10,
        "tp": 9,
        "fp": 0,
        "fn": 1,
        "tn": 0,
        "recall": 0.9,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 10,
        "pos": 10,
        "tp": 8,
        "fp": 0,
        "fn": 2,
        "tn": 0,
        "recall": 0.8,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "issue_3": {
      "external_commands_1pct_fpr": {
        "n": 2,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 2,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.6576,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 2,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 2,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.6576,
        "ap": null,
        "auc": null
      }
    },
    "pipe_filter": {
      "external_commands_1pct_fpr": {
        "n": 10,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 10,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.2775,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 10,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 10,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.2775,
        "ap": null,
        "auc": null
      }
    },
    "near_miss": {
      "external_commands_1pct_fpr": {
        "n": 18,
        "pos": 0,
        "tp": 0,
        "fp": 1,
        "fn": 0,
        "tn": 17,
        "recall": null,
        "fpr": 0.0556,
        "fpr_wilson_upper": 0.2576,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 18,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 18,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.1759,
        "ap": null,
        "auc": null
      }
    },
    "routine_dev": {
      "external_commands_1pct_fpr": {
        "n": 13,
        "pos": 0,
        "tp": 0,
        "fp": 4,
        "fn": 0,
        "tn": 9,
        "recall": null,
        "fpr": 0.3077,
        "fpr_wilson_upper": 0.5763,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 13,
        "pos": 0,
        "tp": 0,
        "fp": 1,
        "fn": 0,
        "tn": 12,
        "recall": null,
        "fpr": 0.0769,
        "fpr_wilson_upper": 0.3331,
        "ap": null,
        "auc": null
      }
    },
    "aug159_ps_download_benign": {
      "external_commands_1pct_fpr": {
        "n": 28,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 28,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.1206,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 28,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 28,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.1206,
        "ap": null,
        "auc": null
      }
    },
    "agent_shell": {
      "external_commands_1pct_fpr": {
        "n": 275,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 275,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.0138,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 275,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 275,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.0138,
        "ap": null,
        "auc": null
      }
    },
    "agent_code": {
      "external_commands_1pct_fpr": {
        "n": 831,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 831,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.0046,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 831,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 831,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.0046,
        "ap": null,
        "auc": null
      }
    },
    "aug_heredoc_benign": {
      "external_commands_1pct_fpr": {
        "n": 40,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 40,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.0876,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 40,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 40,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.0876,
        "ap": null,
        "auc": null
      }
    },
    "aug_ops_benign": {
      "external_commands_1pct_fpr": {
        "n": 13,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 13,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.2281,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 13,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 13,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.2281,
        "ap": null,
        "auc": null
      }
    },
    "aug_ops_attack": {
      "external_commands_1pct_fpr": {
        "n": 12,
        "pos": 12,
        "tp": 12,
        "fp": 0,
        "fn": 0,
        "tn": 0,
        "recall": 1.0,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 12,
        "pos": 12,
        "tp": 12,
        "fp": 0,
        "fn": 0,
        "tn": 0,
        "recall": 1.0,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "issue159_bash_exec": {
      "external_commands_1pct_fpr": {
        "n": 17,
        "pos": 17,
        "tp": 17,
        "fp": 0,
        "fn": 0,
        "tn": 0,
        "recall": 1.0,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 17,
        "pos": 17,
        "tp": 16,
        "fp": 0,
        "fn": 1,
        "tn": 0,
        "recall": 0.9412,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "issue159_ps_exec": {
      "external_commands_1pct_fpr": {
        "n": 10,
        "pos": 10,
        "tp": 10,
        "fp": 0,
        "fn": 0,
        "tn": 0,
        "recall": 1.0,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 10,
        "pos": 10,
        "tp": 9,
        "fp": 0,
        "fn": 1,
        "tn": 0,
        "recall": 0.9,
        "fpr": null,
        "fpr_wilson_upper": null,
        "ap": null,
        "auc": null
      }
    },
    "issue159_bash_benign": {
      "external_commands_1pct_fpr": {
        "n": 14,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 14,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.2153,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 14,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 14,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.2153,
        "ap": null,
        "auc": null
      }
    },
    "issue159_ps_benign": {
      "external_commands_1pct_fpr": {
        "n": 8,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 8,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.3244,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 8,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 8,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.3244,
        "ap": null,
        "auc": null
      }
    },
    "heredoc_benign_fixture": {
      "external_commands_1pct_fpr": {
        "n": 6,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 6,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.3903,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 6,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 6,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.3903,
        "ap": null,
        "auc": null
      }
    },
    "ops_benign_fixture": {
      "external_commands_1pct_fpr": {
        "n": 1,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 1,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.7935,
        "ap": null,
        "auc": null
      },
      "block_0_1pct_fpr": {
        "n": 1,
        "pos": 0,
        "tp": 0,
        "fp": 0,
        "fn": 0,
        "tn": 1,
        "recall": null,
        "fpr": 0.0,
        "fpr_wilson_upper": 0.7935,
        "ap": null,
        "auc": null
      }
    }
  },
  "issue128_combined": {
    "external_commands_1pct_fpr": {
      "issue_attacks_recall": "16/20",
      "control_fp": 5
    },
    "block_0_1pct_fpr": {
      "issue_attacks_recall": "8/20",
      "control_fp": 1
    }
  }
}