{
  "model": {
    "kind": "char_cnn_transformer",
    "initialization": "random_weights",
    "parameters": 1959361,
    "config": {
      "max_length": 1000,
      "embed_dim": 128,
      "width": 192,
      "layers": 4,
      "heads": 8,
      "dropout": 0.1
    },
    "device": "cuda",
    "epochs": 12,
    "best_epoch": 10,
    "batch_size": 128,
    "learning_rate": 0.0006,
    "seed": 42,
    "temperature": 1.6057,
    "training_seconds": 322.2,
    "torch_version": "2.11.0+cu128"
  },
  "threshold": 0.5241324296053946,
  "target_fpr": 0.001,
  "rows": 96822,
  "sources": [
    "agent_attack_line",
    "agent_attack_request",
    "agent_attack_value",
    "agent_line",
    "agent_multilingual",
    "agent_other_attack",
    "agent_request",
    "agent_value",
    "mereani",
    "synthetic",
    "twin_urls"
  ],
  "cleaning": {
    "input_rows": 96835,
    "empty_rows": 0,
    "conflicting_texts": 0,
    "duplicate_rows": 0,
    "output_rows": 96835,
    "challenge_overlap_rows_removed": 13
  },
  "held_out_test": {
    "samples": 14524,
    "roc_auc": 0.99995,
    "pr_auc": 0.99986,
    "detection_at_1pct_false_alarms": 0.9982,
    "detection_at_0.1pct_false_alarms": 0.9971,
    "threshold": 0.5241,
    "detection_rate": 0.9953,
    "false_alarm_rate": 0.0001,
    "precision": 0.9997,
    "confusion": {
      "tp": 3392,
      "fp": 1,
      "fn": 16,
      "tn": 11115
    },
    "by_source": {
      "agent_attack_line": {
        "attacks": 544,
        "caught": 539,
        "benign": 0,
        "false_alarms": 0
      },
      "agent_attack_request": {
        "attacks": 311,
        "caught": 311,
        "benign": 0,
        "false_alarms": 0
      },
      "agent_attack_value": {
        "attacks": 398,
        "caught": 398,
        "benign": 0,
        "false_alarms": 0
      },
      "agent_line": {
        "attacks": 0,
        "caught": 0,
        "benign": 693,
        "false_alarms": 0
      },
      "agent_multilingual": {
        "attacks": 0,
        "caught": 0,
        "benign": 450,
        "false_alarms": 0
      },
      "agent_other_attack": {
        "attacks": 0,
        "caught": 0,
        "benign": 333,
        "false_alarms": 0
      },
      "agent_request": {
        "attacks": 0,
        "caught": 0,
        "benign": 832,
        "false_alarms": 0
      },
      "agent_value": {
        "attacks": 0,
        "caught": 0,
        "benign": 723,
        "false_alarms": 0
      },
      "mereani": {
        "attacks": 2155,
        "caught": 2144,
        "benign": 4210,
        "false_alarms": 1
      },
      "synthetic": {
        "attacks": 0,
        "caught": 0,
        "benign": 1860,
        "false_alarms": 0
      },
      "twin_urls": {
        "attacks": 0,
        "caught": 0,
        "benign": 2015,
        "false_alarms": 0
      }
    }
  },
  "challenge_set": {
    "attacks": 60,
    "caught": 60,
    "missed": [],
    "benign": 80,
    "false_alarms": []
  },
  "external_benchmark": {
    "attack_sources": {
      "seclists_brutelogic": {
        "attacks": 101,
        "caught": 89,
        "unseen_attacks": 99,
        "unseen_caught": 87
      },
      "seclists_jhaddix": {
        "attacks": 106,
        "caught": 92,
        "unseen_attacks": 64,
        "unseen_caught": 50
      },
      "seclists_mario": {
        "attacks": 137,
        "caught": 115,
        "unseen_attacks": 135,
        "unseen_caught": 113
      },
      "seclists_ofjaaah": {
        "attacks": 198,
        "caught": 171,
        "unseen_attacks": 196,
        "unseen_caught": 169
      },
      "seclists_payloadbox": {
        "attacks": 1213,
        "caught": 1132,
        "unseen_attacks": 874,
        "unseen_caught": 797
      },
      "seclists_polyglots": {
        "attacks": 13,
        "caught": 13,
        "unseen_attacks": 13,
        "unseen_caught": 13
      },
      "seclists_portswigger": {
        "attacks": 6042,
        "caught": 6042,
        "unseen_attacks": 6041,
        "unseen_caught": 6041
      },
      "seclists_rsnake": {
        "attacks": 66,
        "caught": 62,
        "unseen_attacks": 12,
        "unseen_caught": 8
      },
      "seclists_somdev": {
        "attacks": 20,
        "caught": 18,
        "unseen_attacks": 19,
        "unseen_caught": 17
      }
    },
    "benign_sources": {
      "oasst": {
        "benign": 1994,
        "false_alarms": 5
      },
      "wikitext": {
        "benign": 4999,
        "false_alarms": 0
      }
    },
    "attacks": 7896,
    "caught": 7734,
    "unseen_attacks": 7453,
    "unseen_caught": 7295,
    "benign": 6993,
    "false_alarms": 5,
    "false_alarm_examples": [
      "sure, here's an example implementation of an observer pattern in rust based on the rxjs library: ```rust use std::sync::{arc, mutex}; use std::collections::hashmap; type eventhandler<t> = arc<mutex<bo",
      "to draw a circle with a radius of 10 pixels in a html canvas using javascript, you can use the following code: <html> <head> <script> function drawcircle() { var canvas = document.getelementbyid(\"myca",
      "to create a frame by frame animation in react.js, you can follow these steps: 1. create a new react component for your animation. you can use the usestate hook to keep track of the current frame index",
      "there are several libraries available to help you work with websockets in javascript, each with varying levels of abstraction. if you're looking for the simplest and highest level of abstraction, you ",
      "here's a basic example of creating a circle with each library, as well as some relevant differences and considerations for each library. to set up each library, you can either download the library fil"
    ]
  },
  "linear_baseline": {
    "held_out_test": {
      "samples": 14524,
      "roc_auc": 0.99995,
      "pr_auc": 0.99984,
      "detection_at_1pct_false_alarms": 1.0,
      "detection_at_0.1pct_false_alarms": 0.9891,
      "threshold": 0.7474,
      "detection_rate": 0.9856,
      "false_alarm_rate": 0.0007,
      "precision": 0.9976,
      "confusion": {
        "tp": 3359,
        "fp": 8,
        "fn": 49,
        "tn": 11108
      }
    },
    "challenge_set": {
      "attacks": 60,
      "caught": 60,
      "missed": [],
      "benign": 80,
      "false_alarms": []
    },
    "external_benchmark": {
      "attack_sources": {
        "seclists_brutelogic": {
          "attacks": 101,
          "caught": 87,
          "unseen_attacks": 99,
          "unseen_caught": 85
        },
        "seclists_jhaddix": {
          "attacks": 106,
          "caught": 101,
          "unseen_attacks": 64,
          "unseen_caught": 61
        },
        "seclists_mario": {
          "attacks": 137,
          "caught": 117,
          "unseen_attacks": 135,
          "unseen_caught": 116
        },
        "seclists_ofjaaah": {
          "attacks": 198,
          "caught": 175,
          "unseen_attacks": 196,
          "unseen_caught": 173
        },
        "seclists_payloadbox": {
          "attacks": 1213,
          "caught": 1139,
          "unseen_attacks": 874,
          "unseen_caught": 808
        },
        "seclists_polyglots": {
          "attacks": 13,
          "caught": 13,
          "unseen_attacks": 13,
          "unseen_caught": 13
        },
        "seclists_portswigger": {
          "attacks": 6042,
          "caught": 6042,
          "unseen_attacks": 6041,
          "unseen_caught": 6041
        },
        "seclists_rsnake": {
          "attacks": 66,
          "caught": 62,
          "unseen_attacks": 12,
          "unseen_caught": 8
        },
        "seclists_somdev": {
          "attacks": 20,
          "caught": 19,
          "unseen_attacks": 19,
          "unseen_caught": 18
        }
      },
      "benign_sources": {
        "oasst": {
          "benign": 1994,
          "false_alarms": 0
        },
        "wikitext": {
          "benign": 4999,
          "false_alarms": 0
        }
      },
      "attacks": 7896,
      "caught": 7755,
      "unseen_attacks": 7453,
      "unseen_caught": 7323,
      "benign": 6993,
      "false_alarms": 0,
      "false_alarm_examples": []
    }
  }
}