data/security-benchmarks.js

window.SECURITY_BENCHMARKS = {
  snapshot_utc: "2026-09-07T17:30:00Z",
  benchmarks: [
    {
      slug: "exploitbench",
      title: "ExploitBench",
      status: "Active",
      family: "Exploit development",
      metric: "5-tier ladder",
      summary: "Measures end-to-end exploit development on 41 real, patched V8 vulnerabilities with production mitigations enabled.",
      scale: "41 V8 bugs · 16 capabilities · 5 tiers",
      task: "Given a vulnerable V8 build and the patch that fixed it, develop an exploit against the production attack surface. The capability ladder runs from code coverage and bug reproduction through target-specific primitives, generic primitives outside the V8 heap sandbox, and full control-flow hijack or arbitrary code execution.",
      grading: "Sixteen capabilities are checked programmatically. Lower tiers use differential execution against patched builds; higher tiers use challenge-response checks replayed across randomized heap layouts. No human or LLM judge is required.",
      why: "The shared public exploit ladder reported by both Anthropic and OpenAI. It exposes the difference between merely reproducing a crash and converting it into a sandbox escape or ACE.",
      caveat: "Results depend on the run arm, scaffold, seed count, and turn budget. Compare like-for-like configurations.",
      reported_by: ["Anthropic", "OpenAI", "ExploitBench"],
      references: [
        { title: "ExploitBench", url: "https://exploitbench.ai/" },
        { title: "Anthropic: Measuring LLMs’ ability to develop exploits", url: "https://www.anthropic.com/research/exploit-evals" },
        { title: "ExploitBench paper", url: "https://arxiv.org/abs/2605.14153" }
      ]
    },
    {
      slug: "exploitgym",
      title: "ExploitGym",
      status: "Active",
      family: "Exploit development",
      metric: "ACE + flag",
      summary: "Tests whether agents can turn a known crash into unauthorized code execution across a large, diverse target set.",
      scale: "898 vulnerabilities · userspace, Linux kernel, and V8",
      task: "The agent receives vulnerable source and build information, vulnerability information and a proof-of-vulnerability, a compiled runtime, and a remote target. It must weaponize the bug, execute code at an otherwise unreachable privilege level, and retrieve a dynamic secret flag.",
      grading: "A run succeeds only when the correct dynamic flag is submitted and a judge attributes the result to the intended vulnerability. The harness can toggle mitigations including the V8 heap sandbox and Linux KASLR.",
      why: "The successor to CyberGym for frontier claims: it tests the hard crash-to-exploit conversion rather than stopping at vulnerability reproduction.",
      caveat: "Mitigation settings materially change difficulty. Report enabled defenses alongside aggregate success rates.",
      reported_by: ["Anthropic", "OpenAI", "UC Berkeley RDI"],
      references: [
        { title: "ExploitGym paper", url: "https://arxiv.org/abs/2605.11086" },
        { title: "UC Berkeley RDI overview", url: "https://rdi.berkeley.edu/blog/exploitgym/" },
        { title: "Anthropic exploit evaluations", url: "https://www.anthropic.com/research/exploit-evals" }
      ]
    },
    {
      slug: "post-cutoff-zero-day-labs",
      title: "Post-cutoff / 0-day lab evals",
      status: "Active",
      family: "Fresh-target labs",
      metric: "Verified exploit",
      summary: "Private, freshness-controlled evaluations on recent ports, Firefox 147, and supervised browser or OS zero-days.",
      scale: "Vendor-specific · post-cutoff and undisclosed targets",
      task: "Evaluate exploit discovery and development on targets that are too recent or private to appear in training data. Publicly described examples include fresh ExploitBench ports, patched Firefox 147 JavaScript-engine flaws, and supervised campaigns against browser and operating-system zero-days.",
      grading: "Strong versions require a verifier-confirmed exploit outcome—such as a working shell exploit, register control, sandbox escape, privilege escalation, or ACE—rather than a prose assessment. Human supervision and disclosure workflows are often part of the lab protocol.",
      why: "Load-bearing evidence for Critical-tier claims because freshness and private targets reduce benchmark memorization and contamination concerns.",
      caveat: "This is an evaluation family, not one standardized public benchmark. Scores across vendors are not directly comparable without target, mitigation, scaffold, budget, and verifier details.",
      reported_by: ["Anthropic", "OpenAI"],
      references: [
        { title: "Anthropic: Assessing Claude Mythos Preview", url: "https://www.anthropic.com/research/mythos-preview" },
        { title: "Anthropic exploit evaluations", url: "https://www.anthropic.com/research/exploit-evals" },
        { title: "OpenAI: GPT-6 Astra", url: "https://openai.com/index/gpt-6-astra/" }
      ]
    },
    {
      slug: "sre-bench",
      title: "SRE-Bench",
      status: "Active",
      family: "Reverse engineering",
      metric: "Task success",
      summary: "A contamination-resistant reverse-engineering benchmark using protected binaries whose source is withheld from agents.",
      scale: "262 binaries · 1,572 tasks · 5 scenarios",
      task: "Given a protected executable and analysis tools—but no source—recover specified behavior, structure, or hidden state and submit the artifact or answer required by a task-specific verifier.",
      grading: "Each task uses a concrete deterministic check. The programs were built from scratch and hardened with an in-house anti-analysis suite to reduce training-data contamination.",
      why: "Held-out binary understanding is a distinct frontier capability and is reported on the Astra card.",
      caveat: "The benchmark is under active development; counts, definitions, and procedures may change before final release.",
      reported_by: ["DAPLab", "OpenAI"],
      references: [
        { title: "SRE-Bench", url: "https://daplab.cs.columbia.edu/benchmarks/sre-bench/" },
        { title: "SRE-Bench paper", url: "https://arxiv.org/abs/2608.11469" },
        { title: "OpenAI: GPT-6 Astra", url: "https://openai.com/index/gpt-6-astra/" }
      ]
    },
    {
      slug: "bountybench",
      title: "BountyBench",
      status: "Active",
      family: "Real systems",
      metric: "Success + $ impact",
      summary: "Evaluates vulnerability detection, exploitation, and patching in real systems and maps successes to actual bounty value.",
      scale: "25 systems · 40 bounties · 120 tasks",
      task: "Agents work in configured real-world codebases and services across three phases: Detect an unknown vulnerability, Exploit a described vulnerability, or Patch it without breaking normal behavior.",
      grading: "Exploit scripts are checked with task-specific verifiers. Patches must block the reference exploit and pass invariant tests. Results can be aggregated as task success and the historical dollar value of solved bounties.",
      why: "Connects offense and defense to operational systems and economic impact instead of CTF flags alone.",
      caveat: "The original paper snapshot and later community environment releases have different task counts; this card uses the 25-system, 40-bounty paper version.",
      reported_by: ["Stanford", "Berkeley", "BountyBench"],
      references: [
        { title: "BountyBench paper", url: "https://arxiv.org/abs/2505.15216" },
        { title: "BountyBench repository", url: "https://github.com/bountybench/bountybench" },
        { title: "Stanford RegLab overview", url: "https://reglab.stanford.edu/publications/bountybench-dollar-impact-of-ai-agent-attackers-and-defenders-on-real-world-cybersecurity-systems/" }
      ]
    },
    {
      slug: "sec-bench-pro",
      title: "SEC-Bench Pro",
      status: "Active",
      family: "Vulnerability discovery",
      metric: "Validated PoC",
      summary: "A self-evolving benchmark for long-horizon bug hunting and PoC generation in V8, SpiderMonkey, and Linux.",
      scale: "344 verified cases · 3 critical projects",
      task: "Agents inspect large, security-critical codebases and produce proof-of-concept inputs for disclosed vulnerabilities. Each instance packages metadata, a prompt, a harness, and vulnerable, fixed, and latest execution images.",
      grading: "The submitted PoC is run across image triples. Project-specific crash oracles and a judge distinguish the target vulnerability from unrelated crashes and check behavior on fixed or latest images.",
      why: "Still useful secondary evidence for JS-engine and kernel vulnerability discovery, with reproducible environments and evolving snapshots.",
      caveat: "It primarily measures vulnerability discovery and reproduction, not the full exploit ladder or reliable ACE.",
      reported_by: ["SEC-bench", "OpenAI"],
      references: [
        { title: "SEC-Bench Pro", url: "https://sec-bench.github.io/" },
        { title: "SEC-Bench Pro repository", url: "https://github.com/SEC-bench/SEC-bench-Pro" },
        { title: "SEC-Bench Pro paper", url: "https://arxiv.org/abs/2605.26548" }
      ]
    },
    {
      slug: "3cb",
      title: "3CB",
      status: "Active",
      family: "ATT&CK coverage",
      metric: "Flag success",
      summary: "Catastrophic Cyber Capabilities Benchmark samples offensive skills across the MITRE ATT&CK taxonomy.",
      scale: "15 challenges · 11 public · 4 holdout",
      task: "Complete controlled cyber-offense challenges mapped to MITRE ATT&CK sub-techniques, spanning reconnaissance, initial access, execution, persistence, privilege escalation, lateral movement, collection, and impact.",
      grading: "Each sandboxed challenge has a hidden flag; success requires executing the relevant technique and retrieving it. Holdout tasks test generalization beyond the public challenge set.",
      why: "Useful for breadth and ATT&CK coverage, especially when comparing agent scaffolds.",
      caveat: "A small CTF-style suite is not a strong ceiling for current frontier exploit-development capability.",
      reported_by: ["Gray Swan AI", "Academic benchmark"],
      references: [
        { title: "3CB paper", url: "https://arxiv.org/abs/2410.09114" }
      ]
    },
    {
      slug: "autopenbench",
      title: "AutoPenBench",
      status: "Active",
      family: "Pentest agents",
      metric: "Success + milestones",
      summary: "Compares autonomous and human-assisted agents on multi-stage penetration-testing scenarios.",
      scale: "33 vulnerable containers · 2 difficulty levels",
      task: "Agents execute unrestricted commands against basic in-vitro targets and more complex real-world penetration-testing scenarios. The suite records intermediate milestones as well as end-to-end success.",
      grading: "Success rate measures completed compromises; milestone completion exposes partial progress. The reference work compares fully autonomous and semi-autonomous agent architectures.",
      why: "Useful for pentest-agent and human-assistance comparisons rather than frontier saturation claims.",
      caveat: "Its target scale and difficulty are below newer crash-to-ACE and private zero-day evaluations.",
      reported_by: ["Academic benchmark"],
      references: [
        { title: "AutoPenBench paper", url: "https://arxiv.org/abs/2410.03225" },
        { title: "AutoPenBench repository", url: "https://github.com/lucagioacchini/auto-pen-bench" }
      ]
    },
    {
      slug: "cybench",
      title: "Cybench",
      status: "Retired",
      family: "CTF",
      metric: "Flag + subtasks",
      summary: "Forty professional CTF challenges with intermediary subtasks for partial-credit capability measurement.",
      scale: "40 tasks · 4 competitions · 6 categories",
      task: "Solve recent professional CTF challenges in a Kali Linux environment using local files and task servers. Categories include cryptography, web security, reverse engineering, forensics, exploitation, and miscellaneous skills.",
      grading: "Final flags are checked against answer keys; sequential subtasks measure intermediate progress. Human first-solve time provides an external difficulty signal.",
      why: "Historically a model-card and AISI default, but no longer the headline evidence for frontier exploit capability.",
      caveat: "Retired here means dropped from the frontier system-card role—not that the public benchmark or leaderboard has shut down.",
      reported_by: ["Stanford CRFM", "UK AISI"],
      references: [
        { title: "Cybench", url: "https://cybench.github.io/" },
        { title: "Stanford CRFM overview", url: "https://crfm.stanford.edu/2024/08/19/cybench.html" }
      ]
    },
    {
      slug: "nyu-ctf-bench",
      title: "NYU CTF Bench",
      status: "Retired",
      family: "CTF",
      metric: "Challenge score",
      summary: "A large open-source collection of university-level CSAW CTF challenges for interactive security agents.",
      scale: "200 challenges · CSAW 2016–2024",
      task: "Solve CTF challenges across six security domains with shell tools, binary-analysis utilities, and interactive target environments.",
      grading: "Challenge completion and point-based scoring are aggregated across the suite.",
      why: "Still valuable for reproducible academic work and broad CTF-agent testing.",
      caveat: "Public writeups and age create contamination risk, making it unsuitable as a modern frontier capability ceiling.",
      reported_by: ["NYU", "NeurIPS 2024"],
      references: [
        { title: "NYU CTF Bench", url: "https://nyu-llm-ctf.github.io/" },
        { title: "NeurIPS paper", url: "https://proceedings.neurips.cc/paper_files/paper/2024/hash/69d97a6493fbf016fff0a751f253ad18-Abstract-Datasets_and_Benchmarks_Track.html" }
      ]
    },
    {
      slug: "cve-bench",
      title: "CVE-Bench",
      status: "Retired",
      family: "Web exploitation",
      metric: "Goal completion",
      summary: "Tests agents against critical real-world web application CVEs with concrete attack outcomes.",
      scale: "40 critical CVEs",
      task: "Attack a vulnerable web application to achieve outcomes such as denial of service, file or database access, remote code execution, privilege escalation, administrator login, or outbound requests.",
      grading: "Inspect-based evaluators verify the specified impact for each CVE in a controlled target environment.",
      why: "A useful real-CVE suite and historical capability marker.",
      caveat: "Retired here reflects the supplied frontier-card taxonomy: it is treated as saturated, not as a claim that the repository is unmaintained.",
      reported_by: ["UIUC", "Academic benchmark"],
      references: [
        { title: "CVE-Bench repository", url: "https://github.com/uiuc-kang-lab/cve-bench" },
        { title: "CVE-Bench paper", url: "https://arxiv.org/abs/2503.17332" }
      ]
    },
    {
      slug: "cybergym",
      title: "CyberGym",
      status: "Retired",
      family: "Vulnerability reproduction",
      metric: "Differential crash",
      summary: "A large OSS-Fuzz-derived benchmark for producing PoCs that reproduce real memory-safety vulnerabilities.",
      scale: "1,507 tasks · 188 projects",
      task: "Given a codebase, vulnerability context, and build environment, produce an input that triggers the target bug in the vulnerable revision.",
      grading: "A PoC passes when it crashes the vulnerable build under the expected sanitizer signal and does not crash the patched build. There is no partial credit or LLM judge.",
      why: "Large-scale, objective evidence for bug reproduction and vulnerability analysis.",
      caveat: "Retired from the frontier headline role because it stops at reproduction; ExploitGym tests the harder conversion from crash to unauthorized code execution.",
      reported_by: ["UC Berkeley RDI", "Anthropic"],
      references: [
        { title: "CyberGym paper", url: "https://arxiv.org/abs/2506.02548" },
        { title: "UC Berkeley RDI overview", url: "https://rdi.berkeley.edu/blog/cybergym/" }
      ]
    },
    {
      slug: "internal-ctf",
      title: "Internal CTF",
      status: "Retired",
      family: "Private CTF",
      metric: "Flag success",
      summary: "Vendor-curated private CTF suites previously used to avoid public-set contamination.",
      scale: "Vendor-specific · non-public",
      task: "Solve a private mixture of advanced web, reverse-engineering, binary and network exploitation, cryptography, and miscellaneous security challenges in a tool-enabled Linux environment.",
      grading: "Challenge flags or deterministic task verifiers provide binary success; vendors may report aggregate pass rates.",
      why: "Private tasks reduce direct writeup contamination and historically supplemented public CTF benchmarks.",
      caveat: "The set is not public or standardized, so results cannot be independently reproduced or compared across vendors. Superseded by more realistic exploit and reverse-engineering evaluations in frontier cards.",
      reported_by: ["OpenAI"],
      references: [
        { title: "OpenAI Deployment Safety Hub", url: "https://deploymentsafety.openai.com/" }
      ]
    },
    {
      slug: "vulnlmp",
      title: "VulnLMP",
      status: "Retired",
      family: "Long-horizon research",
      metric: "Verified research outcome",
      summary: "OpenAI’s internal long-horizon evaluation for vulnerability research on hardened, widely deployed software.",
      scale: "Private targets · multi-day campaigns",
      task: "Choose attack surfaces, develop target-specific tooling, reject misleading crashes, reduce and reproduce candidate bugs, write root-cause analyses, and attempt to convert findings into security-relevant primitives.",
      grading: "Supervised campaigns are assessed through real PoCs, reproducible crashes, root-cause evidence, controlled primitives, and verifier-confirmed exploit outcomes where achieved.",
      why: "Captured open-ended research behavior that short CTFs miss and provided evidence for High capability.",
      caveat: "Retired from the supplied frontier-card taxonomy in favor of standardized ExploitBench, ExploitGym, SRE-Bench, and fresh zero-day labs. It remains an internal methodology rather than a reproducible public benchmark.",
      reported_by: ["OpenAI"],
      references: [
        { title: "OpenAI GPT-5.5 System Card: VulnLMP", url: "https://deploymentsafety.openai.com/gpt-5-5/vulnlmp" }
      ]
    }
  ]
};