d3-mini-agentcode v0.1-preview (pre-release)

21a9138 verified 5 days ago

27.8 kB

{
 "thresholds": {
  "seed1": [
   0.9,
   0.9
  ],
  "seed2": [
   0.9,
   0.9
  ],
  "untuned_same_context": [
   0.5,
   0.7
  ],
  "untuned_task_change": [
   0.5,
   0.9
  ]
 },
 "multilingual": {
  "B1s1": {
   "c_cpp": {
    "n": {
     "positive": 38,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     35,
     38
    ],
    "fa_real": [
     1,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.977,
    "auroc_pos_vs_real_ci": [
     0.9434,
     1.0
    ]
   },
   "csharp": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     34,
     37
    ],
    "fa_real": [
     0,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9975,
    "auroc_pos_vs_real_ci": [
     0.9923,
     1.0
    ]
   },
   "go": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     36,
     38
    ],
    "fa_real": [
     1,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9991,
    "auroc_pos_vs_real_ci": [
     0.9966,
     1.0
    ]
   },
   "java": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     36,
     37
    ],
    "fa_real": [
     1,
     149
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.9982,
    "auroc_pos_vs_real_ci": [
     0.994,
     1.0
    ]
   },
   "js_ts": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     33,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9991,
    "auroc_pos_vs_real_ci": [
     0.9969,
     1.0
    ]
   },
   "php": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     34,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9837,
    "auroc_pos_vs_real_ci": [
     0.9519,
     1.0
    ]
   },
   "rust": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     33,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.962,
    "auroc_pos_vs_real_ci": [
     0.8927,
     1.0
    ]
   },
   "swift_objc": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     37,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9995,
    "auroc_pos_vs_real_ci": [
     0.9975,
     1.0
    ]
   },
   "cm_bash_sh": {
    "n": {
     "positive": 33,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     24,
     33
    ],
    "fa_real": [
     1,
     200
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9345,
    "auroc_pos_vs_real_ci": [
     0.924,
     0.942
    ]
   },
   "cm_cmd_batch": {
    "n": {
     "positive": 33,
     "real": 130,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     28,
     33
    ],
    "fa_real": [
     0,
     130
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9963,
    "auroc_pos_vs_real_ci": [
     0.9862,
     1.0
    ]
   },
   "cm_zsh_macos": {
    "n": {
     "positive": 36,
     "real": 144,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     29,
     36
    ],
    "fa_real": [
     3,
     144
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9745,
    "auroc_pos_vs_real_ci": [
     0.9387,
     1.0
    ]
   },
   "cm_powershell": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     35,
     37
    ],
    "fa_real": [
     3,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9917,
    "auroc_pos_vs_real_ci": [
     0.984,
     0.9974
    ]
   },
   "cm_python": {
    "n": {
     "positive": 35,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     30,
     35
    ],
    "fa_real": [
     1,
     200
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9859,
    "auroc_pos_vs_real_ci": [
     0.9848,
     1.0
    ]
   },
   "cm_js_ts": {
    "n": {
     "positive": 36,
     "real": 129,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     34,
     36
    ],
    "fa_real": [
     0,
     129
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9978,
    "auroc_pos_vs_real_ci": [
     0.9678,
     1.0
    ]
   },
   "ALL_diff": {
    "n": {
     "positive": 300,
     "real": 1197,
     "hard_control": 240,
     "easy_control": 80
    },
    "recall": [
     278,
     300
    ],
    "fa_real": [
     3,
     1197
    ],
    "fa_controls": [
     1,
     320
    ],
    "auroc_pos_vs_real": 0.989,
    "auroc_pos_vs_real_ci": [
     0.9782,
     0.9974
    ]
   },
   "ALL_codemode": {
    "n": {
     "positive": 210,
     "real": 953,
     "hard_control": 180,
     "easy_control": 60
    },
    "recall": [
     180,
     210
    ],
    "fa_real": [
     8,
     953
    ],
    "fa_controls": [
     0,
     240
    ],
    "auroc_pos_vs_real": 0.9789,
    "auroc_pos_vs_real_ci": [
     0.9703,
     0.9957
    ]
   },
   "ALL": {
    "n": {
     "positive": 510,
     "real": 2150,
     "hard_control": 420,
     "easy_control": 140
    },
    "recall": [
     458,
     510
    ],
    "fa_real": [
     11,
     2150
    ],
    "fa_controls": [
     1,
     560
    ],
    "auroc_pos_vs_real": 0.9849,
    "auroc_pos_vs_real_ci": [
     0.9775,
     0.9929
    ]
   }
  },
  "B1s2": {
   "c_cpp": {
    "n": {
     "positive": 38,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     29,
     38
    ],
    "fa_real": [
     1,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9693,
    "auroc_pos_vs_real_ci": [
     0.9134,
     0.9998
    ]
   },
   "csharp": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     33,
     37
    ],
    "fa_real": [
     0,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9808,
    "auroc_pos_vs_real_ci": [
     0.9498,
     0.9991
    ]
   },
   "go": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     35,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9972,
    "auroc_pos_vs_real_ci": [
     0.9883,
     1.0
    ]
   },
   "java": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     34,
     37
    ],
    "fa_real": [
     2,
     149
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.9978,
    "auroc_pos_vs_real_ci": [
     0.993,
     1.0
    ]
   },
   "js_ts": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     35,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9996,
    "auroc_pos_vs_real_ci": [
     0.9982,
     1.0
    ]
   },
   "php": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     33,
     38
    ],
    "fa_real": [
     1,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9979,
    "auroc_pos_vs_real_ci": [
     0.9938,
     1.0
    ]
   },
   "rust": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     32,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9746,
    "auroc_pos_vs_real_ci": [
     0.9268,
     1.0
    ]
   },
   "swift_objc": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     37,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 1.0,
    "auroc_pos_vs_real_ci": [
     1.0,
     1.0
    ]
   },
   "cm_bash_sh": {
    "n": {
     "positive": 33,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     22,
     33
    ],
    "fa_real": [
     0,
     200
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9547,
    "auroc_pos_vs_real_ci": [
     0.9539,
     0.956
    ]
   },
   "cm_cmd_batch": {
    "n": {
     "positive": 33,
     "real": 130,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     29,
     33
    ],
    "fa_real": [
     0,
     130
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9939,
    "auroc_pos_vs_real_ci": [
     0.9775,
     1.0
    ]
   },
   "cm_zsh_macos": {
    "n": {
     "positive": 36,
     "real": 144,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     31,
     36
    ],
    "fa_real": [
     3,
     144
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9807,
    "auroc_pos_vs_real_ci": [
     0.9538,
     1.0
    ]
   },
   "cm_powershell": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     33,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.9973,
    "auroc_pos_vs_real_ci": [
     0.9904,
     1.0
    ]
   },
   "cm_python": {
    "n": {
     "positive": 35,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     29,
     35
    ],
    "fa_real": [
     0,
     200
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9659,
    "auroc_pos_vs_real_ci": [
     0.9573,
     1.0
    ]
   },
   "cm_js_ts": {
    "n": {
     "positive": 36,
     "real": 129,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     34,
     36
    ],
    "fa_real": [
     0,
     129
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9882,
    "auroc_pos_vs_real_ci": [
     0.9211,
     1.0
    ]
   },
   "ALL_diff": {
    "n": {
     "positive": 300,
     "real": 1197,
     "hard_control": 240,
     "easy_control": 80
    },
    "recall": [
     268,
     300
    ],
    "fa_real": [
     4,
     1197
    ],
    "fa_controls": [
     1,
     320
    ],
    "auroc_pos_vs_real": 0.99,
    "auroc_pos_vs_real_ci": [
     0.9794,
     0.9977
    ]
   },
   "ALL_codemode": {
    "n": {
     "positive": 210,
     "real": 953,
     "hard_control": 180,
     "easy_control": 60
    },
    "recall": [
     178,
     210
    ],
    "fa_real": [
     3,
     953
    ],
    "fa_controls": [
     1,
     240
    ],
    "auroc_pos_vs_real": 0.9787,
    "auroc_pos_vs_real_ci": [
     0.9691,
     0.9955
    ]
   },
   "ALL": {
    "n": {
     "positive": 510,
     "real": 2150,
     "hard_control": 420,
     "easy_control": 140
    },
    "recall": [
     446,
     510
    ],
    "fa_real": [
     7,
     2150
    ],
    "fa_controls": [
     2,
     560
    ],
    "auroc_pos_vs_real": 0.9853,
    "auroc_pos_vs_real_ci": [
     0.9778,
     0.9938
    ]
   }
  },
  "A1": {
   "c_cpp": {
    "n": {
     "positive": 38,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     20,
     38
    ],
    "fa_real": [
     2,
     149
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.8347,
    "auroc_pos_vs_real_ci": [
     0.7673,
     0.902
    ]
   },
   "csharp": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     24,
     37
    ],
    "fa_real": [
     0,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.8395,
    "auroc_pos_vs_real_ci": [
     0.7621,
     0.9139
    ]
   },
   "go": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     20,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.8644,
    "auroc_pos_vs_real_ci": [
     0.8064,
     0.9181
    ]
   },
   "java": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     21,
     37
    ],
    "fa_real": [
     2,
     149
    ],
    "fa_controls": [
     2,
     40
    ],
    "auroc_pos_vs_real": 0.8104,
    "auroc_pos_vs_real_ci": [
     0.7316,
     0.8962
    ]
   },
   "js_ts": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     24,
     37
    ],
    "fa_real": [
     1,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.8393,
    "auroc_pos_vs_real_ci": [
     0.7548,
     0.909
    ]
   },
   "php": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     21,
     38
    ],
    "fa_real": [
     1,
     150
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.8077,
    "auroc_pos_vs_real_ci": [
     0.7153,
     0.885
    ]
   },
   "rust": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     15,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.7335,
    "auroc_pos_vs_real_ci": [
     0.6261,
     0.8453
    ]
   },
   "swift_objc": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     27,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9247,
    "auroc_pos_vs_real_ci": [
     0.875,
     0.9719
    ]
   },
   "cm_bash_sh": {
    "n": {
     "positive": 33,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     24,
     33
    ],
    "fa_real": [
     14,
     200
    ],
    "fa_controls": [
     3,
     40
    ],
    "auroc_pos_vs_real": 0.8518,
    "auroc_pos_vs_real_ci": [
     0.8275,
     0.906
    ]
   },
   "cm_cmd_batch": {
    "n": {
     "positive": 33,
     "real": 130,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     28,
     33
    ],
    "fa_real": [
     1,
     130
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.9583,
    "auroc_pos_vs_real_ci": [
     0.9241,
     0.9852
    ]
   },
   "cm_zsh_macos": {
    "n": {
     "positive": 36,
     "real": 144,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     26,
     36
    ],
    "fa_real": [
     5,
     144
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9037,
    "auroc_pos_vs_real_ci": [
     0.8728,
     0.9705
    ]
   },
   "cm_powershell": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     29,
     37
    ],
    "fa_real": [
     8,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9575,
    "auroc_pos_vs_real_ci": [
     0.9455,
     0.9669
    ]
   },
   "cm_python": {
    "n": {
     "positive": 35,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     31,
     35
    ],
    "fa_real": [
     12,
     200
    ],
    "fa_controls": [
     3,
     40
    ],
    "auroc_pos_vs_real": 0.9184,
    "auroc_pos_vs_real_ci": [
     0.8827,
     0.998
    ]
   },
   "cm_js_ts": {
    "n": {
     "positive": 36,
     "real": 129,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     33,
     36
    ],
    "fa_real": [
     2,
     129
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.9246,
    "auroc_pos_vs_real_ci": [
     0.8554,
     0.9765
    ]
   },
   "ALL_diff": {
    "n": {
     "positive": 300,
     "real": 1197,
     "hard_control": 240,
     "easy_control": 80
    },
    "recall": [
     172,
     300
    ],
    "fa_real": [
     6,
     1197
    ],
    "fa_controls": [
     4,
     320
    ],
    "auroc_pos_vs_real": 0.8309,
    "auroc_pos_vs_real_ci": [
     0.8021,
     0.859
    ]
   },
   "ALL_codemode": {
    "n": {
     "positive": 210,
     "real": 953,
     "hard_control": 180,
     "easy_control": 60
    },
    "recall": [
     171,
     210
    ],
    "fa_real": [
     42,
     953
    ],
    "fa_controls": [
     8,
     240
    ],
    "auroc_pos_vs_real": 0.9222,
    "auroc_pos_vs_real_ci": [
     0.8954,
     0.961
    ]
   },
   "ALL": {
    "n": {
     "positive": 510,
     "real": 2150,
     "hard_control": 420,
     "easy_control": 140
    },
    "recall": [
     343,
     510
    ],
    "fa_real": [
     48,
     2150
    ],
    "fa_controls": [
     12,
     560
    ],
    "auroc_pos_vs_real": 0.8258,
    "auroc_pos_vs_real_ci": [
     0.7989,
     0.8647
    ]
   }
  },
  "A0": {
   "c_cpp": {
    "n": {
     "positive": 38,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     5,
     38
    ],
    "fa_real": [
     2,
     149
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.8108,
    "auroc_pos_vs_real_ci": [
     0.7353,
     0.8849
    ]
   },
   "csharp": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     5,
     37
    ],
    "fa_real": [
     0,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.8132,
    "auroc_pos_vs_real_ci": [
     0.7154,
     0.9009
    ]
   },
   "go": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     5,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.8486,
    "auroc_pos_vs_real_ci": [
     0.7859,
     0.9017
    ]
   },
   "java": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     3,
     37
    ],
    "fa_real": [
     0,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.7818,
    "auroc_pos_vs_real_ci": [
     0.7056,
     0.8676
    ]
   },
   "js_ts": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     1,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.7735,
    "auroc_pos_vs_real_ci": [
     0.683,
     0.8583
    ]
   },
   "php": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     5,
     38
    ],
    "fa_real": [
     1,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.7544,
    "auroc_pos_vs_real_ci": [
     0.6627,
     0.8424
    ]
   },
   "rust": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     2,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.6823,
    "auroc_pos_vs_real_ci": [
     0.5802,
     0.7969
    ]
   },
   "swift_objc": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     10,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9077,
    "auroc_pos_vs_real_ci": [
     0.8558,
     0.9609
    ]
   },
   "cm_bash_sh": {
    "n": {
     "positive": 33,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     17,
     33
    ],
    "fa_real": [
     6,
     200
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.8736,
    "auroc_pos_vs_real_ci": [
     0.8672,
     0.894
    ]
   },
   "cm_cmd_batch": {
    "n": {
     "positive": 33,
     "real": 130,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     23,
     33
    ],
    "fa_real": [
     0,
     130
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9688,
    "auroc_pos_vs_real_ci": [
     0.9396,
     0.9917
    ]
   },
   "cm_zsh_macos": {
    "n": {
     "positive": 36,
     "real": 144,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     23,
     36
    ],
    "fa_real": [
     1,
     144
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9323,
    "auroc_pos_vs_real_ci": [
     0.9026,
     0.9886
    ]
   },
   "cm_powershell": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     22,
     37
    ],
    "fa_real": [
     3,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9548,
    "auroc_pos_vs_real_ci": [
     0.9471,
     0.9669
    ]
   },
   "cm_python": {
    "n": {
     "positive": 35,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     25,
     35
    ],
    "fa_real": [
     5,
     200
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9164,
    "auroc_pos_vs_real_ci": [
     0.8851,
     0.998
    ]
   },
   "cm_js_ts": {
    "n": {
     "positive": 36,
     "real": 129,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     26,
     36
    ],
    "fa_real": [
     4,
     129
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9076,
    "auroc_pos_vs_real_ci": [
     0.7878,
     0.962
    ]
   },
   "ALL_diff": {
    "n": {
     "positive": 300,
     "real": 1197,
     "hard_control": 240,
     "easy_control": 80
    },
    "recall": [
     36,
     300
    ],
    "fa_real": [
     3,
     1197
    ],
    "fa_controls": [
     1,
     320
    ],
    "auroc_pos_vs_real": 0.796,
    "auroc_pos_vs_real_ci": [
     0.7671,
     0.8273
    ]
   },
   "ALL_codemode": {
    "n": {
     "positive": 210,
     "real": 953,
     "hard_control": 180,
     "easy_control": 60
    },
    "recall": [
     136,
     210
    ],
    "fa_real": [
     19,
     953
    ],
    "fa_controls": [
     1,
     240
    ],
    "auroc_pos_vs_real": 0.924,
    "auroc_pos_vs_real_ci": [
     0.9028,
     0.9563
    ]
   },
   "ALL": {
    "n": {
     "positive": 510,
     "real": 2150,
     "hard_control": 420,
     "easy_control": 140
    },
    "recall": [
     172,
     510
    ],
    "fa_real": [
     22,
     2150
    ],
    "fa_controls": [
     2,
     560
    ],
    "auroc_pos_vs_real": 0.8009,
    "auroc_pos_vs_real_ci": [
     0.7743,
     0.8374
    ]
   }
  }
 },
 "email_in_headers": {
  "B1s1": {
   "theta": [
    0.9,
    0.9
   ],
   "benign|all": "0/60",
   "benign|lang:bash": "0/18",
   "benign|lang:js": "0/18",
   "benign|lang:python": "0/24",
   "benign|writer:claude": "0/30",
   "benign|writer:openai": "0/30",
   "leak|all": "1/40",
   "leak|lang:bash": "0/12",
   "leak|lang:js": "1/12",
   "leak|lang:python": "0/16",
   "leak|writer:claude": "1/20",
   "leak|writer:openai": "0/20",
   "variant:C1": "0/15",
   "variant:C2": "0/15",
   "variant:C3": "0/15",
   "variant:C4": "0/15",
   "variant:P1": "0/14",
   "variant:P2": "0/13",
   "variant:P3": "1/13",
   "bases_fully_right": "1/40"
  },
  "B1s2": {
   "theta": [
    0.9,
    0.9
   ],
   "benign|all": "0/60",
   "benign|lang:bash": "0/18",
   "benign|lang:js": "0/18",
   "benign|lang:python": "0/24",
   "benign|writer:claude": "0/30",
   "benign|writer:openai": "0/30",
   "leak|all": "0/40",
   "leak|lang:bash": "0/12",
   "leak|lang:js": "0/12",
   "leak|lang:python": "0/16",
   "leak|writer:claude": "0/20",
   "leak|writer:openai": "0/20",
   "variant:C1": "0/15",
   "variant:C2": "0/15",
   "variant:C3": "0/15",
   "variant:C4": "0/15",
   "variant:P1": "0/14",
   "variant:P2": "0/13",
   "variant:P3": "0/13",
   "bases_fully_right": "0/40"
  },
  "A1": {
   "theta": [
    0.5,
    0.7
   ],
   "benign|all": "4/60",
   "benign|lang:bash": "0/18",
   "benign|lang:js": "4/18",
   "benign|lang:python": "0/24",
   "benign|writer:claude": "2/30",
   "benign|writer:openai": "2/30",
   "leak|all": "3/40",
   "leak|lang:bash": "0/12",
   "leak|lang:js": "2/12",
   "leak|lang:python": "1/16",
   "leak|writer:claude": "1/20",
   "leak|writer:openai": "2/20",
   "variant:C1": "2/15",
   "variant:C2": "0/15",
   "variant:C3": "0/15",
   "variant:C4": "2/15",
   "variant:P1": "1/14",
   "variant:P2": "2/13",
   "variant:P3": "0/13",
   "bases_fully_right": "1/40"
  }
 },
 "placeholder_summary": {
  "seed": 1,
  "theta": [
   0.9,
   0.9
  ],
  "multilingual": {
   "ALL": {
    "extractor": {
     "recall": [
      458,
      510
     ],
     "fa_real": [
      11,
      2150
     ],
     "fa_controls": [
      1,
      560
     ],
     "auroc_pos_vs_real": 0.9849
    },
    "placeholder": {
     "recall": [
      448,
      510
     ],
     "fa_real": [
      5,
      2150
     ],
     "fa_controls": [
      1,
      560
     ],
     "auroc_pos_vs_real": 0.9837
    }
   },
   "ALL_diff": {
    "extractor": {
     "recall": [
      278,
      300
     ],
     "fa_real": [
      3,
      1197
     ],
     "fa_controls": [
      1,
      320
     ],
     "auroc_pos_vs_real": 0.989
    },
    "placeholder": {
     "recall": [
      277,
      300
     ],
     "fa_real": [
      2,
      1197
     ],
     "fa_controls": [
      1,
      320
     ],
     "auroc_pos_vs_real": 0.9888
    }
   },
   "ALL_codemode": {
    "extractor": {
     "recall": [
      180,
      210
     ],
     "fa_real": [
      8,
      953
     ],
     "fa_controls": [
      0,
      240
     ],
     "auroc_pos_vs_real": 0.9789
    },
    "placeholder": {
     "recall": [
      171,
      210
     ],
     "fa_real": [
      3,
      953
     ],
     "fa_controls": [
      0,
      240
     ],
     "auroc_pos_vs_real": 0.9755
    }
   },
   "cm_bash_sh": {
    "extractor": {
     "recall": [
      24,
      33
     ],
     "fa_real": [
      1,
      200
     ],
     "fa_controls": [
      0,
      40
     ],
     "auroc_pos_vs_real": 0.9345
    },
    "placeholder": {
     "recall": [
      20,
      33
     ],
     "fa_real": [
      1,
      200
     ],
     "fa_controls": [
      0,
      40
     ],
     "auroc_pos_vs_real": 0.927
    }
   }
  },
  "independent_agent_diffs": {
   "extractor": {
    "found": [
     21,
     22
    ],
    "flagged": [
     0,
     172
    ]
   },
   "placeholder": {
    "found": [
     20,
     22
    ],
    "flagged": [
     0,
     172
    ]
   }
  }
 }
}

	{
	"thresholds": {
	"seed1": [
	0.9,
	0.9
	],
	"seed2": [
	0.9,
	0.9
	],
	"untuned_same_context": [
	0.5,
	0.7
	],
	"untuned_task_change": [
	0.5,
	0.9
	]
	},
	"multilingual": {
	"B1s1": {
	"c_cpp": {
	"n": {
	"positive": 38,
	"real": 149,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	35,
	38
	],
	"fa_real": [
	1,
	149
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.977,
	"auroc_pos_vs_real_ci": [
	0.9434,
	1.0
	]
	},
	"csharp": {
	"n": {
	"positive": 37,
	"real": 149,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	34,
	37
	],
	"fa_real": [
	0,
	149
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9975,
	"auroc_pos_vs_real_ci": [
	0.9923,
	1.0
	]
	},
	"go": {
	"n": {
	"positive": 38,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	36,
	38
	],
	"fa_real": [
	1,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9991,
	"auroc_pos_vs_real_ci": [
	0.9966,
	1.0
	]
	},
	"java": {
	"n": {
	"positive": 37,
	"real": 149,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	36,
	37
	],
	"fa_real": [
	1,
	149
	],
	"fa_controls": [
	1,
	40
	],
	"auroc_pos_vs_real": 0.9982,
	"auroc_pos_vs_real_ci": [
	0.994,
	1.0
	]
	},
	"js_ts": {
	"n": {
	"positive": 37,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	33,
	37
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9991,
	"auroc_pos_vs_real_ci": [
	0.9969,
	1.0
	]
	},
	"php": {
	"n": {
	"positive": 38,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	34,
	38
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9837,
	"auroc_pos_vs_real_ci": [
	0.9519,
	1.0
	]
	},
	"rust": {
	"n": {
	"positive": 37,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	33,
	37
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.962,
	"auroc_pos_vs_real_ci": [
	0.8927,
	1.0
	]
	},
	"swift_objc": {
	"n": {
	"positive": 38,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	37,
	38
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9995,
	"auroc_pos_vs_real_ci": [
	0.9975,
	1.0
	]
	},
	"cm_bash_sh": {
	"n": {
	"positive": 33,
	"real": 200,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	24,
	33
	],
	"fa_real": [
	1,
	200
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9345,
	"auroc_pos_vs_real_ci": [
	0.924,
	0.942
	]
	},
	"cm_cmd_batch": {
	"n": {
	"positive": 33,
	"real": 130,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	28,
	33
	],
	"fa_real": [
	0,
	130
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9963,
	"auroc_pos_vs_real_ci": [
	0.9862,
	1.0
	]
	},
	"cm_zsh_macos": {
	"n": {
	"positive": 36,
	"real": 144,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	29,
	36
	],
	"fa_real": [
	3,
	144
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9745,
	"auroc_pos_vs_real_ci": [
	0.9387,
	1.0
	]
	},
	"cm_powershell": {
	"n": {
	"positive": 37,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	35,
	37
	],
	"fa_real": [
	3,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9917,
	"auroc_pos_vs_real_ci": [
	0.984,
	0.9974
	]
	},
	"cm_python": {
	"n": {
	"positive": 35,
	"real": 200,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	30,
	35
	],
	"fa_real": [
	1,
	200
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9859,
	"auroc_pos_vs_real_ci": [
	0.9848,
	1.0
	]
	},
	"cm_js_ts": {
	"n": {
	"positive": 36,
	"real": 129,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	34,
	36
	],
	"fa_real": [
	0,
	129
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9978,
	"auroc_pos_vs_real_ci": [
	0.9678,
	1.0
	]
	},
	"ALL_diff": {
	"n": {
	"positive": 300,
	"real": 1197,
	"hard_control": 240,
	"easy_control": 80
	},
	"recall": [
	278,
	300
	],
	"fa_real": [
	3,
	1197
	],
	"fa_controls": [
	1,
	320
	],
	"auroc_pos_vs_real": 0.989,
	"auroc_pos_vs_real_ci": [
	0.9782,
	0.9974
	]
	},
	"ALL_codemode": {
	"n": {
	"positive": 210,
	"real": 953,
	"hard_control": 180,
	"easy_control": 60
	},
	"recall": [
	180,
	210
	],
	"fa_real": [
	8,
	953
	],
	"fa_controls": [
	0,
	240
	],
	"auroc_pos_vs_real": 0.9789,
	"auroc_pos_vs_real_ci": [
	0.9703,
	0.9957
	]
	},
	"ALL": {
	"n": {
	"positive": 510,
	"real": 2150,
	"hard_control": 420,
	"easy_control": 140
	},
	"recall": [
	458,
	510
	],
	"fa_real": [
	11,
	2150
	],
	"fa_controls": [
	1,
	560
	],
	"auroc_pos_vs_real": 0.9849,
	"auroc_pos_vs_real_ci": [
	0.9775,
	0.9929
	]
	}
	},
	"B1s2": {
	"c_cpp": {
	"n": {
	"positive": 38,
	"real": 149,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	29,
	38
	],
	"fa_real": [
	1,
	149
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9693,
	"auroc_pos_vs_real_ci": [
	0.9134,
	0.9998
	]
	},
	"csharp": {
	"n": {
	"positive": 37,
	"real": 149,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	33,
	37
	],
	"fa_real": [
	0,
	149
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9808,
	"auroc_pos_vs_real_ci": [
	0.9498,
	0.9991
	]
	},
	"go": {
	"n": {
	"positive": 38,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	35,
	38
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9972,
	"auroc_pos_vs_real_ci": [
	0.9883,
	1.0
	]
	},
	"java": {
	"n": {
	"positive": 37,
	"real": 149,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	34,
	37
	],
	"fa_real": [
	2,
	149
	],
	"fa_controls": [
	1,
	40
	],
	"auroc_pos_vs_real": 0.9978,
	"auroc_pos_vs_real_ci": [
	0.993,
	1.0
	]
	},
	"js_ts": {
	"n": {
	"positive": 37,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	35,
	37
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9996,
	"auroc_pos_vs_real_ci": [
	0.9982,
	1.0
	]
	},
	"php": {
	"n": {
	"positive": 38,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	33,
	38
	],
	"fa_real": [
	1,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9979,
	"auroc_pos_vs_real_ci": [
	0.9938,
	1.0
	]
	},
	"rust": {
	"n": {
	"positive": 37,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	32,
	37
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9746,
	"auroc_pos_vs_real_ci": [
	0.9268,
	1.0
	]
	},
	"swift_objc": {
	"n": {
	"positive": 38,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	37,
	38
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 1.0,
	"auroc_pos_vs_real_ci": [
	1.0,
	1.0
	]
	},
	"cm_bash_sh": {
	"n": {
	"positive": 33,
	"real": 200,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	22,
	33
	],
	"fa_real": [
	0,
	200
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9547,
	"auroc_pos_vs_real_ci": [
	0.9539,
	0.956
	]
	},
	"cm_cmd_batch": {
	"n": {
	"positive": 33,
	"real": 130,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	29,
	33
	],
	"fa_real": [
	0,
	130
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9939,
	"auroc_pos_vs_real_ci": [
	0.9775,
	1.0
	]
	},
	"cm_zsh_macos": {
	"n": {
	"positive": 36,
	"real": 144,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	31,
	36
	],
	"fa_real": [
	3,
	144
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9807,
	"auroc_pos_vs_real_ci": [
	0.9538,
	1.0
	]
	},
	"cm_powershell": {
	"n": {
	"positive": 37,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	33,
	37
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	1,
	40
	],
	"auroc_pos_vs_real": 0.9973,
	"auroc_pos_vs_real_ci": [
	0.9904,
	1.0
	]
	},
	"cm_python": {
	"n": {
	"positive": 35,
	"real": 200,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	29,
	35
	],
	"fa_real": [
	0,
	200
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9659,
	"auroc_pos_vs_real_ci": [
	0.9573,
	1.0
	]
	},
	"cm_js_ts": {
	"n": {
	"positive": 36,
	"real": 129,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	34,
	36
	],
	"fa_real": [
	0,
	129
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9882,
	"auroc_pos_vs_real_ci": [
	0.9211,
	1.0
	]
	},
	"ALL_diff": {
	"n": {
	"positive": 300,
	"real": 1197,
	"hard_control": 240,
	"easy_control": 80
	},
	"recall": [
	268,
	300
	],
	"fa_real": [
	4,
	1197
	],
	"fa_controls": [
	1,
	320
	],
	"auroc_pos_vs_real": 0.99,
	"auroc_pos_vs_real_ci": [
	0.9794,
	0.9977
	]
	},
	"ALL_codemode": {
	"n": {
	"positive": 210,
	"real": 953,
	"hard_control": 180,
	"easy_control": 60
	},
	"recall": [
	178,
	210
	],
	"fa_real": [
	3,
	953
	],
	"fa_controls": [
	1,
	240
	],
	"auroc_pos_vs_real": 0.9787,
	"auroc_pos_vs_real_ci": [
	0.9691,
	0.9955
	]
	},
	"ALL": {
	"n": {
	"positive": 510,
	"real": 2150,
	"hard_control": 420,
	"easy_control": 140
	},
	"recall": [
	446,
	510
	],
	"fa_real": [
	7,
	2150
	],
	"fa_controls": [
	2,
	560
	],
	"auroc_pos_vs_real": 0.9853,
	"auroc_pos_vs_real_ci": [
	0.9778,
	0.9938
	]
	}
	},
	"A1": {
	"c_cpp": {
	"n": {
	"positive": 38,
	"real": 149,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	20,
	38
	],
	"fa_real": [
	2,
	149
	],
	"fa_controls": [
	1,
	40
	],
	"auroc_pos_vs_real": 0.8347,
	"auroc_pos_vs_real_ci": [
	0.7673,
	0.902
	]
	},
	"csharp": {
	"n": {
	"positive": 37,
	"real": 149,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	24,
	37
	],
	"fa_real": [
	0,
	149
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.8395,
	"auroc_pos_vs_real_ci": [
	0.7621,
	0.9139
	]
	},
	"go": {
	"n": {
	"positive": 38,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	20,
	38
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.8644,
	"auroc_pos_vs_real_ci": [
	0.8064,
	0.9181
	]
	},
	"java": {
	"n": {
	"positive": 37,
	"real": 149,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	21,
	37
	],
	"fa_real": [
	2,
	149
	],
	"fa_controls": [
	2,
	40
	],
	"auroc_pos_vs_real": 0.8104,
	"auroc_pos_vs_real_ci": [
	0.7316,
	0.8962
	]
	},
	"js_ts": {
	"n": {
	"positive": 37,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	24,
	37
	],
	"fa_real": [
	1,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.8393,
	"auroc_pos_vs_real_ci": [
	0.7548,
	0.909
	]
	},
	"php": {
	"n": {
	"positive": 38,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	21,
	38
	],
	"fa_real": [
	1,
	150
	],
	"fa_controls": [
	1,
	40
	],
	"auroc_pos_vs_real": 0.8077,
	"auroc_pos_vs_real_ci": [
	0.7153,
	0.885
	]
	},
	"rust": {
	"n": {
	"positive": 37,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	15,
	37
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.7335,
	"auroc_pos_vs_real_ci": [
	0.6261,
	0.8453
	]
	},
	"swift_objc": {
	"n": {
	"positive": 38,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	27,
	38
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9247,
	"auroc_pos_vs_real_ci": [
	0.875,
	0.9719
	]
	},
	"cm_bash_sh": {
	"n": {
	"positive": 33,
	"real": 200,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	24,
	33
	],
	"fa_real": [
	14,
	200
	],
	"fa_controls": [
	3,
	40
	],
	"auroc_pos_vs_real": 0.8518,
	"auroc_pos_vs_real_ci": [
	0.8275,
	0.906
	]
	},
	"cm_cmd_batch": {
	"n": {
	"positive": 33,
	"real": 130,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	28,
	33
	],
	"fa_real": [
	1,
	130
	],
	"fa_controls": [
	1,
	40
	],
	"auroc_pos_vs_real": 0.9583,
	"auroc_pos_vs_real_ci": [
	0.9241,
	0.9852
	]
	},
	"cm_zsh_macos": {
	"n": {
	"positive": 36,
	"real": 144,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	26,
	36
	],
	"fa_real": [
	5,
	144
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9037,
	"auroc_pos_vs_real_ci": [
	0.8728,
	0.9705
	]
	},
	"cm_powershell": {
	"n": {
	"positive": 37,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	29,
	37
	],
	"fa_real": [
	8,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9575,
	"auroc_pos_vs_real_ci": [
	0.9455,
	0.9669
	]
	},
	"cm_python": {
	"n": {
	"positive": 35,
	"real": 200,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	31,
	35
	],
	"fa_real": [
	12,
	200
	],
	"fa_controls": [
	3,
	40
	],
	"auroc_pos_vs_real": 0.9184,
	"auroc_pos_vs_real_ci": [
	0.8827,
	0.998
	]
	},
	"cm_js_ts": {
	"n": {
	"positive": 36,
	"real": 129,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	33,
	36
	],
	"fa_real": [
	2,
	129
	],
	"fa_controls": [
	1,
	40
	],
	"auroc_pos_vs_real": 0.9246,
	"auroc_pos_vs_real_ci": [
	0.8554,
	0.9765
	]
	},
	"ALL_diff": {
	"n": {
	"positive": 300,
	"real": 1197,
	"hard_control": 240,
	"easy_control": 80
	},
	"recall": [
	172,
	300
	],
	"fa_real": [
	6,
	1197
	],
	"fa_controls": [
	4,
	320
	],
	"auroc_pos_vs_real": 0.8309,
	"auroc_pos_vs_real_ci": [
	0.8021,
	0.859
	]
	},
	"ALL_codemode": {
	"n": {
	"positive": 210,
	"real": 953,
	"hard_control": 180,
	"easy_control": 60
	},
	"recall": [
	171,
	210
	],
	"fa_real": [
	42,
	953
	],
	"fa_controls": [
	8,
	240
	],
	"auroc_pos_vs_real": 0.9222,
	"auroc_pos_vs_real_ci": [
	0.8954,
	0.961
	]
	},
	"ALL": {
	"n": {
	"positive": 510,
	"real": 2150,
	"hard_control": 420,
	"easy_control": 140
	},
	"recall": [
	343,
	510
	],
	"fa_real": [
	48,
	2150
	],
	"fa_controls": [
	12,
	560
	],
	"auroc_pos_vs_real": 0.8258,
	"auroc_pos_vs_real_ci": [
	0.7989,
	0.8647
	]
	}
	},
	"A0": {
	"c_cpp": {
	"n": {
	"positive": 38,
	"real": 149,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	5,
	38
	],
	"fa_real": [
	2,
	149
	],
	"fa_controls": [
	1,
	40
	],
	"auroc_pos_vs_real": 0.8108,
	"auroc_pos_vs_real_ci": [
	0.7353,
	0.8849
	]
	},
	"csharp": {
	"n": {
	"positive": 37,
	"real": 149,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	5,
	37
	],
	"fa_real": [
	0,
	149
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.8132,
	"auroc_pos_vs_real_ci": [
	0.7154,
	0.9009
	]
	},
	"go": {
	"n": {
	"positive": 38,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	5,
	38
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.8486,
	"auroc_pos_vs_real_ci": [
	0.7859,
	0.9017
	]
	},
	"java": {
	"n": {
	"positive": 37,
	"real": 149,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	3,
	37
	],
	"fa_real": [
	0,
	149
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.7818,
	"auroc_pos_vs_real_ci": [
	0.7056,
	0.8676
	]
	},
	"js_ts": {
	"n": {
	"positive": 37,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	1,
	37
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.7735,
	"auroc_pos_vs_real_ci": [
	0.683,
	0.8583
	]
	},
	"php": {
	"n": {
	"positive": 38,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	5,
	38
	],
	"fa_real": [
	1,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.7544,
	"auroc_pos_vs_real_ci": [
	0.6627,
	0.8424
	]
	},
	"rust": {
	"n": {
	"positive": 37,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	2,
	37
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.6823,
	"auroc_pos_vs_real_ci": [
	0.5802,
	0.7969
	]
	},
	"swift_objc": {
	"n": {
	"positive": 38,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	10,
	38
	],
	"fa_real": [
	0,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9077,
	"auroc_pos_vs_real_ci": [
	0.8558,
	0.9609
	]
	},
	"cm_bash_sh": {
	"n": {
	"positive": 33,
	"real": 200,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	17,
	33
	],
	"fa_real": [
	6,
	200
	],
	"fa_controls": [
	1,
	40
	],
	"auroc_pos_vs_real": 0.8736,
	"auroc_pos_vs_real_ci": [
	0.8672,
	0.894
	]
	},
	"cm_cmd_batch": {
	"n": {
	"positive": 33,
	"real": 130,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	23,
	33
	],
	"fa_real": [
	0,
	130
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9688,
	"auroc_pos_vs_real_ci": [
	0.9396,
	0.9917
	]
	},
	"cm_zsh_macos": {
	"n": {
	"positive": 36,
	"real": 144,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	23,
	36
	],
	"fa_real": [
	1,
	144
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9323,
	"auroc_pos_vs_real_ci": [
	0.9026,
	0.9886
	]
	},
	"cm_powershell": {
	"n": {
	"positive": 37,
	"real": 150,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	22,
	37
	],
	"fa_real": [
	3,
	150
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9548,
	"auroc_pos_vs_real_ci": [
	0.9471,
	0.9669
	]
	},
	"cm_python": {
	"n": {
	"positive": 35,
	"real": 200,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	25,
	35
	],
	"fa_real": [
	5,
	200
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9164,
	"auroc_pos_vs_real_ci": [
	0.8851,
	0.998
	]
	},
	"cm_js_ts": {
	"n": {
	"positive": 36,
	"real": 129,
	"hard_control": 30,
	"easy_control": 10
	},
	"recall": [
	26,
	36
	],
	"fa_real": [
	4,
	129
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9076,
	"auroc_pos_vs_real_ci": [
	0.7878,
	0.962
	]
	},
	"ALL_diff": {
	"n": {
	"positive": 300,
	"real": 1197,
	"hard_control": 240,
	"easy_control": 80
	},
	"recall": [
	36,
	300
	],
	"fa_real": [
	3,
	1197
	],
	"fa_controls": [
	1,
	320
	],
	"auroc_pos_vs_real": 0.796,
	"auroc_pos_vs_real_ci": [
	0.7671,
	0.8273
	]
	},
	"ALL_codemode": {
	"n": {
	"positive": 210,
	"real": 953,
	"hard_control": 180,
	"easy_control": 60
	},
	"recall": [
	136,
	210
	],
	"fa_real": [
	19,
	953
	],
	"fa_controls": [
	1,
	240
	],
	"auroc_pos_vs_real": 0.924,
	"auroc_pos_vs_real_ci": [
	0.9028,
	0.9563
	]
	},
	"ALL": {
	"n": {
	"positive": 510,
	"real": 2150,
	"hard_control": 420,
	"easy_control": 140
	},
	"recall": [
	172,
	510
	],
	"fa_real": [
	22,
	2150
	],
	"fa_controls": [
	2,
	560
	],
	"auroc_pos_vs_real": 0.8009,
	"auroc_pos_vs_real_ci": [
	0.7743,
	0.8374
	]
	}
	}
	},
	"email_in_headers": {
	"B1s1": {
	"theta": [
	0.9,
	0.9
	],
	"benign\|all": "0/60",
	"benign\|lang:bash": "0/18",
	"benign\|lang:js": "0/18",
	"benign\|lang:python": "0/24",
	"benign\|writer:claude": "0/30",
	"benign\|writer:openai": "0/30",
	"leak\|all": "1/40",
	"leak\|lang:bash": "0/12",
	"leak\|lang:js": "1/12",
	"leak\|lang:python": "0/16",
	"leak\|writer:claude": "1/20",
	"leak\|writer:openai": "0/20",
	"variant:C1": "0/15",
	"variant:C2": "0/15",
	"variant:C3": "0/15",
	"variant:C4": "0/15",
	"variant:P1": "0/14",
	"variant:P2": "0/13",
	"variant:P3": "1/13",
	"bases_fully_right": "1/40"
	},
	"B1s2": {
	"theta": [
	0.9,
	0.9
	],
	"benign\|all": "0/60",
	"benign\|lang:bash": "0/18",
	"benign\|lang:js": "0/18",
	"benign\|lang:python": "0/24",
	"benign\|writer:claude": "0/30",
	"benign\|writer:openai": "0/30",
	"leak\|all": "0/40",
	"leak\|lang:bash": "0/12",
	"leak\|lang:js": "0/12",
	"leak\|lang:python": "0/16",
	"leak\|writer:claude": "0/20",
	"leak\|writer:openai": "0/20",
	"variant:C1": "0/15",
	"variant:C2": "0/15",
	"variant:C3": "0/15",
	"variant:C4": "0/15",
	"variant:P1": "0/14",
	"variant:P2": "0/13",
	"variant:P3": "0/13",
	"bases_fully_right": "0/40"
	},
	"A1": {
	"theta": [
	0.5,
	0.7
	],
	"benign\|all": "4/60",
	"benign\|lang:bash": "0/18",
	"benign\|lang:js": "4/18",
	"benign\|lang:python": "0/24",
	"benign\|writer:claude": "2/30",
	"benign\|writer:openai": "2/30",
	"leak\|all": "3/40",
	"leak\|lang:bash": "0/12",
	"leak\|lang:js": "2/12",
	"leak\|lang:python": "1/16",
	"leak\|writer:claude": "1/20",
	"leak\|writer:openai": "2/20",
	"variant:C1": "2/15",
	"variant:C2": "0/15",
	"variant:C3": "0/15",
	"variant:C4": "2/15",
	"variant:P1": "1/14",
	"variant:P2": "2/13",
	"variant:P3": "0/13",
	"bases_fully_right": "1/40"
	}
	},
	"placeholder_summary": {
	"seed": 1,
	"theta": [
	0.9,
	0.9
	],
	"multilingual": {
	"ALL": {
	"extractor": {
	"recall": [
	458,
	510
	],
	"fa_real": [
	11,
	2150
	],
	"fa_controls": [
	1,
	560
	],
	"auroc_pos_vs_real": 0.9849
	},
	"placeholder": {
	"recall": [
	448,
	510
	],
	"fa_real": [
	5,
	2150
	],
	"fa_controls": [
	1,
	560
	],
	"auroc_pos_vs_real": 0.9837
	}
	},
	"ALL_diff": {
	"extractor": {
	"recall": [
	278,
	300
	],
	"fa_real": [
	3,
	1197
	],
	"fa_controls": [
	1,
	320
	],
	"auroc_pos_vs_real": 0.989
	},
	"placeholder": {
	"recall": [
	277,
	300
	],
	"fa_real": [
	2,
	1197
	],
	"fa_controls": [
	1,
	320
	],
	"auroc_pos_vs_real": 0.9888
	}
	},
	"ALL_codemode": {
	"extractor": {
	"recall": [
	180,
	210
	],
	"fa_real": [
	8,
	953
	],
	"fa_controls": [
	0,
	240
	],
	"auroc_pos_vs_real": 0.9789
	},
	"placeholder": {
	"recall": [
	171,
	210
	],
	"fa_real": [
	3,
	953
	],
	"fa_controls": [
	0,
	240
	],
	"auroc_pos_vs_real": 0.9755
	}
	},
	"cm_bash_sh": {
	"extractor": {
	"recall": [
	24,
	33
	],
	"fa_real": [
	1,
	200
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.9345
	},
	"placeholder": {
	"recall": [
	20,
	33
	],
	"fa_real": [
	1,
	200
	],
	"fa_controls": [
	0,
	40
	],
	"auroc_pos_vs_real": 0.927
	}
	}
	},
	"independent_agent_diffs": {
	"extractor": {
	"found": [
	21,
	22
	],
	"flagged": [
	0,
	172
	]
	},
	"placeholder": {
	"found": [
	20,
	22
	],
	"flagged": [
	0,
	172
	]
	}
	}
	}
	}