{
 "thresholds": {
  "seed1": [
   0.9,
   0.9
  ],
  "seed2": [
   0.9,
   0.9
  ],
  "untuned_same_context": [
   0.5,
   0.7
  ],
  "untuned_task_change": [
   0.5,
   0.9
  ]
 },
 "multilingual": {
  "B1s1": {
   "c_cpp": {
    "n": {
     "positive": 38,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     35,
     38
    ],
    "fa_real": [
     1,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.977,
    "auroc_pos_vs_real_ci": [
     0.9434,
     1.0
    ]
   },
   "csharp": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     34,
     37
    ],
    "fa_real": [
     0,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9975,
    "auroc_pos_vs_real_ci": [
     0.9923,
     1.0
    ]
   },
   "go": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     36,
     38
    ],
    "fa_real": [
     1,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9991,
    "auroc_pos_vs_real_ci": [
     0.9966,
     1.0
    ]
   },
   "java": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     36,
     37
    ],
    "fa_real": [
     1,
     149
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.9982,
    "auroc_pos_vs_real_ci": [
     0.994,
     1.0
    ]
   },
   "js_ts": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     33,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9991,
    "auroc_pos_vs_real_ci": [
     0.9969,
     1.0
    ]
   },
   "php": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     34,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9837,
    "auroc_pos_vs_real_ci": [
     0.9519,
     1.0
    ]
   },
   "rust": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     33,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.962,
    "auroc_pos_vs_real_ci": [
     0.8927,
     1.0
    ]
   },
   "swift_objc": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     37,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9995,
    "auroc_pos_vs_real_ci": [
     0.9975,
     1.0
    ]
   },
   "cm_bash_sh": {
    "n": {
     "positive": 33,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     24,
     33
    ],
    "fa_real": [
     1,
     200
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9345,
    "auroc_pos_vs_real_ci": [
     0.924,
     0.942
    ]
   },
   "cm_cmd_batch": {
    "n": {
     "positive": 33,
     "real": 130,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     28,
     33
    ],
    "fa_real": [
     0,
     130
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9963,
    "auroc_pos_vs_real_ci": [
     0.9862,
     1.0
    ]
   },
   "cm_zsh_macos": {
    "n": {
     "positive": 36,
     "real": 144,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     29,
     36
    ],
    "fa_real": [
     3,
     144
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9745,
    "auroc_pos_vs_real_ci": [
     0.9387,
     1.0
    ]
   },
   "cm_powershell": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     35,
     37
    ],
    "fa_real": [
     3,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9917,
    "auroc_pos_vs_real_ci": [
     0.984,
     0.9974
    ]
   },
   "cm_python": {
    "n": {
     "positive": 35,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     30,
     35
    ],
    "fa_real": [
     1,
     200
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9859,
    "auroc_pos_vs_real_ci": [
     0.9848,
     1.0
    ]
   },
   "cm_js_ts": {
    "n": {
     "positive": 36,
     "real": 129,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     34,
     36
    ],
    "fa_real": [
     0,
     129
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9978,
    "auroc_pos_vs_real_ci": [
     0.9678,
     1.0
    ]
   },
   "ALL_diff": {
    "n": {
     "positive": 300,
     "real": 1197,
     "hard_control": 240,
     "easy_control": 80
    },
    "recall": [
     278,
     300
    ],
    "fa_real": [
     3,
     1197
    ],
    "fa_controls": [
     1,
     320
    ],
    "auroc_pos_vs_real": 0.989,
    "auroc_pos_vs_real_ci": [
     0.9782,
     0.9974
    ]
   },
   "ALL_codemode": {
    "n": {
     "positive": 210,
     "real": 953,
     "hard_control": 180,
     "easy_control": 60
    },
    "recall": [
     180,
     210
    ],
    "fa_real": [
     8,
     953
    ],
    "fa_controls": [
     0,
     240
    ],
    "auroc_pos_vs_real": 0.9789,
    "auroc_pos_vs_real_ci": [
     0.9703,
     0.9957
    ]
   },
   "ALL": {
    "n": {
     "positive": 510,
     "real": 2150,
     "hard_control": 420,
     "easy_control": 140
    },
    "recall": [
     458,
     510
    ],
    "fa_real": [
     11,
     2150
    ],
    "fa_controls": [
     1,
     560
    ],
    "auroc_pos_vs_real": 0.9849,
    "auroc_pos_vs_real_ci": [
     0.9775,
     0.9929
    ]
   }
  },
  "B1s2": {
   "c_cpp": {
    "n": {
     "positive": 38,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     29,
     38
    ],
    "fa_real": [
     1,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9693,
    "auroc_pos_vs_real_ci": [
     0.9134,
     0.9998
    ]
   },
   "csharp": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     33,
     37
    ],
    "fa_real": [
     0,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9808,
    "auroc_pos_vs_real_ci": [
     0.9498,
     0.9991
    ]
   },
   "go": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     35,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9972,
    "auroc_pos_vs_real_ci": [
     0.9883,
     1.0
    ]
   },
   "java": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     34,
     37
    ],
    "fa_real": [
     2,
     149
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.9978,
    "auroc_pos_vs_real_ci": [
     0.993,
     1.0
    ]
   },
   "js_ts": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     35,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9996,
    "auroc_pos_vs_real_ci": [
     0.9982,
     1.0
    ]
   },
   "php": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     33,
     38
    ],
    "fa_real": [
     1,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9979,
    "auroc_pos_vs_real_ci": [
     0.9938,
     1.0
    ]
   },
   "rust": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     32,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9746,
    "auroc_pos_vs_real_ci": [
     0.9268,
     1.0
    ]
   },
   "swift_objc": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     37,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 1.0,
    "auroc_pos_vs_real_ci": [
     1.0,
     1.0
    ]
   },
   "cm_bash_sh": {
    "n": {
     "positive": 33,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     22,
     33
    ],
    "fa_real": [
     0,
     200
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9547,
    "auroc_pos_vs_real_ci": [
     0.9539,
     0.956
    ]
   },
   "cm_cmd_batch": {
    "n": {
     "positive": 33,
     "real": 130,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     29,
     33
    ],
    "fa_real": [
     0,
     130
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9939,
    "auroc_pos_vs_real_ci": [
     0.9775,
     1.0
    ]
   },
   "cm_zsh_macos": {
    "n": {
     "positive": 36,
     "real": 144,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     31,
     36
    ],
    "fa_real": [
     3,
     144
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9807,
    "auroc_pos_vs_real_ci": [
     0.9538,
     1.0
    ]
   },
   "cm_powershell": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     33,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.9973,
    "auroc_pos_vs_real_ci": [
     0.9904,
     1.0
    ]
   },
   "cm_python": {
    "n": {
     "positive": 35,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     29,
     35
    ],
    "fa_real": [
     0,
     200
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9659,
    "auroc_pos_vs_real_ci": [
     0.9573,
     1.0
    ]
   },
   "cm_js_ts": {
    "n": {
     "positive": 36,
     "real": 129,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     34,
     36
    ],
    "fa_real": [
     0,
     129
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9882,
    "auroc_pos_vs_real_ci": [
     0.9211,
     1.0
    ]
   },
   "ALL_diff": {
    "n": {
     "positive": 300,
     "real": 1197,
     "hard_control": 240,
     "easy_control": 80
    },
    "recall": [
     268,
     300
    ],
    "fa_real": [
     4,
     1197
    ],
    "fa_controls": [
     1,
     320
    ],
    "auroc_pos_vs_real": 0.99,
    "auroc_pos_vs_real_ci": [
     0.9794,
     0.9977
    ]
   },
   "ALL_codemode": {
    "n": {
     "positive": 210,
     "real": 953,
     "hard_control": 180,
     "easy_control": 60
    },
    "recall": [
     178,
     210
    ],
    "fa_real": [
     3,
     953
    ],
    "fa_controls": [
     1,
     240
    ],
    "auroc_pos_vs_real": 0.9787,
    "auroc_pos_vs_real_ci": [
     0.9691,
     0.9955
    ]
   },
   "ALL": {
    "n": {
     "positive": 510,
     "real": 2150,
     "hard_control": 420,
     "easy_control": 140
    },
    "recall": [
     446,
     510
    ],
    "fa_real": [
     7,
     2150
    ],
    "fa_controls": [
     2,
     560
    ],
    "auroc_pos_vs_real": 0.9853,
    "auroc_pos_vs_real_ci": [
     0.9778,
     0.9938
    ]
   }
  },
  "A1": {
   "c_cpp": {
    "n": {
     "positive": 38,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     20,
     38
    ],
    "fa_real": [
     2,
     149
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.8347,
    "auroc_pos_vs_real_ci": [
     0.7673,
     0.902
    ]
   },
   "csharp": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     24,
     37
    ],
    "fa_real": [
     0,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.8395,
    "auroc_pos_vs_real_ci": [
     0.7621,
     0.9139
    ]
   },
   "go": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     20,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.8644,
    "auroc_pos_vs_real_ci": [
     0.8064,
     0.9181
    ]
   },
   "java": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     21,
     37
    ],
    "fa_real": [
     2,
     149
    ],
    "fa_controls": [
     2,
     40
    ],
    "auroc_pos_vs_real": 0.8104,
    "auroc_pos_vs_real_ci": [
     0.7316,
     0.8962
    ]
   },
   "js_ts": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     24,
     37
    ],
    "fa_real": [
     1,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.8393,
    "auroc_pos_vs_real_ci": [
     0.7548,
     0.909
    ]
   },
   "php": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     21,
     38
    ],
    "fa_real": [
     1,
     150
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.8077,
    "auroc_pos_vs_real_ci": [
     0.7153,
     0.885
    ]
   },
   "rust": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     15,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.7335,
    "auroc_pos_vs_real_ci": [
     0.6261,
     0.8453
    ]
   },
   "swift_objc": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     27,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9247,
    "auroc_pos_vs_real_ci": [
     0.875,
     0.9719
    ]
   },
   "cm_bash_sh": {
    "n": {
     "positive": 33,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     24,
     33
    ],
    "fa_real": [
     14,
     200
    ],
    "fa_controls": [
     3,
     40
    ],
    "auroc_pos_vs_real": 0.8518,
    "auroc_pos_vs_real_ci": [
     0.8275,
     0.906
    ]
   },
   "cm_cmd_batch": {
    "n": {
     "positive": 33,
     "real": 130,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     28,
     33
    ],
    "fa_real": [
     1,
     130
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.9583,
    "auroc_pos_vs_real_ci": [
     0.9241,
     0.9852
    ]
   },
   "cm_zsh_macos": {
    "n": {
     "positive": 36,
     "real": 144,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     26,
     36
    ],
    "fa_real": [
     5,
     144
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9037,
    "auroc_pos_vs_real_ci": [
     0.8728,
     0.9705
    ]
   },
   "cm_powershell": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     29,
     37
    ],
    "fa_real": [
     8,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9575,
    "auroc_pos_vs_real_ci": [
     0.9455,
     0.9669
    ]
   },
   "cm_python": {
    "n": {
     "positive": 35,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     31,
     35
    ],
    "fa_real": [
     12,
     200
    ],
    "fa_controls": [
     3,
     40
    ],
    "auroc_pos_vs_real": 0.9184,
    "auroc_pos_vs_real_ci": [
     0.8827,
     0.998
    ]
   },
   "cm_js_ts": {
    "n": {
     "positive": 36,
     "real": 129,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     33,
     36
    ],
    "fa_real": [
     2,
     129
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.9246,
    "auroc_pos_vs_real_ci": [
     0.8554,
     0.9765
    ]
   },
   "ALL_diff": {
    "n": {
     "positive": 300,
     "real": 1197,
     "hard_control": 240,
     "easy_control": 80
    },
    "recall": [
     172,
     300
    ],
    "fa_real": [
     6,
     1197
    ],
    "fa_controls": [
     4,
     320
    ],
    "auroc_pos_vs_real": 0.8309,
    "auroc_pos_vs_real_ci": [
     0.8021,
     0.859
    ]
   },
   "ALL_codemode": {
    "n": {
     "positive": 210,
     "real": 953,
     "hard_control": 180,
     "easy_control": 60
    },
    "recall": [
     171,
     210
    ],
    "fa_real": [
     42,
     953
    ],
    "fa_controls": [
     8,
     240
    ],
    "auroc_pos_vs_real": 0.9222,
    "auroc_pos_vs_real_ci": [
     0.8954,
     0.961
    ]
   },
   "ALL": {
    "n": {
     "positive": 510,
     "real": 2150,
     "hard_control": 420,
     "easy_control": 140
    },
    "recall": [
     343,
     510
    ],
    "fa_real": [
     48,
     2150
    ],
    "fa_controls": [
     12,
     560
    ],
    "auroc_pos_vs_real": 0.8258,
    "auroc_pos_vs_real_ci": [
     0.7989,
     0.8647
    ]
   }
  },
  "A0": {
   "c_cpp": {
    "n": {
     "positive": 38,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     5,
     38
    ],
    "fa_real": [
     2,
     149
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.8108,
    "auroc_pos_vs_real_ci": [
     0.7353,
     0.8849
    ]
   },
   "csharp": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     5,
     37
    ],
    "fa_real": [
     0,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.8132,
    "auroc_pos_vs_real_ci": [
     0.7154,
     0.9009
    ]
   },
   "go": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     5,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.8486,
    "auroc_pos_vs_real_ci": [
     0.7859,
     0.9017
    ]
   },
   "java": {
    "n": {
     "positive": 37,
     "real": 149,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     3,
     37
    ],
    "fa_real": [
     0,
     149
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.7818,
    "auroc_pos_vs_real_ci": [
     0.7056,
     0.8676
    ]
   },
   "js_ts": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     1,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.7735,
    "auroc_pos_vs_real_ci": [
     0.683,
     0.8583
    ]
   },
   "php": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     5,
     38
    ],
    "fa_real": [
     1,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.7544,
    "auroc_pos_vs_real_ci": [
     0.6627,
     0.8424
    ]
   },
   "rust": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     2,
     37
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.6823,
    "auroc_pos_vs_real_ci": [
     0.5802,
     0.7969
    ]
   },
   "swift_objc": {
    "n": {
     "positive": 38,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     10,
     38
    ],
    "fa_real": [
     0,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9077,
    "auroc_pos_vs_real_ci": [
     0.8558,
     0.9609
    ]
   },
   "cm_bash_sh": {
    "n": {
     "positive": 33,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     17,
     33
    ],
    "fa_real": [
     6,
     200
    ],
    "fa_controls": [
     1,
     40
    ],
    "auroc_pos_vs_real": 0.8736,
    "auroc_pos_vs_real_ci": [
     0.8672,
     0.894
    ]
   },
   "cm_cmd_batch": {
    "n": {
     "positive": 33,
     "real": 130,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     23,
     33
    ],
    "fa_real": [
     0,
     130
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9688,
    "auroc_pos_vs_real_ci": [
     0.9396,
     0.9917
    ]
   },
   "cm_zsh_macos": {
    "n": {
     "positive": 36,
     "real": 144,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     23,
     36
    ],
    "fa_real": [
     1,
     144
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9323,
    "auroc_pos_vs_real_ci": [
     0.9026,
     0.9886
    ]
   },
   "cm_powershell": {
    "n": {
     "positive": 37,
     "real": 150,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     22,
     37
    ],
    "fa_real": [
     3,
     150
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9548,
    "auroc_pos_vs_real_ci": [
     0.9471,
     0.9669
    ]
   },
   "cm_python": {
    "n": {
     "positive": 35,
     "real": 200,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     25,
     35
    ],
    "fa_real": [
     5,
     200
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9164,
    "auroc_pos_vs_real_ci": [
     0.8851,
     0.998
    ]
   },
   "cm_js_ts": {
    "n": {
     "positive": 36,
     "real": 129,
     "hard_control": 30,
     "easy_control": 10
    },
    "recall": [
     26,
     36
    ],
    "fa_real": [
     4,
     129
    ],
    "fa_controls": [
     0,
     40
    ],
    "auroc_pos_vs_real": 0.9076,
    "auroc_pos_vs_real_ci": [
     0.7878,
     0.962
    ]
   },
   "ALL_diff": {
    "n": {
     "positive": 300,
     "real": 1197,
     "hard_control": 240,
     "easy_control": 80
    },
    "recall": [
     36,
     300
    ],
    "fa_real": [
     3,
     1197
    ],
    "fa_controls": [
     1,
     320
    ],
    "auroc_pos_vs_real": 0.796,
    "auroc_pos_vs_real_ci": [
     0.7671,
     0.8273
    ]
   },
   "ALL_codemode": {
    "n": {
     "positive": 210,
     "real": 953,
     "hard_control": 180,
     "easy_control": 60
    },
    "recall": [
     136,
     210
    ],
    "fa_real": [
     19,
     953
    ],
    "fa_controls": [
     1,
     240
    ],
    "auroc_pos_vs_real": 0.924,
    "auroc_pos_vs_real_ci": [
     0.9028,
     0.9563
    ]
   },
   "ALL": {
    "n": {
     "positive": 510,
     "real": 2150,
     "hard_control": 420,
     "easy_control": 140
    },
    "recall": [
     172,
     510
    ],
    "fa_real": [
     22,
     2150
    ],
    "fa_controls": [
     2,
     560
    ],
    "auroc_pos_vs_real": 0.8009,
    "auroc_pos_vs_real_ci": [
     0.7743,
     0.8374
    ]
   }
  }
 },
 "email_in_headers": {
  "B1s1": {
   "theta": [
    0.9,
    0.9
   ],
   "benign|all": "0/60",
   "benign|lang:bash": "0/18",
   "benign|lang:js": "0/18",
   "benign|lang:python": "0/24",
   "benign|writer:claude": "0/30",
   "benign|writer:openai": "0/30",
   "leak|all": "1/40",
   "leak|lang:bash": "0/12",
   "leak|lang:js": "1/12",
   "leak|lang:python": "0/16",
   "leak|writer:claude": "1/20",
   "leak|writer:openai": "0/20",
   "variant:C1": "0/15",
   "variant:C2": "0/15",
   "variant:C3": "0/15",
   "variant:C4": "0/15",
   "variant:P1": "0/14",
   "variant:P2": "0/13",
   "variant:P3": "1/13",
   "bases_fully_right": "1/40"
  },
  "B1s2": {
   "theta": [
    0.9,
    0.9
   ],
   "benign|all": "0/60",
   "benign|lang:bash": "0/18",
   "benign|lang:js": "0/18",
   "benign|lang:python": "0/24",
   "benign|writer:claude": "0/30",
   "benign|writer:openai": "0/30",
   "leak|all": "0/40",
   "leak|lang:bash": "0/12",
   "leak|lang:js": "0/12",
   "leak|lang:python": "0/16",
   "leak|writer:claude": "0/20",
   "leak|writer:openai": "0/20",
   "variant:C1": "0/15",
   "variant:C2": "0/15",
   "variant:C3": "0/15",
   "variant:C4": "0/15",
   "variant:P1": "0/14",
   "variant:P2": "0/13",
   "variant:P3": "0/13",
   "bases_fully_right": "0/40"
  },
  "A1": {
   "theta": [
    0.5,
    0.7
   ],
   "benign|all": "4/60",
   "benign|lang:bash": "0/18",
   "benign|lang:js": "4/18",
   "benign|lang:python": "0/24",
   "benign|writer:claude": "2/30",
   "benign|writer:openai": "2/30",
   "leak|all": "3/40",
   "leak|lang:bash": "0/12",
   "leak|lang:js": "2/12",
   "leak|lang:python": "1/16",
   "leak|writer:claude": "1/20",
   "leak|writer:openai": "2/20",
   "variant:C1": "2/15",
   "variant:C2": "0/15",
   "variant:C3": "0/15",
   "variant:C4": "2/15",
   "variant:P1": "1/14",
   "variant:P2": "2/13",
   "variant:P3": "0/13",
   "bases_fully_right": "1/40"
  }
 },
 "placeholder_summary": {
  "seed": 1,
  "theta": [
   0.9,
   0.9
  ],
  "multilingual": {
   "ALL": {
    "extractor": {
     "recall": [
      458,
      510
     ],
     "fa_real": [
      11,
      2150
     ],
     "fa_controls": [
      1,
      560
     ],
     "auroc_pos_vs_real": 0.9849
    },
    "placeholder": {
     "recall": [
      448,
      510
     ],
     "fa_real": [
      5,
      2150
     ],
     "fa_controls": [
      1,
      560
     ],
     "auroc_pos_vs_real": 0.9837
    }
   },
   "ALL_diff": {
    "extractor": {
     "recall": [
      278,
      300
     ],
     "fa_real": [
      3,
      1197
     ],
     "fa_controls": [
      1,
      320
     ],
     "auroc_pos_vs_real": 0.989
    },
    "placeholder": {
     "recall": [
      277,
      300
     ],
     "fa_real": [
      2,
      1197
     ],
     "fa_controls": [
      1,
      320
     ],
     "auroc_pos_vs_real": 0.9888
    }
   },
   "ALL_codemode": {
    "extractor": {
     "recall": [
      180,
      210
     ],
     "fa_real": [
      8,
      953
     ],
     "fa_controls": [
      0,
      240
     ],
     "auroc_pos_vs_real": 0.9789
    },
    "placeholder": {
     "recall": [
      171,
      210
     ],
     "fa_real": [
      3,
      953
     ],
     "fa_controls": [
      0,
      240
     ],
     "auroc_pos_vs_real": 0.9755
    }
   },
   "cm_bash_sh": {
    "extractor": {
     "recall": [
      24,
      33
     ],
     "fa_real": [
      1,
      200
     ],
     "fa_controls": [
      0,
      40
     ],
     "auroc_pos_vs_real": 0.9345
    },
    "placeholder": {
     "recall": [
      20,
      33
     ],
     "fa_real": [
      1,
      200
     ],
     "fa_controls": [
      0,
      40
     ],
     "auroc_pos_vs_real": 0.927
    }
   }
  },
  "independent_agent_diffs": {
   "extractor": {
    "found": [
     21,
     22
    ],
    "flagged": [
     0,
     172
    ]
   },
   "placeholder": {
    "found": [
     20,
     22
    ],
    "flagged": [
     0,
     172
    ]
   }
  }
 }
}