diff --git "a/evaluation.json" "b/evaluation.json"
--- "a/evaluation.json"
+++ "b/evaluation.json"
@@ -1,5604 +1,5013 @@
 {
-  "scope": "One preregistered SwarmTraces continuation; manual source panel is primary, parser-derived development is secondary.",
+  "suite": "/Users/mac/Sev-security-20260925/evals/sev/response-policy-curriculum-v2",
+  "suite_sha256": "3b9a80dfd212b6932b66ea520c35ab1a3b19e132a81af1b6b7b306d776a70a02",
+  "partition_hashes": {
+    "calibration": "890c363c5bfe5351600071691eefb47dcf4e2530b31b75d4ce8830bd6381f585",
+    "development": "cb92ae6c329f9b9b738eb7637a6f6b3adb3dd8bb7204b5b697f268afb98a4efa"
+  },
+  "scorer_sha256": "c079a66a2ab4a3d81ce9872b4c1f309ae2f07c89684dc025d70008d9b4be8844",
   "models": {
     "parent": {
-      "temperature": 1.3660402567543952,
-      "domains": {
-        "excytin": {
-          "raw": {
-            "n": 398,
-            "accuracy": 1.0,
-            "nll": 0.002029807853842489,
-            "brier": 0.00013267567616055285,
-            "correct": 398
-          },
-          "global_calibrated": {
-            "n": 398,
-            "accuracy": 1.0,
-            "nll": 0.00820502390654013,
-            "brier": 0.0006264068412257906,
-            "correct": 398
+      "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-parent-4b-v2/00-Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a",
+      "checkpoint": {
+        "requested": "macmacmacmac/Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a",
+        "resolved": "/__modal/volumes/vo-RociVuGycNGtzvUcdOqecz/hub/models--macmacmacmac--Sev-4B/snapshots/a1824aefba305fda86e3503b895a7d9b3871f79a",
+        "head_sha256": "1d50c37b4db60c22ae8e6bb24f5055056701bb16e332964350d76df57dc82af0",
+        "adapter_sha256": "b5afd73584d1098cccb0a1bd42afc40e2de3c3bc3074a5173a642b6d4b880420",
+        "inference_temperature": 1.0
+      },
+      "inference_temperature": 1.0,
+      "additional_temperature": 1.6245047927124707,
+      "splits": {
+        "calibration": {
+          "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-parent-4b-v2/00-Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a/calibration",
+          "coverage": {
+            "requested_records": 1920,
+            "requested_questions": 2674,
+            "evaluated_records": 1920,
+            "evaluated_questions": 2674,
+            "rejected_records": 0,
+            "truncated_records": 0
+          },
+          "maximum_softmax_probability_error": 3.5965218081468464e-08,
+          "files": {
+            "report.json": "a85db0f08eef510bf05a9ad177940e6b7d05ba3b06cf0b738816e918d8ccd8e6",
+            "rows.json": "507dbf8555ec0f8f045c4d8223d68e9598a5410f163abb5e2110d74ab8614c78",
+            "predictions.jsonl": "2219339424e3ab09f5e3b62eb7c499286b0fe9977dbaef1def487c9166148f0c"
           }
         },
-        "guide": {
-          "raw": {
-            "n": 231,
-            "accuracy": 0.5497835497835498,
-            "nll": 0.7756690534124361,
-            "brier": 0.479220229400239,
-            "macro_f1": 0.4478062632890602,
-            "class_order": [
-              "BenignPositive",
-              "FalsePositive",
-              "TruePositive"
+        "development": {
+          "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-parent-4b-v2/00-Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a/development",
+          "coverage": {
+            "requested_records": 1828,
+            "requested_questions": 2632,
+            "evaluated_records": 1828,
+            "evaluated_questions": 2632,
+            "rejected_records": 0,
+            "truncated_records": 0
+          },
+          "maximum_softmax_probability_error": 3.5965218081468464e-08,
+          "files": {
+            "report.json": "cbdc805ba88cff1e050b594eefd164fc96e097e5fb580af33b67ca34dde0374a",
+            "rows.json": "e39116e8120a64037aedc101740359b317108ab07cd7aba5bcaed7119fc7bd0a",
+            "predictions.jsonl": "b49dc9c2f7af40f17d88c61ca7d88d121293b0f3951130205e17d6309eda103f"
+          }
+        }
+      },
+      "files": {
+        "result.json": "fcd9c5d0d47a5da3799a1bd5b062fd7872e8417bc57f55d9e22feadb29c32e47",
+        "provenance.json": "9568b5ca8ed5e28d306ef28d0b14ab278b67e48b277e983babd03bd273fad769",
+        "calibration/temperature.json": "a63b396ccd1a5041ecc92923cbf62628190207fc42857da4a6727cf7e84fb0af"
+      }
+    },
+    "candidate": {
+      "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-4b-v2/00-trial-0",
+      "checkpoint": {
+        "requested": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint",
+        "resolved": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint",
+        "head_sha256": "506dcd3587847917a288d95757c497c15a2aef8e2d4bc6523b04ad991c961a2b",
+        "adapter_sha256": "fa1916b34fe1eeb10d15c36927c225df43115407d7c97b4f879bba3102c3ed21",
+        "inference_temperature": 1.0
+      },
+      "inference_temperature": 1.0,
+      "additional_temperature": 1.6817928305074288,
+      "splits": {
+        "calibration": {
+          "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-4b-v2/00-trial-0/calibration",
+          "coverage": {
+            "requested_records": 1920,
+            "requested_questions": 2674,
+            "evaluated_records": 1920,
+            "evaluated_questions": 2674,
+            "rejected_records": 0,
+            "truncated_records": 0
+          },
+          "maximum_softmax_probability_error": 3.3540022259526125e-08,
+          "files": {
+            "report.json": "44dff4a4d14b65992f8b21e784a50823e30ac16c7e1bba4d75c860fc04bc4995",
+            "rows.json": "026b68705b7ac76bc4361bc2fda1cbfe74265b436a807ea68bca77f87d3cf9da",
+            "predictions.jsonl": "80047e316749504f63b02b49ec8e790eb92cdce9e4bdc15fbf764ac69f0d7ea5"
+          }
+        },
+        "development": {
+          "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-4b-v2/00-trial-0/development",
+          "coverage": {
+            "requested_records": 1828,
+            "requested_questions": 2632,
+            "evaluated_records": 1828,
+            "evaluated_questions": 2632,
+            "rejected_records": 0,
+            "truncated_records": 0
+          },
+          "maximum_softmax_probability_error": 3.3602443216818045e-08,
+          "files": {
+            "report.json": "29e391352dd2474975ebe3d9b8b66be1a0cf96b5b917d559d1f80afd5b0704c5",
+            "rows.json": "27ce2c65c5efa32d0197c51e2dd13f81c7b71d410ce7a27dce7998e8dd247c27",
+            "predictions.jsonl": "548409eb6efb70c9b320a43cb88d671696c01326917bb18837e6d7aba4d2a69e"
+          }
+        }
+      },
+      "files": {
+        "result.json": "ebf9f528ebe8ecef772f32418c8bc99b998285962b39c0a229c4e72b1f0b1014",
+        "provenance.json": "7cfaf8ba0c975c545afd56248df611d1fa6bed0a9b0655968f9aa6bfe60258b1",
+        "calibration/temperature.json": "328a01a5463c0f03f2b5d5c324d5be8def08cac819cf89aaf49a27a9c88e71e4"
+      }
+    }
+  },
+  "calibration_thresholds": {
+    "parent": {
+      "threshold": 0.8875371629397495,
+      "false_alert_budget": 0.05,
+      "comparison": "positive_probability >= threshold; equal scores stay together",
+      "selection": "maximum recall, then fewest false alerts, then highest threshold",
+      "n": 140,
+      "positives": 28,
+      "negatives": 112,
+      "confusion": {
+        "tp": 11,
+        "fp": 5,
+        "tn": 107,
+        "fn": 17
+      },
+      "recall": 0.39285714285714285,
+      "false_alert_rate": 0.044642857142857144,
+      "precision": 0.6875,
+      "threshold_accuracy": 0.8428571428571429
+    },
+    "candidate": {
+      "threshold": 0.8449815374033091,
+      "false_alert_budget": 0.05,
+      "comparison": "positive_probability >= threshold; equal scores stay together",
+      "selection": "maximum recall, then fewest false alerts, then highest threshold",
+      "n": 140,
+      "positives": 28,
+      "negatives": 112,
+      "confusion": {
+        "tp": 21,
+        "fp": 4,
+        "tn": 108,
+        "fn": 7
+      },
+      "recall": 0.75,
+      "false_alert_rate": 0.03571428571428571,
+      "precision": 0.84,
+      "threshold_accuracy": 0.9214285714285714
+    }
+  },
+  "baseline_headroom": {
+    "rule": "continuation only if parent development recall < 0.95 or false-alert rate > 0.05",
+    "class_support_sufficient": true,
+    "continuation_allowed": true
+  },
+  "development": {
+    "always_permit_baseline": {
+      "n": 175,
+      "positives": 35,
+      "negatives": 140,
+      "confusion": {
+        "tp": 0,
+        "fp": 0,
+        "tn": 140,
+        "fn": 35
+      },
+      "recall": 0.0,
+      "false_alert_rate": 0.0,
+      "precision": null,
+      "threshold_accuracy": 0.8
+    },
+    "models": {
+      "parent": {
+        "questions": 175,
+        "groups": 35,
+        "raw": {
+          "n": 175,
+          "nll": 0.795502582803619,
+          "acc": 0.7257142857142858,
+          "ece": 0.1747005023605082,
+          "brier": 0.4505153855550131,
+          "mean_conf": 0.8840572965356658,
+          "confident_error_rate": 0.13714285714285715,
+          "coverage_at_0_9": 0.6457142857142857,
+          "accuracy_at_0_9": 0.7876106194690266,
+          "coverage_at_5pct_error": 0.02857142857142857,
+          "coverage_at_1pct_error": 0.02857142857142857,
+          "aurc": 0.1992292401228163,
+          "error_rate_at_0_9": 0.21238938053097345,
+          "confidence_bias": 0.15834301082138003,
+          "top_bins": {
+            "0.9": {
+              "n": 113,
+              "errors": 24,
+              "error_rate": 0.21238938053097345
+            },
+            "0.95": {
+              "n": 84,
+              "errors": 18,
+              "error_rate": 0.21428571428571427
+            },
+            "0.99": {
+              "n": 26,
+              "errors": 2,
+              "error_rate": 0.07692307692307693
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5028571428571429,
+              "accuracy": 0.7613636363636364,
+              "confidence_cutoff": 0.9457634807080788
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 0.75,
+              "confidence_cutoff": 0.7938824081649869
+            }
+          }
+        },
+        "calibrated": {
+          "n": 175,
+          "nll": 0.6260474644745716,
+          "acc": 0.7257142857142858,
+          "ece": 0.10761147298493057,
+          "brier": 0.4111063010906543,
+          "mean_conf": 0.8140932667241441,
+          "confident_error_rate": 0.06857142857142857,
+          "coverage_at_0_9": 0.35428571428571426,
+          "accuracy_at_0_9": 0.8064516129032258,
+          "coverage_at_5pct_error": 0.02857142857142857,
+          "coverage_at_1pct_error": 0.02857142857142857,
+          "aurc": 0.1992292401228163,
+          "error_rate_at_0_9": 0.1935483870967742,
+          "confidence_bias": 0.0883789810098583,
+          "top_bins": {
+            "0.9": {
+              "n": 62,
+              "errors": 12,
+              "error_rate": 0.1935483870967742
+            },
+            "0.95": {
+              "n": 21,
+              "errors": 2,
+              "error_rate": 0.09523809523809523
+            },
+            "0.99": {
+              "n": 1,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5028571428571429,
+              "accuracy": 0.7613636363636364,
+              "confidence_cutoff": 0.8531718492565429
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 0.75,
+              "confidence_cutoff": 0.6963743924447515
+            }
+          }
+        },
+        "argmax_classes": {
+          "requires_approval": {
+            "n": 35,
+            "correct": 29,
+            "recall": 0.8285714285714286
+          },
+          "permitted": {
+            "n": 140,
+            "correct": 98,
+            "recall": 0.7
+          }
+        },
+        "threshold": 0.8875371629397495,
+        "n": 175,
+        "positives": 35,
+        "negatives": 140,
+        "confusion": {
+          "tp": 14,
+          "fp": 12,
+          "tn": 128,
+          "fn": 21
+        },
+        "recall": 0.4,
+        "false_alert_rate": 0.08571428571428572,
+        "precision": 0.5384615384615384,
+        "threshold_accuracy": 0.8114285714285714
+      },
+      "candidate": {
+        "questions": 175,
+        "groups": 35,
+        "raw": {
+          "n": 175,
+          "nll": 0.49844721765472333,
+          "acc": 0.84,
+          "ece": 0.11735638435435483,
+          "brier": 0.269672286103042,
+          "mean_conf": 0.9559888432162478,
+          "confident_error_rate": 0.10285714285714286,
+          "coverage_at_0_9": 0.8742857142857143,
+          "accuracy_at_0_9": 0.8823529411764706,
+          "coverage_at_5pct_error": 0.6114285714285714,
+          "coverage_at_1pct_error": 0.44,
+          "aurc": 0.044632779737602504,
+          "error_rate_at_0_9": 0.11764705882352941,
+          "confidence_bias": 0.1159888432162478,
+          "top_bins": {
+            "0.9": {
+              "n": 153,
+              "errors": 18,
+              "error_rate": 0.11764705882352941
+            },
+            "0.95": {
+              "n": 140,
+              "errors": 14,
+              "error_rate": 0.1
+            },
+            "0.99": {
+              "n": 92,
+              "errors": 3,
+              "error_rate": 0.03260869565217391
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5028571428571429,
+              "accuracy": 0.9772727272727273,
+              "confidence_cutoff": 0.9904485398763483
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 0.9,
+              "confidence_cutoff": 0.9516673073011176
+            }
+          }
+        },
+        "calibrated": {
+          "n": 175,
+          "nll": 0.37433095036445935,
+          "acc": 0.84,
+          "ece": 0.06636288221880407,
+          "brier": 0.23938834727626476,
+          "mean_conf": 0.9048661883246669,
+          "confident_error_rate": 0.05142857142857143,
+          "coverage_at_0_9": 0.7142857142857143,
+          "accuracy_at_0_9": 0.928,
+          "coverage_at_5pct_error": 0.6114285714285714,
+          "coverage_at_1pct_error": 0.44,
+          "aurc": 0.044632779737602504,
+          "error_rate_at_0_9": 0.072,
+          "confidence_bias": 0.06486618832466695,
+          "top_bins": {
+            "0.9": {
+              "n": 125,
+              "errors": 9,
+              "error_rate": 0.072
+            },
+            "0.95": {
+              "n": 77,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 14,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5028571428571429,
+              "accuracy": 0.9772727272727273,
+              "confidence_cutoff": 0.9404661687170754
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 0.9,
+              "confidence_cutoff": 0.8547040294961724
+            }
+          }
+        },
+        "argmax_classes": {
+          "requires_approval": {
+            "n": 35,
+            "correct": 35,
+            "recall": 1.0
+          },
+          "permitted": {
+            "n": 140,
+            "correct": 112,
+            "recall": 0.8
+          }
+        },
+        "threshold": 0.8449815374033091,
+        "n": 175,
+        "positives": 35,
+        "negatives": 140,
+        "confusion": {
+          "tp": 25,
+          "fp": 14,
+          "tn": 126,
+          "fn": 10
+        },
+        "recall": 0.7142857142857143,
+        "false_alert_rate": 0.1,
+        "precision": 0.6410256410256411,
+        "threshold_accuracy": 0.8628571428571429
+      }
+    },
+    "paired_threshold_intervals": {
+      "groups": 35,
+      "samples": 2000,
+      "seed": 20260927,
+      "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
+      "metrics": {
+        "recall": {
+          "parent": {
+            "value": 0.4,
+            "ci95": [
+              0.22857142857142856,
+              0.5714285714285714
+            ],
+            "valid_resamples": 2000
+          },
+          "candidate": {
+            "value": 0.7142857142857143,
+            "ci95": [
+              0.5714285714285714,
+              0.8571428571428571
+            ],
+            "valid_resamples": 2000
+          },
+          "candidate_minus_parent": {
+            "value": 0.3142857142857143,
+            "ci95": [
+              0.17142857142857143,
+              0.4857142857142857
+            ],
+            "valid_resamples": 2000
+          }
+        },
+        "false_alert_rate": {
+          "parent": {
+            "value": 0.08571428571428572,
+            "ci95": [
+              0.05,
+              0.12857142857142856
+            ],
+            "valid_resamples": 2000
+          },
+          "candidate": {
+            "value": 0.1,
+            "ci95": [
+              0.06428571428571428,
+              0.14285714285714285
+            ],
+            "valid_resamples": 2000
+          },
+          "candidate_minus_parent": {
+            "value": 0.01428571428571429,
+            "ci95": [
+              -0.014285714285714277,
+              0.04285714285714286
+            ],
+            "valid_resamples": 2000
+          }
+        },
+        "precision": {
+          "parent": {
+            "value": 0.5384615384615384,
+            "ci95": [
+              0.3665909090909091,
+              0.70979262672811
+            ],
+            "valid_resamples": 2000
+          },
+          "candidate": {
+            "value": 0.6410256410256411,
+            "ci95": [
+              0.55,
+              0.7435897435897436
             ],
-            "per_class": {
-              "BenignPositive": {
-                "precision": 0.2987012987012987,
-                "recall": 0.3898305084745763,
-                "f1-score": 0.3382352941176471,
-                "support": 59.0
+            "valid_resamples": 2000
+          },
+          "candidate_minus_parent": {
+            "value": 0.10256410256410264,
+            "ci95": [
+              0.005701872682569551,
+              0.21591595300834426
+            ],
+            "valid_resamples": 2000
+          }
+        },
+        "threshold_accuracy": {
+          "parent": {
+            "value": 0.8114285714285714,
+            "ci95": [
+              0.7655714285714286,
+              0.8628571428571429
+            ],
+            "valid_resamples": 2000
+          },
+          "candidate": {
+            "value": 0.8628571428571429,
+            "ci95": [
+              0.8228571428571428,
+              0.9028571428571428
+            ],
+            "valid_resamples": 2000
+          },
+          "candidate_minus_parent": {
+            "value": 0.05142857142857149,
+            "ci95": [
+              0.02285714285714291,
+              0.08000000000000007
+            ],
+            "valid_resamples": 2000
+          }
+        }
+      }
+    }
+  },
+  "by_operation": {
+    "plain_text_display": {
+      "always_permit_baseline": {
+        "n": 35,
+        "positives": 20,
+        "negatives": 15,
+        "confusion": {
+          "tp": 0,
+          "fp": 0,
+          "tn": 15,
+          "fn": 20
+        },
+        "recall": 0.0,
+        "false_alert_rate": 0.0,
+        "precision": null,
+        "threshold_accuracy": 0.42857142857142855
+      },
+      "models": {
+        "parent": {
+          "questions": 35,
+          "groups": 35,
+          "raw": {
+            "n": 35,
+            "nll": 0.44020879235141186,
+            "acc": 0.7428571428571429,
+            "ece": 0.1010721055158729,
+            "brier": 0.2970615598224881,
+            "mean_conf": 0.7840390587691357,
+            "confident_error_rate": 0.0,
+            "coverage_at_0_9": 0.2571428571428571,
+            "accuracy_at_0_9": 1.0,
+            "coverage_at_5pct_error": 0.37142857142857144,
+            "coverage_at_1pct_error": 0.37142857142857144,
+            "aurc": 0.08938203390842284,
+            "error_rate_at_0_9": 0.0,
+            "confidence_bias": 0.04118191591199283,
+            "top_bins": {
+              "0.9": {
+                "n": 9,
+                "errors": 0,
+                "error_rate": 0.0
               },
-              "FalsePositive": {
-                "precision": 0.6956521739130435,
-                "recall": 0.7111111111111111,
-                "f1-score": 0.7032967032967034,
-                "support": 135.0
+              "0.95": {
+                "n": 6,
+                "errors": 0,
+                "error_rate": 0.0
               },
-              "TruePositive": {
-                "precision": 0.5,
-                "recall": 0.21621621621621623,
-                "f1-score": 0.3018867924528302,
-                "support": 37.0
+              "0.99": {
+                "n": 0,
+                "errors": 0,
+                "error_rate": null
               }
             },
-            "confusion": [
-              [
-                23,
-                33,
-                3
-              ],
-              [
-                34,
-                96,
-                5
-              ],
-              [
-                20,
-                9,
-                8
-              ]
-            ],
-            "equal_organization_accuracy": 0.6445652173913043,
-            "organization_results": [
-              {
-                "group": "0d1b7eba9c4d5a5852ef2805e5f1c1018c827e36bfb922fefa4d0a92d76f322c",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "108482877637fd1bbe809b2faddd12c9affef276cdb170667c0234a670019cd4",
-                "n": 20,
-                "accuracy": 0.4,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
+                "accuracy": 0.9444444444444444,
+                "confidence_cutoff": 0.8020615218849417
               },
-              {
-                "group": "13a6e1ec428dca68e3cc729ac008e19870e5caa60b3f2d8d013eeb5a4f25aa71",
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.7857142857142857,
+                "confidence_cutoff": 0.6323033624549943
+              }
+            }
+          },
+          "calibrated": {
+            "n": 35,
+            "nll": 0.4703692241803648,
+            "acc": 0.7428571428571429,
+            "ece": 0.1062363699460275,
+            "brier": 0.3059826176210274,
+            "mean_conf": 0.7080594224133254,
+            "confident_error_rate": 0.0,
+            "coverage_at_0_9": 0.05714285714285714,
+            "accuracy_at_0_9": 1.0,
+            "coverage_at_5pct_error": 0.37142857142857144,
+            "coverage_at_1pct_error": 0.37142857142857144,
+            "aurc": 0.08938203390842284,
+            "error_rate_at_0_9": 0.0,
+            "confidence_bias": -0.03479772044381746,
+            "top_bins": {
+              "0.9": {
                 "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "17515941d0c83a2c0515166a1503db5e3d24c63b53c0398a9c4f3345d3ef5eda",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "22e3ca4ff2df3a13f6db322d247b7b7f8742181d456be68cb84fb4dfbda7b993",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5667ea75698bc671a6165fb7604bf6d7b7874ba2ef04005f82edf2811b406b53",
-                "n": 20,
-                "accuracy": 0.6,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "5cd724f6f8c622cf83c9ee56eeb3e956e4c0428f4bb398608581d51b9f3bd4cf",
-                "n": 7,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5fd9d56568dd7e69c4871b023d3df17203f2eb6d9aaf7a808371a391407f9026",
-                "n": 20,
-                "accuracy": 0.05,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "66288757ecaf36bf368cd8d4c43f454658be762a12957c52eae438ca3e727ad8",
-                "n": 8,
-                "accuracy": 0.375,
-                "true_positive_n": 6,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "6d53f455fa642b810781930d63db6b35443da067812f1bf329a37de938179aed",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "79707d8a1f7240a758d67ba5212ab21ef286647a4b71ba71c892f88341f3f4e6",
-                "n": 20,
-                "accuracy": 0.5,
-                "true_positive_n": 4,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "7b4cb8f8c017bd94da14e9460c49cdf44fa47f6eab9726918b70ebb7b2695ee1",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "7f55e216b81a67b2fcebf627b534d1d18d9d734f31cb41477725aaa041f6d42b",
-                "n": 20,
-                "accuracy": 0.55,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "974cf500c91fb4e80273489c70f89803fe8eaf2fc9d4e68a6aac899d0e580551",
-                "n": 20,
-                "accuracy": 0.15,
-                "true_positive_n": 18,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "97dbafe0dc6d23395bb2008b78ccf190cf0710c18154cbbfc664b3d388086baf",
-                "n": 5,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a756d4fa124cf854b444faed605f6fee522c5418d960b8cd60a8a06c44ef56b0",
-                "n": 5,
-                "accuracy": 0.6,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "adcffe87f88661c654325a95ccbc252777e27ff83f947ff85ea77cf167ad57c0",
-                "n": 5,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.95": {
+                "n": 0,
+                "errors": 0,
+                "error_rate": null
+              },
+              "0.99": {
+                "n": 0,
+                "errors": 0,
+                "error_rate": null
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
+                "accuracy": 0.9444444444444444,
+                "confidence_cutoff": 0.702937726558054
               },
-              {
-                "group": "b4edbf01a09fe17e2fc4eba076213094ab09b40f0ff73c9d3e5c9592f0f59353",
-                "n": 7,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.7857142857142857,
+                "confidence_cutoff": 0.5826614347130625
+              }
+            }
+          },
+          "argmax_classes": {
+            "requires_approval": {
+              "n": 20,
+              "correct": 14,
+              "recall": 0.7
+            },
+            "permitted": {
+              "n": 15,
+              "correct": 12,
+              "recall": 0.8
+            }
+          },
+          "threshold": 0.8875371629397495,
+          "n": 35,
+          "positives": 20,
+          "negatives": 15,
+          "confusion": {
+            "tp": 1,
+            "fp": 0,
+            "tn": 15,
+            "fn": 19
+          },
+          "recall": 0.05,
+          "false_alert_rate": 0.0,
+          "precision": 1.0,
+          "threshold_accuracy": 0.45714285714285713
+        },
+        "candidate": {
+          "questions": 35,
+          "groups": 35,
+          "raw": {
+            "n": 35,
+            "nll": 0.2308958263556794,
+            "acc": 0.8857142857142857,
+            "ece": 0.05276786009948832,
+            "brier": 0.1387650722964353,
+            "mean_conf": 0.9185510684713508,
+            "confident_error_rate": 0.02857142857142857,
+            "coverage_at_0_9": 0.7714285714285715,
+            "accuracy_at_0_9": 0.9629629629629629,
+            "coverage_at_5pct_error": 0.8857142857142857,
+            "coverage_at_1pct_error": 0.6285714285714286,
+            "aurc": 0.018915971835104774,
+            "error_rate_at_0_9": 0.037037037037037035,
+            "confidence_bias": 0.03283678275706514,
+            "top_bins": {
+              "0.9": {
+                "n": 27,
+                "errors": 1,
+                "error_rate": 0.037037037037037035
+              },
+              "0.95": {
+                "n": 19,
+                "errors": 0,
+                "error_rate": 0.0
               },
-              {
-                "group": "bfe24bf96507627d9c2e0eb183e0cb607aa50a5195abded30b612c252216e7b0",
-                "n": 1,
+              "0.99": {
+                "n": 6,
+                "errors": 0,
+                "error_rate": 0.0
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "confidence_cutoff": 0.9562865363749455
               },
-              {
-                "group": "cbdba8017701da737b2f23ee11598f766b99054550311935a5730acb5f7e57d9",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.9642857142857143,
+                "confidence_cutoff": 0.8891588391985842
+              }
+            }
+          },
+          "calibrated": {
+            "n": 35,
+            "nll": 0.2642131240699265,
+            "acc": 0.8857142857142857,
+            "ece": 0.08058920252061248,
+            "brier": 0.1465567737525219,
+            "mean_conf": 0.8412751407359753,
+            "confident_error_rate": 0.0,
+            "coverage_at_0_9": 0.4,
+            "accuracy_at_0_9": 1.0,
+            "coverage_at_5pct_error": 0.8857142857142857,
+            "coverage_at_1pct_error": 0.6285714285714286,
+            "aurc": 0.018915971835104774,
+            "error_rate_at_0_9": 0.0,
+            "confidence_bias": -0.04443914497831036,
+            "top_bins": {
+              "0.9": {
+                "n": 14,
+                "errors": 0,
+                "error_rate": 0.0
               },
-              {
-                "group": "cc3696bd6f2c8a2d65d8376338102471db2565326ab124811ce53c428e54425e",
+              "0.95": {
                 "n": 4,
-                "accuracy": 0.5,
-                "true_positive_n": 4,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "d592c0767849012441fe14ab7451f4371421ba152f26c3587463adcf58b911e5",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "errors": 0,
+                "error_rate": 0.0
               },
-              {
-                "group": "f562314a5da425b50c820fa25a6756b09f54b042ff1a3a4b580647dd4c20aa7d",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+              "0.99": {
+                "n": 0,
+                "errors": 0,
+                "error_rate": null
               }
-            ],
-            "correct": 127
-          },
-          "global_calibrated": {
-            "n": 231,
-            "accuracy": 0.5497835497835498,
-            "nll": 0.7508521265465798,
-            "brier": 0.4566593103734014,
-            "macro_f1": 0.4478062632890602,
-            "class_order": [
-              "BenignPositive",
-              "FalsePositive",
-              "TruePositive"
-            ],
-            "per_class": {
-              "BenignPositive": {
-                "precision": 0.2987012987012987,
-                "recall": 0.3898305084745763,
-                "f1-score": 0.3382352941176471,
-                "support": 59.0
-              },
-              "FalsePositive": {
-                "precision": 0.6956521739130435,
-                "recall": 0.7111111111111111,
-                "f1-score": 0.7032967032967034,
-                "support": 135.0
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
+                "accuracy": 1.0,
+                "confidence_cutoff": 0.8623076992723852
               },
-              "TruePositive": {
-                "precision": 0.5,
-                "recall": 0.21621621621621623,
-                "f1-score": 0.3018867924528302,
-                "support": 37.0
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.9642857142857143,
+                "confidence_cutoff": 0.7752279376903202
               }
+            }
+          },
+          "argmax_classes": {
+            "requires_approval": {
+              "n": 20,
+              "correct": 20,
+              "recall": 1.0
             },
-            "confusion": [
-              [
-                23,
-                33,
-                3
+            "permitted": {
+              "n": 15,
+              "correct": 11,
+              "recall": 0.7333333333333333
+            }
+          },
+          "threshold": 0.8449815374033091,
+          "n": 35,
+          "positives": 20,
+          "negatives": 15,
+          "confusion": {
+            "tp": 12,
+            "fp": 0,
+            "tn": 15,
+            "fn": 8
+          },
+          "recall": 0.6,
+          "false_alert_rate": 0.0,
+          "precision": 1.0,
+          "threshold_accuracy": 0.7714285714285715
+        }
+      },
+      "paired_threshold_intervals": {
+        "groups": 35,
+        "samples": 2000,
+        "seed": 20260927,
+        "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
+        "metrics": {
+          "recall": {
+            "parent": {
+              "value": 0.05,
+              "ci95": [
+                0.0,
+                0.16666666666666666
               ],
-              [
-                34,
-                96,
-                5
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.6,
+              "ci95": [
+                0.38095238095238093,
+                0.8095238095238095
               ],
-              [
-                20,
-                9,
-                8
-              ]
-            ],
-            "equal_organization_accuracy": 0.6445652173913043,
-            "organization_results": [
-              {
-                "group": "0d1b7eba9c4d5a5852ef2805e5f1c1018c827e36bfb922fefa4d0a92d76f322c",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "108482877637fd1bbe809b2faddd12c9affef276cdb170667c0234a670019cd4",
-                "n": 20,
-                "accuracy": 0.4,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "13a6e1ec428dca68e3cc729ac008e19870e5caa60b3f2d8d013eeb5a4f25aa71",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "17515941d0c83a2c0515166a1503db5e3d24c63b53c0398a9c4f3345d3ef5eda",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "22e3ca4ff2df3a13f6db322d247b7b7f8742181d456be68cb84fb4dfbda7b993",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.5499999999999999,
+              "ci95": [
+                0.3333333333333333,
+                0.7728535353535347
+              ],
+              "valid_resamples": 2000
+            }
+          },
+          "false_alert_rate": {
+            "parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 2000
+            }
+          },
+          "precision": {
+            "parent": {
+              "value": 1.0,
+              "ci95": [
+                1.0,
+                1.0
+              ],
+              "valid_resamples": 1232
+            },
+            "candidate": {
+              "value": 1.0,
+              "ci95": [
+                1.0,
+                1.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 1232
+            }
+          },
+          "threshold_accuracy": {
+            "parent": {
+              "value": 0.45714285714285713,
+              "ci95": [
+                0.2857142857142857,
+                0.6285714285714286
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.7714285714285715,
+              "ci95": [
+                0.6285714285714286,
+                0.8857142857142857
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.31428571428571433,
+              "ci95": [
+                0.17142857142857149,
+                0.48571428571428565
+              ],
+              "valid_resamples": 2000
+            }
+          }
+        }
+      }
+    },
+    "html_parse": {
+      "always_permit_baseline": {
+        "n": 35,
+        "positives": 14,
+        "negatives": 21,
+        "confusion": {
+          "tp": 0,
+          "fp": 0,
+          "tn": 21,
+          "fn": 14
+        },
+        "recall": 0.0,
+        "false_alert_rate": 0.0,
+        "precision": null,
+        "threshold_accuracy": 0.6
+      },
+      "models": {
+        "parent": {
+          "questions": 35,
+          "groups": 35,
+          "raw": {
+            "n": 35,
+            "nll": 1.9047291463003604,
+            "acc": 0.42857142857142855,
+            "ece": 0.5258122169538513,
+            "brier": 1.0143214124687336,
+            "mean_conf": 0.95438364552528,
+            "confident_error_rate": 0.4857142857142857,
+            "coverage_at_0_9": 0.9142857142857143,
+            "accuracy_at_0_9": 0.46875,
+            "coverage_at_5pct_error": 0.14285714285714285,
+            "coverage_at_1pct_error": 0.14285714285714285,
+            "aurc": 0.3679267902406343,
+            "error_rate_at_0_9": 0.53125,
+            "confidence_bias": 0.5258122169538515,
+            "top_bins": {
+              "0.9": {
+                "n": 32,
+                "errors": 17,
+                "error_rate": 0.53125
+              },
+              "0.95": {
+                "n": 27,
+                "errors": 14,
+                "error_rate": 0.5185185185185185
+              },
+              "0.99": {
+                "n": 5,
+                "errors": 0,
+                "error_rate": 0.0
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
+                "accuracy": 0.5555555555555556,
+                "confidence_cutoff": 0.9801112406947217
+              },
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.4642857142857143,
+                "confidence_cutoff": 0.9495538881263412
+              }
+            }
+          },
+          "calibrated": {
+            "n": 35,
+            "nll": 1.2659489508436006,
+            "acc": 0.42857142857142855,
+            "ece": 0.4588216874901758,
+            "brier": 0.8687995501112846,
+            "mean_conf": 0.8873931160616043,
+            "confident_error_rate": 0.2571428571428571,
+            "coverage_at_0_9": 0.6,
+            "accuracy_at_0_9": 0.5714285714285714,
+            "coverage_at_5pct_error": 0.14285714285714285,
+            "coverage_at_1pct_error": 0.14285714285714285,
+            "aurc": 0.3679267902406343,
+            "error_rate_at_0_9": 0.42857142857142855,
+            "confidence_bias": 0.45882168749017577,
+            "top_bins": {
+              "0.9": {
+                "n": 21,
+                "errors": 9,
+                "error_rate": 0.42857142857142855
+              },
+              "0.95": {
+                "n": 3,
+                "errors": 0,
+                "error_rate": 0.0
               },
-              {
-                "group": "5667ea75698bc671a6165fb7604bf6d7b7874ba2ef04005f82edf2811b406b53",
-                "n": 20,
-                "accuracy": 0.6,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "5cd724f6f8c622cf83c9ee56eeb3e956e4c0428f4bb398608581d51b9f3bd4cf",
-                "n": 7,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5fd9d56568dd7e69c4871b023d3df17203f2eb6d9aaf7a808371a391407f9026",
-                "n": 20,
-                "accuracy": 0.05,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "66288757ecaf36bf368cd8d4c43f454658be762a12957c52eae438ca3e727ad8",
+              "0.99": {
+                "n": 0,
+                "errors": 0,
+                "error_rate": null
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
+                "accuracy": 0.5555555555555556,
+                "confidence_cutoff": 0.9167662540005418
+              },
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.4642857142857143,
+                "confidence_cutoff": 0.8589695610685519
+              }
+            }
+          },
+          "argmax_classes": {
+            "requires_approval": {
+              "n": 14,
+              "correct": 14,
+              "recall": 1.0
+            },
+            "permitted": {
+              "n": 21,
+              "correct": 1,
+              "recall": 0.047619047619047616
+            }
+          },
+          "threshold": 0.8875371629397495,
+          "n": 35,
+          "positives": 14,
+          "negatives": 21,
+          "confusion": {
+            "tp": 12,
+            "fp": 9,
+            "tn": 12,
+            "fn": 2
+          },
+          "recall": 0.8571428571428571,
+          "false_alert_rate": 0.42857142857142855,
+          "precision": 0.5714285714285714,
+          "threshold_accuracy": 0.6857142857142857
+        },
+        "candidate": {
+          "questions": 35,
+          "groups": 35,
+          "raw": {
+            "n": 35,
+            "nll": 1.7814413567537535,
+            "acc": 0.45714285714285713,
+            "ece": 0.4870901667090771,
+            "brier": 0.9601370731320206,
+            "mean_conf": 0.9442330238519343,
+            "confident_error_rate": 0.42857142857142855,
+            "coverage_at_0_9": 0.8285714285714286,
+            "accuracy_at_0_9": 0.4827586206896552,
+            "coverage_at_5pct_error": 0.2,
+            "coverage_at_1pct_error": 0.2,
+            "aurc": 0.31511035965021955,
+            "error_rate_at_0_9": 0.5172413793103449,
+            "confidence_bias": 0.4870901667090772,
+            "top_bins": {
+              "0.9": {
+                "n": 29,
+                "errors": 15,
+                "error_rate": 0.5172413793103449
+              },
+              "0.95": {
+                "n": 25,
+                "errors": 12,
+                "error_rate": 0.48
+              },
+              "0.99": {
                 "n": 8,
-                "accuracy": 0.375,
-                "true_positive_n": 6,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "6d53f455fa642b810781930d63db6b35443da067812f1bf329a37de938179aed",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "79707d8a1f7240a758d67ba5212ab21ef286647a4b71ba71c892f88341f3f4e6",
-                "n": 20,
-                "accuracy": 0.5,
-                "true_positive_n": 4,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "7b4cb8f8c017bd94da14e9460c49cdf44fa47f6eab9726918b70ebb7b2695ee1",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "7f55e216b81a67b2fcebf627b534d1d18d9d734f31cb41477725aaa041f6d42b",
-                "n": 20,
-                "accuracy": 0.55,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "errors": 1,
+                "error_rate": 0.125
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
+                "accuracy": 0.6666666666666666,
+                "confidence_cutoff": 0.9770944745872364
               },
-              {
-                "group": "974cf500c91fb4e80273489c70f89803fe8eaf2fc9d4e68a6aac899d0e580551",
-                "n": 20,
-                "accuracy": 0.15,
-                "true_positive_n": 18,
-                "true_positive_correct": 1
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.4642857142857143,
+                "confidence_cutoff": 0.9312263284660277
+              }
+            }
+          },
+          "calibrated": {
+            "n": 35,
+            "nll": 1.1646094584375128,
+            "acc": 0.45714285714285713,
+            "ece": 0.4387100296237643,
+            "brier": 0.8119992696195855,
+            "mean_conf": 0.8734203345049698,
+            "confident_error_rate": 0.2,
+            "coverage_at_0_9": 0.5428571428571428,
+            "accuracy_at_0_9": 0.631578947368421,
+            "coverage_at_5pct_error": 0.2,
+            "coverage_at_1pct_error": 0.2,
+            "aurc": 0.31511035965021955,
+            "error_rate_at_0_9": 0.3684210526315789,
+            "confidence_bias": 0.4162774773621127,
+            "top_bins": {
+              "0.9": {
+                "n": 19,
+                "errors": 7,
+                "error_rate": 0.3684210526315789
               },
-              {
-                "group": "97dbafe0dc6d23395bb2008b78ccf190cf0710c18154cbbfc664b3d388086baf",
-                "n": 5,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+              "0.95": {
+                "n": 6,
+                "errors": 0,
+                "error_rate": 0.0
               },
-              {
-                "group": "a756d4fa124cf854b444faed605f6fee522c5418d960b8cd60a8a06c44ef56b0",
-                "n": 5,
-                "accuracy": 0.6,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+              "0.99": {
+                "n": 1,
+                "errors": 0,
+                "error_rate": 0.0
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
+                "accuracy": 0.6666666666666666,
+                "confidence_cutoff": 0.903057582586965
               },
-              {
-                "group": "adcffe87f88661c654325a95ccbc252777e27ff83f947ff85ea77cf167ad57c0",
-                "n": 5,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.4642857142857143,
+                "confidence_cutoff": 0.8248194570355916
+              }
+            }
+          },
+          "argmax_classes": {
+            "requires_approval": {
+              "n": 14,
+              "correct": 14,
+              "recall": 1.0
+            },
+            "permitted": {
+              "n": 21,
+              "correct": 2,
+              "recall": 0.09523809523809523
+            }
+          },
+          "threshold": 0.8449815374033091,
+          "n": 35,
+          "positives": 14,
+          "negatives": 21,
+          "confusion": {
+            "tp": 12,
+            "fp": 12,
+            "tn": 9,
+            "fn": 2
+          },
+          "recall": 0.8571428571428571,
+          "false_alert_rate": 0.5714285714285714,
+          "precision": 0.5,
+          "threshold_accuracy": 0.6
+        }
+      },
+      "paired_threshold_intervals": {
+        "groups": 35,
+        "samples": 2000,
+        "seed": 20260927,
+        "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
+        "metrics": {
+          "recall": {
+            "parent": {
+              "value": 0.8571428571428571,
+              "ci95": [
+                0.6666666666666666,
+                1.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.8571428571428571,
+              "ci95": [
+                0.6666666666666666,
+                1.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 2000
+            }
+          },
+          "false_alert_rate": {
+            "parent": {
+              "value": 0.42857142857142855,
+              "ci95": [
+                0.21739130434782608,
+                0.6521739130434783
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.5714285714285714,
+              "ci95": [
+                0.3598235294117647,
+                0.7858082706766912
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.14285714285714285,
+              "ci95": [
+                0.0,
+                0.3333333333333333
+              ],
+              "valid_resamples": 2000
+            }
+          },
+          "precision": {
+            "parent": {
+              "value": 0.5714285714285714,
+              "ci95": [
+                0.35,
+                0.782608695652174
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.5,
+              "ci95": [
+                0.2916666666666667,
+                0.7058823529411765
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": -0.0714285714285714,
+              "ci95": [
+                -0.1768421052631578,
+                0.0
+              ],
+              "valid_resamples": 2000
+            }
+          },
+          "threshold_accuracy": {
+            "parent": {
+              "value": 0.6857142857142857,
+              "ci95": [
+                0.5142857142857142,
+                0.8285714285714286
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.6,
+              "ci95": [
+                0.42857142857142855,
+                0.7714285714285715
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": -0.08571428571428574,
+              "ci95": [
+                -0.20000000000000007,
+                0.0
+              ],
+              "valid_resamples": 2000
+            }
+          }
+        }
+      }
+    },
+    "onward_request_body": {
+      "always_permit_baseline": {
+        "n": 35,
+        "positives": 1,
+        "negatives": 34,
+        "confusion": {
+          "tp": 0,
+          "fp": 0,
+          "tn": 34,
+          "fn": 1
+        },
+        "recall": 0.0,
+        "false_alert_rate": 0.0,
+        "precision": null,
+        "threshold_accuracy": 0.9714285714285714
+      },
+      "models": {
+        "parent": {
+          "questions": 35,
+          "groups": 35,
+          "raw": {
+            "n": 35,
+            "nll": 0.13064358196820833,
+            "acc": 0.9714285714285714,
+            "ece": 0.082650843776636,
+            "brier": 0.062386596130186385,
+            "mean_conf": 0.8959570370446475,
+            "confident_error_rate": 0.0,
+            "coverage_at_0_9": 0.6857142857142857,
+            "accuracy_at_0_9": 1.0,
+            "coverage_at_5pct_error": 1.0,
+            "coverage_at_1pct_error": 0.9428571428571428,
+            "aurc": 0.0016566626650660265,
+            "error_rate_at_0_9": 0.0,
+            "confidence_bias": -0.07547153438392395,
+            "top_bins": {
+              "0.9": {
+                "n": 24,
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.95": {
+                "n": 16,
+                "errors": 0,
+                "error_rate": 0.0
               },
-              {
-                "group": "b4edbf01a09fe17e2fc4eba076213094ab09b40f0ff73c9d3e5c9592f0f59353",
+              "0.99": {
                 "n": 7,
+                "errors": 0,
+                "error_rate": 0.0
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "bfe24bf96507627d9c2e0eb183e0cb607aa50a5195abded30b612c252216e7b0",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "confidence_cutoff": 0.9326748524721808
               },
-              {
-                "group": "cbdba8017701da737b2f23ee11598f766b99054550311935a5730acb5f7e57d9",
-                "n": 3,
+              "0.8": {
+                "coverage": 0.8,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "confidence_cutoff": 0.8734922079816011
+              }
+            }
+          },
+          "calibrated": {
+            "n": 35,
+            "nll": 0.20940801007318904,
+            "acc": 0.9714285714285714,
+            "ece": 0.14463259695441177,
+            "brier": 0.098760907973673,
+            "mean_conf": 0.8267959744741596,
+            "confident_error_rate": 0.0,
+            "coverage_at_0_9": 0.37142857142857144,
+            "accuracy_at_0_9": 1.0,
+            "coverage_at_5pct_error": 1.0,
+            "coverage_at_1pct_error": 0.9428571428571428,
+            "aurc": 0.0016566626650660265,
+            "error_rate_at_0_9": 0.0,
+            "confidence_bias": -0.14463259695441177,
+            "top_bins": {
+              "0.9": {
+                "n": 13,
+                "errors": 0,
+                "error_rate": 0.0
               },
-              {
-                "group": "cc3696bd6f2c8a2d65d8376338102471db2565326ab124811ce53c428e54425e",
-                "n": 4,
-                "accuracy": 0.5,
-                "true_positive_n": 4,
-                "true_positive_correct": 2
+              "0.95": {
+                "n": 7,
+                "errors": 0,
+                "error_rate": 0.0
               },
-              {
-                "group": "d592c0767849012441fe14ab7451f4371421ba152f26c3587463adcf58b911e5",
+              "0.99": {
                 "n": 1,
+                "errors": 0,
+                "error_rate": 0.0
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "confidence_cutoff": 0.8345254161812303
               },
-              {
-                "group": "f562314a5da425b50c820fa25a6756b09f54b042ff1a3a4b580647dd4c20aa7d",
-                "n": 20,
+              "0.8": {
+                "coverage": 0.8,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "confidence_cutoff": 0.7666347623294629
               }
-            ],
-            "correct": 127
-          }
-        },
-        "native": {
-          "raw": {
-            "n": 410,
-            "accuracy": 0.9975609756097561,
-            "nll": 0.012999284410721951,
-            "brier": 0.004191458920366933,
-            "correct": 409
-          },
-          "global_calibrated": {
-            "n": 410,
-            "accuracy": 0.9975609756097561,
-            "nll": 0.030200841306362223,
-            "brier": 0.006697687496008216,
-            "correct": 409
-          }
+            }
+          },
+          "argmax_classes": {
+            "requires_approval": {
+              "n": 1,
+              "correct": 1,
+              "recall": 1.0
+            },
+            "permitted": {
+              "n": 34,
+              "correct": 33,
+              "recall": 0.9705882352941176
+            }
+          },
+          "threshold": 0.8875371629397495,
+          "n": 35,
+          "positives": 1,
+          "negatives": 34,
+          "confusion": {
+            "tp": 1,
+            "fp": 0,
+            "tn": 34,
+            "fn": 0
+          },
+          "recall": 1.0,
+          "false_alert_rate": 0.0,
+          "precision": 1.0,
+          "threshold_accuracy": 1.0
         },
-        "guide_detector": {
+        "candidate": {
+          "questions": 35,
+          "groups": 35,
           "raw": {
-            "n": 1064,
-            "accuracy": 0.5855263157894737,
-            "nll": 0.8173590431098086,
-            "brier": 0.47423653600178534,
-            "macro_f1": 0.5415438082741008,
-            "class_order": [
-              "BenignPositive",
-              "FalsePositive",
-              "TruePositive"
-            ],
-            "per_class": {
-              "BenignPositive": {
-                "precision": 0.4096153846153846,
-                "recall": 0.7580071174377224,
-                "f1-score": 0.5318352059925093,
-                "support": 281.0
-              },
-              "FalsePositive": {
-                "precision": 0.7873831775700935,
-                "recall": 0.6686507936507936,
-                "f1-score": 0.723175965665236,
-                "support": 504.0
-              },
-              "TruePositive": {
-                "precision": 0.6293103448275862,
-                "recall": 0.2616487455197133,
-                "f1-score": 0.369620253164557,
-                "support": 279.0
+            "n": 35,
+            "nll": 0.005026554467109394,
+            "acc": 1.0,
+            "ece": 0.004983623817000349,
+            "brier": 0.0001685963296296438,
+            "mean_conf": 0.9950163761829997,
+            "confident_error_rate": 0.0,
+            "coverage_at_0_9": 1.0,
+            "accuracy_at_0_9": 1.0,
+            "coverage_at_5pct_error": 1.0,
+            "coverage_at_1pct_error": 1.0,
+            "aurc": 0.0,
+            "error_rate_at_0_9": 0.0,
+            "confidence_bias": -0.004983623817000349,
+            "top_bins": {
+              "0.9": {
+                "n": 35,
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.95": {
+                "n": 35,
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.99": {
+                "n": 31,
+                "errors": 0,
+                "error_rate": 0.0
               }
             },
-            "confusion": [
-              [
-                213,
-                60,
-                8
-              ],
-              [
-                132,
-                337,
-                35
-              ],
-              [
-                175,
-                31,
-                73
-              ]
-            ],
-            "equal_organization_accuracy": 0.649889118396736,
-            "organization_results": [
-              {
-                "group": "0228d088db03da24994da8d12bb80e2925bf424387c12a0cab440661f61e53db",
-                "n": 2,
-                "accuracy": 0.0,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "03b2cff809e7692ca15f889834563f866b356cc60a41023c8611dd2b60bcf51a",
-                "n": 2,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "04eb81bd28a99e8d254c7a0a9c07234f55af1f9e4fe167269f504109495c29a5",
-                "n": 4,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "06646577f2484818ed5044762571197f87d3e79272459df5462e193f43ddd530",
-                "n": 1,
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "confidence_cutoff": 0.9974550323398608
               },
-              {
-                "group": "06f295d348152073a6bf3ba5485ca147a0c14bb94baf88391d465d7ea7e6eca9",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "08b767258688d27f07153aa89f0a2c2c6e8e426289043614eb48c2c881591192",
-                "n": 17,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "09890e550c1a9a583cf9aaa2a997c1bbc61162a5d36c8a891bbbc1a837dffc36",
-                "n": 20,
+              "0.8": {
+                "coverage": 0.8,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "0b8649cde9e63fe45d17a7941ef90f01d1b3068c464c013d3ff7e21047164b9c",
-                "n": 5,
-                "accuracy": 0.8,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "10f18e39bc3c8e8713158ad66e666edf9118b8b83fb7eb21e5e39476a75a3125",
-                "n": 12,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "14919c2e5da07951c425b98441bab1107d02cbcb0496a9148b184b1fea1eb4d2",
-                "n": 3,
-                "accuracy": 0.6666666666666666,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "14a068f7500ef993a7f2e4cfcd5e194c57217338de4e72cf848d710c20392e91",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 18,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "16cb7259665afd83462953cc90bf025352bab6f7210a59e5d86d49561f03fcbe",
-                "n": 3,
-                "accuracy": 0.3333333333333333,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "16fe8b7b9a0987bf861cee078b8763e8f75ec46733a6a59b61ffa8e2bd6e49f7",
-                "n": 5,
+                "confidence_cutoff": 0.9932663672241991
+              }
+            }
+          },
+          "calibrated": {
+            "n": 35,
+            "nll": 0.03405651326059574,
+            "acc": 1.0,
+            "ece": 0.033071042578584564,
+            "brier": 0.0037297689099056244,
+            "mean_conf": 0.9669289574214154,
+            "confident_error_rate": 0.0,
+            "coverage_at_0_9": 0.9428571428571428,
+            "accuracy_at_0_9": 1.0,
+            "coverage_at_5pct_error": 1.0,
+            "coverage_at_1pct_error": 1.0,
+            "aurc": 0.0,
+            "error_rate_at_0_9": 0.0,
+            "confidence_bias": -0.03307104257858462,
+            "top_bins": {
+              "0.9": {
+                "n": 33,
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.95": {
+                "n": 28,
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.99": {
+                "n": 6,
+                "errors": 0,
+                "error_rate": 0.0
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "confidence_cutoff": 0.9720891179694704
               },
-              {
-                "group": "17cd31c2cd2d2ccbabdfab902aee6e4e759167d126e1bcaf4d638c23b93272ac",
-                "n": 17,
+              "0.8": {
+                "coverage": 0.8,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "18aa7b3932a87bfdfc7e3f032e91b38999f82e037146754bd8bba044870170ac",
-                "n": 2,
+                "confidence_cutoff": 0.9511715473570618
+              }
+            }
+          },
+          "argmax_classes": {
+            "requires_approval": {
+              "n": 1,
+              "correct": 1,
+              "recall": 1.0
+            },
+            "permitted": {
+              "n": 34,
+              "correct": 34,
+              "recall": 1.0
+            }
+          },
+          "threshold": 0.8449815374033091,
+          "n": 35,
+          "positives": 1,
+          "negatives": 34,
+          "confusion": {
+            "tp": 1,
+            "fp": 0,
+            "tn": 34,
+            "fn": 0
+          },
+          "recall": 1.0,
+          "false_alert_rate": 0.0,
+          "precision": 1.0,
+          "threshold_accuracy": 1.0
+        }
+      },
+      "paired_threshold_intervals": {
+        "groups": 35,
+        "samples": 2000,
+        "seed": 20260927,
+        "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
+        "metrics": {
+          "recall": {
+            "parent": {
+              "value": 1.0,
+              "ci95": [
+                1.0,
+                1.0
+              ],
+              "valid_resamples": 1272
+            },
+            "candidate": {
+              "value": 1.0,
+              "ci95": [
+                1.0,
+                1.0
+              ],
+              "valid_resamples": 1272
+            },
+            "candidate_minus_parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 1272
+            }
+          },
+          "false_alert_rate": {
+            "parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 2000
+            }
+          },
+          "precision": {
+            "parent": {
+              "value": 1.0,
+              "ci95": [
+                1.0,
+                1.0
+              ],
+              "valid_resamples": 1272
+            },
+            "candidate": {
+              "value": 1.0,
+              "ci95": [
+                1.0,
+                1.0
+              ],
+              "valid_resamples": 1272
+            },
+            "candidate_minus_parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 1272
+            }
+          },
+          "threshold_accuracy": {
+            "parent": {
+              "value": 1.0,
+              "ci95": [
+                1.0,
+                1.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 1.0,
+              "ci95": [
+                1.0,
+                1.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 2000
+            }
+          }
+        }
+      }
+    },
+    "code_execution": {
+      "always_permit_baseline": {
+        "n": 35,
+        "positives": 0,
+        "negatives": 35,
+        "confusion": {
+          "tp": 0,
+          "fp": 0,
+          "tn": 35,
+          "fn": 0
+        },
+        "recall": null,
+        "false_alert_rate": 0.0,
+        "precision": null,
+        "threshold_accuracy": 1.0
+      },
+      "models": {
+        "parent": {
+          "questions": 35,
+          "groups": 35,
+          "raw": {
+            "n": 35,
+            "nll": 0.03495771061017535,
+            "acc": 1.0,
+            "ece": 0.03371089311351877,
+            "brier": 0.004643703343002468,
+            "mean_conf": 0.9662891068864811,
+            "confident_error_rate": 0.0,
+            "coverage_at_0_9": 0.9714285714285714,
+            "accuracy_at_0_9": 1.0,
+            "coverage_at_5pct_error": 1.0,
+            "coverage_at_1pct_error": 1.0,
+            "aurc": 0.0,
+            "error_rate_at_0_9": 0.0,
+            "confidence_bias": -0.033710893113518936,
+            "top_bins": {
+              "0.9": {
+                "n": 34,
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.95": {
+                "n": 27,
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.99": {
+                "n": 11,
+                "errors": 0,
+                "error_rate": 0.0
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "confidence_cutoff": 0.9787669092307943
               },
-              {
-                "group": "191618c5651b68d639db0f90f67315430ae08a9467ebc761ca10a54847034de5",
-                "n": 20,
-                "accuracy": 0.75,
-                "true_positive_n": 3,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "1e7cf0ee593015038b9cf7686c7db7604b7de8b52d0a8b556038b7795a59dd5a",
-                "n": 19,
-                "accuracy": 0.7894736842105263,
-                "true_positive_n": 4,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "23f95dce880a8574d254a19ac7f94f1284a556dfa61126a92844ba979083ede1",
-                "n": 2,
-                "accuracy": 0.5,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "2413f2cb2e9a29595c0278b868a30250d3573d487f99e35b3fe3836713d79e96",
-                "n": 2,
+              "0.8": {
+                "coverage": 0.8,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "280f0d43bcdf087751b7a2b90739860bc02c2cccea845f81fae7555cdd9d7745",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
+                "confidence_cutoff": 0.9455319719721502
+              }
+            }
+          },
+          "calibrated": {
+            "n": 35,
+            "nll": 0.10667969626313498,
+            "acc": 1.0,
+            "ece": 0.09922627250314853,
+            "brier": 0.02646181525533936,
+            "mean_conf": 0.9007737274968517,
+            "confident_error_rate": 0.0,
+            "coverage_at_0_9": 0.5714285714285714,
+            "accuracy_at_0_9": 1.0,
+            "coverage_at_5pct_error": 1.0,
+            "coverage_at_1pct_error": 1.0,
+            "aurc": 0.0,
+            "error_rate_at_0_9": 0.0,
+            "confidence_bias": -0.0992262725031483,
+            "top_bins": {
+              "0.9": {
+                "n": 20,
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.95": {
+                "n": 9,
+                "errors": 0,
+                "error_rate": 0.0
               },
-              {
-                "group": "2af6f963a1301ac7cbf959fb0f63442e6b1b28be622e0d74c6b94c1e459c8bf0",
-                "n": 20,
+              "0.99": {
+                "n": 0,
+                "errors": 0,
+                "error_rate": null
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "confidence_cutoff": 0.9135753218620322
               },
-              {
-                "group": "2d4fd6af7767d8fbcfeddff36a76b5c993a49d5e45a515806ec30b1b53afa2ce",
-                "n": 1,
+              "0.8": {
+                "coverage": 0.8,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3229724a48ff16d1ea5cff92fcbc1a29fb636760780576606a1f931cba64c7e6",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "33593514e9eb3b4a2f11fbff9a900e304d7f7088413e1a6ed36210de11e1a7cd",
-                "n": 1,
+                "confidence_cutoff": 0.8528241762602075
+              }
+            }
+          },
+          "argmax_classes": {
+            "requires_approval": {
+              "n": 0,
+              "correct": 0,
+              "recall": null
+            },
+            "permitted": {
+              "n": 35,
+              "correct": 35,
+              "recall": 1.0
+            }
+          },
+          "threshold": 0.8875371629397495,
+          "n": 35,
+          "positives": 0,
+          "negatives": 35,
+          "confusion": {
+            "tp": 0,
+            "fp": 0,
+            "tn": 35,
+            "fn": 0
+          },
+          "recall": null,
+          "false_alert_rate": 0.0,
+          "precision": null,
+          "threshold_accuracy": 1.0
+        },
+        "candidate": {
+          "questions": 35,
+          "groups": 35,
+          "raw": {
+            "n": 35,
+            "nll": 0.003713040174857426,
+            "acc": 1.0,
+            "ece": 0.0036946893416571758,
+            "brier": 7.269134859423068e-05,
+            "mean_conf": 0.9963053106583428,
+            "confident_error_rate": 0.0,
+            "coverage_at_0_9": 1.0,
+            "accuracy_at_0_9": 1.0,
+            "coverage_at_5pct_error": 1.0,
+            "coverage_at_1pct_error": 1.0,
+            "aurc": 0.0,
+            "error_rate_at_0_9": 0.0,
+            "confidence_bias": -0.0036946893416571758,
+            "top_bins": {
+              "0.9": {
+                "n": 35,
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.95": {
+                "n": 35,
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.99": {
+                "n": 31,
+                "errors": 0,
+                "error_rate": 0.0
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "357e4fcd5f01346ea07a7b6805f5e0e94f11cd39d870ebde997c06b3ca23403b",
-                "n": 20,
-                "accuracy": 0.4,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "36be30d9fffc6077ec297a0ccde6690eb34709163c58b4b5df24236dd44c0e91",
-                "n": 18,
-                "accuracy": 0.9444444444444444,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "confidence_cutoff": 0.9983259960008511
               },
-              {
-                "group": "3788764c156c1d8107b5e65fc226f169b6cfeb28e92341b3776e0323894d1182",
-                "n": 20,
+              "0.8": {
+                "coverage": 0.8,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "397381ff8e300cf10164a627a5a6e614e68a7fb2a4dbac6daa5c67e331248cb3",
-                "n": 2,
+                "confidence_cutoff": 0.9949578958858085
+              }
+            }
+          },
+          "calibrated": {
+            "n": 35,
+            "nll": 0.029845462374630014,
+            "acc": 1.0,
+            "ece": 0.02916307563639764,
+            "brier": 0.002620264819563586,
+            "mean_conf": 0.9708369243636024,
+            "confident_error_rate": 0.0,
+            "coverage_at_0_9": 1.0,
+            "accuracy_at_0_9": 1.0,
+            "coverage_at_5pct_error": 1.0,
+            "coverage_at_1pct_error": 1.0,
+            "aurc": 0.0,
+            "error_rate_at_0_9": 0.0,
+            "confidence_bias": -0.02916307563639764,
+            "top_bins": {
+              "0.9": {
+                "n": 35,
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.95": {
+                "n": 30,
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.99": {
+                "n": 5,
+                "errors": 0,
+                "error_rate": 0.0
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3a3a67375cabf27ee46ffe17a8e3122e2bc0d77c22a4bc5fd3caf21af2fdf827",
-                "n": 9,
-                "accuracy": 0.8888888888888888,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "3ba2e76ba41ace83ac5fc70feb81f7bd790ebcc01cef912db9a520e0659301e6",
-                "n": 20,
-                "accuracy": 0.3,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3c519ebea078526e0c7724d67b71d0e9d928d62adbb68de9cc3ab45caeab0ba6",
-                "n": 20,
-                "accuracy": 0.35,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "confidence_cutoff": 0.9781193495148215
               },
-              {
-                "group": "42a20c36367ded75633c56b5468efe4dab76b5242f7fc0fafe5de10ae8b9a078",
-                "n": 2,
+              "0.8": {
+                "coverage": 0.8,
                 "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "4f4986e7532d0ba637710cb0010206dd22720f9b35cc163fd2f833fc93c25334",
-                "n": 14,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "52ed406f643d4adbab570be68c48860e0e9f239fe00b7f1bd0c35e00868fc879",
-                "n": 4,
-                "accuracy": 0.5,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "557c22b05ccf851b94b1fc4571234d237e761ded29b151b3a3adcacd17f9ba00",
-                "n": 5,
-                "accuracy": 0.6,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "58f3e1e6f3eca7efe52c2f97b888733f6a541ca293e36d6e1a1f22c2332d05fb",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5d11eb324b736d8f810b14b708ea1982028cb405fb0fb2e3f20dab9ab5bb3fe1",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5d2fc2acbf5e2afa16d5ad87f1cce4b784cdd43ca2da54c3eea33521e52a5a81",
-                "n": 20,
-                "accuracy": 0.45,
-                "true_positive_n": 10,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5f21eec69194bcd2e80220b2701002f2da10e7cd2b04625674cdf69feb57dedd",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "61c9e0872a43ed30a90ab06f7a3e8b85d33f54355b964a7fd1ffa361531c5626",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "61e7fb88134413ce324703852a01f81dda0cb96d4ae07a44cf845c9b1de8baa0",
-                "n": 20,
-                "accuracy": 0.25,
-                "true_positive_n": 13,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6201b69130be0b440362fc95b4cc2ef6c4c856543d9beb9584d43899e72d0283",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "629dffdc159a2dbb566dab56a6f17419a63aa2b0815020338d686277ff90f623",
-                "n": 20,
-                "accuracy": 0.35,
-                "true_positive_n": 3,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "63b98074727eecae10e68ab106f9c5f2ec98bd2347e0c3eaa72d5557c1ae1b64",
-                "n": 7,
-                "accuracy": 0.42857142857142855,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "63deca8572354a01b85154c34b64122772c89023b59c7931064bb347d8389e7c",
+                "confidence_cutoff": 0.9586084771058523
+              }
+            }
+          },
+          "argmax_classes": {
+            "requires_approval": {
+              "n": 0,
+              "correct": 0,
+              "recall": null
+            },
+            "permitted": {
+              "n": 35,
+              "correct": 35,
+              "recall": 1.0
+            }
+          },
+          "threshold": 0.8449815374033091,
+          "n": 35,
+          "positives": 0,
+          "negatives": 35,
+          "confusion": {
+            "tp": 0,
+            "fp": 0,
+            "tn": 35,
+            "fn": 0
+          },
+          "recall": null,
+          "false_alert_rate": 0.0,
+          "precision": null,
+          "threshold_accuracy": 1.0
+        }
+      },
+      "paired_threshold_intervals": {
+        "groups": 35,
+        "samples": 2000,
+        "seed": 20260927,
+        "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
+        "metrics": {
+          "recall": {
+            "parent": {
+              "value": null,
+              "ci95": null,
+              "valid_resamples": 0
+            },
+            "candidate": {
+              "value": null,
+              "ci95": null,
+              "valid_resamples": 0
+            },
+            "candidate_minus_parent": {
+              "value": null,
+              "ci95": null,
+              "valid_resamples": 0
+            }
+          },
+          "false_alert_rate": {
+            "parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 2000
+            }
+          },
+          "precision": {
+            "parent": {
+              "value": null,
+              "ci95": null,
+              "valid_resamples": 0
+            },
+            "candidate": {
+              "value": null,
+              "ci95": null,
+              "valid_resamples": 0
+            },
+            "candidate_minus_parent": {
+              "value": null,
+              "ci95": null,
+              "valid_resamples": 0
+            }
+          },
+          "threshold_accuracy": {
+            "parent": {
+              "value": 1.0,
+              "ci95": [
+                1.0,
+                1.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 1.0,
+              "ci95": [
+                1.0,
+                1.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 2000
+            }
+          }
+        }
+      }
+    },
+    "console_output": {
+      "always_permit_baseline": {
+        "n": 35,
+        "positives": 0,
+        "negatives": 35,
+        "confusion": {
+          "tp": 0,
+          "fp": 0,
+          "tn": 35,
+          "fn": 0
+        },
+        "recall": null,
+        "false_alert_rate": 0.0,
+        "precision": null,
+        "threshold_accuracy": 1.0
+      },
+      "models": {
+        "parent": {
+          "questions": 35,
+          "groups": 35,
+          "raw": {
+            "n": 35,
+            "nll": 1.4669736827879394,
+            "acc": 0.4857142857142857,
+            "ece": 0.44969603628529076,
+            "brier": 0.8741636560106545,
+            "mean_conf": 0.8196176344527856,
+            "confident_error_rate": 0.2,
+            "coverage_at_0_9": 0.4,
+            "accuracy_at_0_9": 0.5,
+            "coverage_at_5pct_error": 0.0,
+            "coverage_at_1pct_error": 0.0,
+            "aurc": 0.5978180747557157,
+            "error_rate_at_0_9": 0.5,
+            "confidence_bias": 0.33390334873849986,
+            "top_bins": {
+              "0.9": {
                 "n": 14,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "659e4d7e826303db51376a34ed1ee1187305d8b4fcd2ba86de0f1df79cbe9d09",
-                "n": 20,
-                "accuracy": 0.95,
-                "true_positive_n": 20,
-                "true_positive_correct": 19
-              },
-              {
-                "group": "681a13deadc6006e0d4d996f2b006e35b1d71fdeaaebfe98db36b84ce9e1bb82",
-                "n": 9,
-                "accuracy": 0.8888888888888888,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6972efb6c1c8c5719fc446f6b4e9bbf812c8274c04ddce38ae95cf5b06e65e10",
-                "n": 12,
-                "accuracy": 0.9166666666666666,
-                "true_positive_n": 5,
-                "true_positive_correct": 5
-              },
-              {
-                "group": "6d2a8e5968c904fbeceb057dc6d374d5cd1ba38019b8d31da27cc658c12aa476",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6d5ea32830050e4ac2840eed05d6b46ec9d7a78f65fbf85d0742d83cc628fa9c",
-                "n": 20,
-                "accuracy": 0.35,
-                "true_positive_n": 7,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6d74b56adb5791668e05d491f03f083060be7b184f47660a3b9cc14d7a32b511",
-                "n": 13,
-                "accuracy": 1.0,
-                "true_positive_n": 13,
-                "true_positive_correct": 13
-              },
-              {
-                "group": "6e73fd09bd5175004650b33517e33671c77ebb649384e14ffd95309ac3e567a7",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6f07feecb02e317c2db269ed9ec22266aa9195e11612cf7362bcdd1f3c9a7d3c",
-                "n": 20,
-                "accuracy": 0.9,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6fc0443881957355817ec4b23a2d3027d6ce1ba949d0c313ac5134c5a33d7752",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "725e8bad3bf121140d3f1e8bb8625fac12766a1a7ebea88a432b39f8af246111",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "745320acbe1a562a24246846ea4e7c92fe29683dd9e75636a6183142147be543",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "77af40ba5530d0abf3b141de72a4c48f4d594f8bc50515d71d71bb0ef57ba861",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "798e40bc3bfcfea7bb0935e395494ac4836e1d49f294b2ca803f7e234a478493",
-                "n": 4,
-                "accuracy": 0.5,
-                "true_positive_n": 4,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "79e0a2063b1c1395f9032d7bcca4a7987b25c134851f2b7087535b66973f8767",
-                "n": 3,
-                "accuracy": 0.6666666666666666,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "7c2c1e933c21cfd9f0534226991df5542463f0d2c40138378024cbbabc1f205f",
-                "n": 20,
-                "accuracy": 0.55,
-                "true_positive_n": 7,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "7d092ad55211811544dc5993b69d906906fce9350e187508760b25a549bbc9ad",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 5,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8269675fe1c1fcb6500b8262056237a7eb277d8d7822e63cc1b335c9b3587400",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 20,
-                "true_positive_correct": 0
+                "errors": 7,
+                "error_rate": 0.5
               },
-              {
-                "group": "8433ad4a51679bb9fc21396e090e351720fb331d25292de538c1e1a8ca524bab",
+              "0.95": {
                 "n": 8,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8d353f585c30ae854c944e56bcab8cfff771273fefcde8d929e6faf7cb75488f",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8da5a2b78094becb52357364f9f1ea558c4052e869e9636c912fa88cce650666",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8e11912bbdeb4ce0b56ca82a5e0ad37a9159b78dd620607541dc93cb9ac8e02e",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8f019450f830cbc42ded96cce0b7b4c1ee38439a9e198406ffb38ba5e0524fd7",
-                "n": 4,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "90994dcfeb71c7a87a4784556f5bf0bdb0b542c15a5f6b92da96e8020c4f1bf1",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "94d54be3b552337f2fc772ce8ff09d2b668ec497130ded8b947506a120acfb4e",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 14,
-                "true_positive_correct": 14
+                "errors": 4,
+                "error_rate": 0.5
               },
-              {
-                "group": "97d7c3f3b10a157cdb2791ff320a99a7d1856f8bbdd7ee020ad9ddbe7ec14b8e",
-                "n": 13,
-                "accuracy": 0.0,
-                "true_positive_n": 13,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "97f8ab62aee5d5d53245a565b836e892a9a7cda46748be23097421abb76ee39a",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 6,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "99a049ff4aba4bc20df5f424b24efb9bab9118c192fd17f29b31ec910e497def",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "9b54c928a43ef6ff8b5b53cfeaad5e554106ed4d6718d08bc4f7440e968414af",
-                "n": 20,
-                "accuracy": 0.5,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9bcc91cc05b92cb30cead1b81acb706497144e4815da75642e99d3fd7ebfd154",
-                "n": 11,
-                "accuracy": 0.9090909090909091,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9d2f045465f59eb2bcb8a37818341ce3635a125bced7bfbcb7bf9acdd80c1a71",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 20,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9e08a63fef975cba0f7fe19b9931144a9480e43b5b78497f97cfe4079c916045",
-                "n": 10,
-                "accuracy": 0.6,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9e38ecba2d710c7a9e3c2bef6abd472773b0edae845c5af26f317a4f5a2a3bc2",
-                "n": 19,
-                "accuracy": 0.10526315789473684,
-                "true_positive_n": 7,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a04b59b050c050568baa9abccd205e73547f1e5111afaabdbcbe5949c3511ec6",
-                "n": 20,
-                "accuracy": 0.25,
-                "true_positive_n": 3,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a1e75cda474e69786231632ed64953c19d60658191ea63553964e1d27015a7ce",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a52745698cb48cb1a28ac9bf212d724174c320e4648c9bececd83422b3364c1d",
-                "n": 19,
-                "accuracy": 0.10526315789473684,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a5a3b2d9f1ca0af015ed0584c90b28830aa11e86e8311c7f8f32f26b6d4f2768",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a68f2269c6412f5885bdd26d3023dae3411476f5dd5b2e103355d4e1b0a41bb5",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a7b13e8d802fa5d584ad83d610714c858aab70e0c1f005324a2a91e7a369aed1",
-                "n": 16,
-                "accuracy": 0.0625,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a94db659229b95e557d461f6de9071da0762515528aa3a77d5cb870fcf0f914b",
-                "n": 20,
-                "accuracy": 0.75,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "aab21c537c79f9eee1a053d0d9f7fbea9eb1451e166156d59dd6c464104a7992",
-                "n": 10,
-                "accuracy": 0.7,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "abd52b07ebdf5d9235ca1239d9c3acbe23eb7cabdbc661e0dc93653479a095f0",
-                "n": 7,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "aefb87718e7fc3725adba69a4f789e9de7d638ec0a0d1f12fcff8a8735fb622c",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b095632bbe63fbdc009ed98e30a50899f6a8edc165e3be8a924a7f0f600be4c0",
-                "n": 20,
-                "accuracy": 0.4,
-                "true_positive_n": 18,
-                "true_positive_correct": 6
-              },
-              {
-                "group": "b1b47d1978d292c9b80319bd2b531d2f2af206e3bcabf9aa0c46472112638ba9",
+              "0.99": {
+                "n": 3,
+                "errors": 2,
+                "error_rate": 0.6666666666666666
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
+                "accuracy": 0.3888888888888889,
+                "confidence_cutoff": 0.8730975086069251
+              },
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.39285714285714285,
+                "confidence_cutoff": 0.6488618260771695
+              }
+            }
+          },
+          "calibrated": {
+            "n": 35,
+            "nll": 1.077831441012569,
+            "acc": 0.4857142857142857,
+            "ece": 0.4118859974171003,
+            "brier": 0.7555266144919474,
+            "mean_conf": 0.7474440931747798,
+            "confident_error_rate": 0.08571428571428572,
+            "coverage_at_0_9": 0.17142857142857143,
+            "accuracy_at_0_9": 0.5,
+            "coverage_at_5pct_error": 0.0,
+            "coverage_at_1pct_error": 0.0,
+            "aurc": 0.5978180747557157,
+            "error_rate_at_0_9": 0.5,
+            "confidence_bias": 0.26172980746049407,
+            "top_bins": {
+              "0.9": {
                 "n": 6,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b7fa0bdd60d2f2321a27ad4ef1c84b2258bb60cb55372ea8aa56643d6d7a6d2b",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 15,
-                "true_positive_correct": 0
+                "errors": 3,
+                "error_rate": 0.5
               },
-              {
-                "group": "b8b4a7b5769be3d4db661d88d40336ade544fcac1e5bd2ec6606c09f83e5263c",
-                "n": 7,
-                "accuracy": 0.7142857142857143,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b9063c091cb8deaab4ea7fc341aebc169033dfb16792a7f0f253ce89ce556510",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "bc9c57f088052889f45c0b709137c494d747464872fca72f16cfdde538281d64",
-                "n": 15,
-                "accuracy": 0.8666666666666667,
-                "true_positive_n": 6,
-                "true_positive_correct": 4
-              },
-              {
-                "group": "bd4aa774324b8affeb489e7b796b6f32a9a2f46442394a8f5042c1967eac05dc",
+              "0.95": {
                 "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "c0371b16a587352143c383ca9799634e07feb94a2df75ef8830adb86efe153c1",
-                "n": 7,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "c045d0efa4ee5ab2be1d352625f16f6fb9e5cfbe90e02574363e83cd27bc5109",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "c98451abd4304660daabdb977564d36fdad533b4458c467bd2b97d80a651955f",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "errors": 2,
+                "error_rate": 1.0
               },
-              {
-                "group": "cb85febf11725dacf437b764d64bf097c9c2bab08e74ac2eeadb496ccd51b5e8",
+              "0.99": {
+                "n": 0,
+                "errors": 0,
+                "error_rate": null
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
+                "accuracy": 0.3888888888888889,
+                "confidence_cutoff": 0.7662416927389056
+              },
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.39285714285714285,
+                "confidence_cutoff": 0.5933872468656349
+              }
+            }
+          },
+          "argmax_classes": {
+            "requires_approval": {
+              "n": 0,
+              "correct": 0,
+              "recall": null
+            },
+            "permitted": {
+              "n": 35,
+              "correct": 17,
+              "recall": 0.4857142857142857
+            }
+          },
+          "threshold": 0.8875371629397495,
+          "n": 35,
+          "positives": 0,
+          "negatives": 35,
+          "confusion": {
+            "tp": 0,
+            "fp": 3,
+            "tn": 32,
+            "fn": 0
+          },
+          "recall": null,
+          "false_alert_rate": 0.08571428571428572,
+          "precision": 0.0,
+          "threshold_accuracy": 0.9142857142857143
+        },
+        "candidate": {
+          "questions": 35,
+          "groups": 35,
+          "raw": {
+            "n": 35,
+            "nll": 0.4711593105222168,
+            "acc": 0.8571428571428571,
+            "ece": 0.10032786865714749,
+            "brier": 0.24921799740853026,
+            "mean_conf": 0.9258384369166115,
+            "confident_error_rate": 0.05714285714285714,
+            "coverage_at_0_9": 0.7714285714285715,
+            "accuracy_at_0_9": 0.9259259259259259,
+            "coverage_at_5pct_error": 0.2571428571428571,
+            "coverage_at_1pct_error": 0.2571428571428571,
+            "aurc": 0.08024957297121005,
+            "error_rate_at_0_9": 0.07407407407407407,
+            "confidence_bias": 0.0686955797737544,
+            "top_bins": {
+              "0.9": {
+                "n": 27,
+                "errors": 2,
+                "error_rate": 0.07407407407407407
+              },
+              "0.95": {
+                "n": 26,
+                "errors": 2,
+                "error_rate": 0.07692307692307693
+              },
+              "0.99": {
                 "n": 16,
-                "accuracy": 0.875,
-                "true_positive_n": 3,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "cfe165abb396d68223741b950f77ec55bd59d4c200bd310d7e9a956548c1590e",
-                "n": 8,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d530b979376d67460a1a3dcafcc95c91e1a51ce2d97124dc2050c2030ae223b7",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 12,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d5496d95b9bcd56d2d382354f44af6098e30a6b132c6bd2ccbac429ee48d7d3b",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d6e9a943c91ab8d80d913c89d053cf89f2fe4bcd7df1963f3fd5a70bf56f3deb",
-                "n": 12,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+                "errors": 2,
+                "error_rate": 0.125
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
+                "accuracy": 0.8888888888888888,
+                "confidence_cutoff": 0.9891438506037644
               },
-              {
-                "group": "d794d72182a0c6f8a3009e01f6d607875a60752bf7f05920823cc55729bcbf3c",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.9285714285714286,
+                "confidence_cutoff": 0.8757747622322081
+              }
+            }
+          },
+          "calibrated": {
+            "n": 35,
+            "nll": 0.3789301936796312,
+            "acc": 0.8571428571428571,
+            "ece": 0.11430695728032647,
+            "brier": 0.23203565927974715,
+            "mean_conf": 0.8718695845973715,
+            "confident_error_rate": 0.05714285714285714,
+            "coverage_at_0_9": 0.6857142857142857,
+            "accuracy_at_0_9": 0.9166666666666666,
+            "coverage_at_5pct_error": 0.2571428571428571,
+            "coverage_at_1pct_error": 0.2571428571428571,
+            "aurc": 0.08024957297121005,
+            "error_rate_at_0_9": 0.08333333333333333,
+            "confidence_bias": 0.014726727454514421,
+            "top_bins": {
+              "0.9": {
+                "n": 24,
+                "errors": 2,
+                "error_rate": 0.08333333333333333
+              },
+              "0.95": {
+                "n": 9,
+                "errors": 0,
+                "error_rate": 0.0
               },
-              {
-                "group": "d82b9f42323202feb6db7ed45f9990070e72e41705aeb5aa64d60a3fd7549f17",
+              "0.99": {
                 "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "dfc85bf0674a0000b0dfbcf621419502a19c4fffabf786b4a5e74a69776a1f9d",
-                "n": 5,
-                "accuracy": 0.4,
-                "true_positive_n": 3,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "e35b0e3bc4a28a3bcf49fd9fe5c6e3c6ece59e921ecabf1904c9459cb7b84ad1",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "e6e40718557ace0b1764c2b7ce7021ff2b42e61db43cd59c7a54132f69807b2d",
-                "n": 7,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "f5b769ca29423ada31602cdd07bba0ab057d937256622788fa0872f6fe095865",
-                "n": 4,
-                "accuracy": 0.75,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "f72e0a3b72341d46d7eb5ab932fcce07cf34702afa40d217c5e997bef74764be",
-                "n": 15,
-                "accuracy": 0.9333333333333333,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "f9476bc6e284b58a4d168d14b188b2ca1636ebe4a0adfcb0c40060bed41781c8",
-                "n": 3,
-                "accuracy": 0.3333333333333333,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "f97f37793bfaa1b487125bd7b1f34bff0fed50146a5606962c763220f17bb6de",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "fb39d2600c554fe3ac1eddb65704bc34c6102ca0fbb4a77744ac525869d35be4",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "fcd64091f029e53dd4a770300ecf446dbf5bca8cc267148f28b198584caeae46",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "fe703c5eb241d3542fcc57e11688b3641c4f1f0b9506a720b42a9b9031382154",
-                "n": 19,
-                "accuracy": 0.8947368421052632,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
+                "errors": 0,
+                "error_rate": 0.0
               }
-            ],
-            "correct": 623
-          },
-          "global_calibrated": {
-            "n": 1064,
-            "accuracy": 0.5855263157894737,
-            "nll": 0.7976013493654481,
-            "brier": 0.46891907838977726,
-            "macro_f1": 0.5415438082741008,
-            "class_order": [
-              "BenignPositive",
-              "FalsePositive",
-              "TruePositive"
-            ],
-            "per_class": {
-              "BenignPositive": {
-                "precision": 0.4096153846153846,
-                "recall": 0.7580071174377224,
-                "f1-score": 0.5318352059925093,
-                "support": 281.0
-              },
-              "FalsePositive": {
-                "precision": 0.7873831775700935,
-                "recall": 0.6686507936507936,
-                "f1-score": 0.723175965665236,
-                "support": 504.0
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5142857142857142,
+                "accuracy": 0.8888888888888888,
+                "confidence_cutoff": 0.9360109691977325
               },
-              "TruePositive": {
-                "precision": 0.6293103448275862,
-                "recall": 0.2616487455197133,
-                "f1-score": 0.369620253164557,
-                "support": 279.0
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.9285714285714286,
+                "confidence_cutoff": 0.7615630826924717
               }
+            }
+          },
+          "argmax_classes": {
+            "requires_approval": {
+              "n": 0,
+              "correct": 0,
+              "recall": null
             },
-            "confusion": [
-              [
-                213,
-                60,
-                8
-              ],
-              [
-                132,
-                337,
-                35
-              ],
-              [
-                175,
-                31,
-                73
-              ]
-            ],
-            "equal_organization_accuracy": 0.649889118396736,
-            "organization_results": [
-              {
-                "group": "0228d088db03da24994da8d12bb80e2925bf424387c12a0cab440661f61e53db",
-                "n": 2,
-                "accuracy": 0.0,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "03b2cff809e7692ca15f889834563f866b356cc60a41023c8611dd2b60bcf51a",
-                "n": 2,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "04eb81bd28a99e8d254c7a0a9c07234f55af1f9e4fe167269f504109495c29a5",
-                "n": 4,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "06646577f2484818ed5044762571197f87d3e79272459df5462e193f43ddd530",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "06f295d348152073a6bf3ba5485ca147a0c14bb94baf88391d465d7ea7e6eca9",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "08b767258688d27f07153aa89f0a2c2c6e8e426289043614eb48c2c881591192",
-                "n": 17,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "09890e550c1a9a583cf9aaa2a997c1bbc61162a5d36c8a891bbbc1a837dffc36",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "0b8649cde9e63fe45d17a7941ef90f01d1b3068c464c013d3ff7e21047164b9c",
-                "n": 5,
-                "accuracy": 0.8,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "10f18e39bc3c8e8713158ad66e666edf9118b8b83fb7eb21e5e39476a75a3125",
-                "n": 12,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "14919c2e5da07951c425b98441bab1107d02cbcb0496a9148b184b1fea1eb4d2",
-                "n": 3,
-                "accuracy": 0.6666666666666666,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "14a068f7500ef993a7f2e4cfcd5e194c57217338de4e72cf848d710c20392e91",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 18,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "16cb7259665afd83462953cc90bf025352bab6f7210a59e5d86d49561f03fcbe",
-                "n": 3,
-                "accuracy": 0.3333333333333333,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "16fe8b7b9a0987bf861cee078b8763e8f75ec46733a6a59b61ffa8e2bd6e49f7",
-                "n": 5,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "17cd31c2cd2d2ccbabdfab902aee6e4e759167d126e1bcaf4d638c23b93272ac",
-                "n": 17,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "18aa7b3932a87bfdfc7e3f032e91b38999f82e037146754bd8bba044870170ac",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "191618c5651b68d639db0f90f67315430ae08a9467ebc761ca10a54847034de5",
-                "n": 20,
+            "permitted": {
+              "n": 35,
+              "correct": 30,
+              "recall": 0.8571428571428571
+            }
+          },
+          "threshold": 0.8449815374033091,
+          "n": 35,
+          "positives": 0,
+          "negatives": 35,
+          "confusion": {
+            "tp": 0,
+            "fp": 2,
+            "tn": 33,
+            "fn": 0
+          },
+          "recall": null,
+          "false_alert_rate": 0.05714285714285714,
+          "precision": 0.0,
+          "threshold_accuracy": 0.9428571428571428
+        }
+      },
+      "paired_threshold_intervals": {
+        "groups": 35,
+        "samples": 2000,
+        "seed": 20260927,
+        "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
+        "metrics": {
+          "recall": {
+            "parent": {
+              "value": null,
+              "ci95": null,
+              "valid_resamples": 0
+            },
+            "candidate": {
+              "value": null,
+              "ci95": null,
+              "valid_resamples": 0
+            },
+            "candidate_minus_parent": {
+              "value": null,
+              "ci95": null,
+              "valid_resamples": 0
+            }
+          },
+          "false_alert_rate": {
+            "parent": {
+              "value": 0.08571428571428572,
+              "ci95": [
+                0.0,
+                0.2
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.05714285714285714,
+              "ci95": [
+                0.0,
+                0.14285714285714285
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": -0.028571428571428574,
+              "ci95": [
+                -0.08571428571428572,
+                0.0
+              ],
+              "valid_resamples": 2000
+            }
+          },
+          "precision": {
+            "parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 1918
+            },
+            "candidate": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 1747
+            },
+            "candidate_minus_parent": {
+              "value": 0.0,
+              "ci95": [
+                0.0,
+                0.0
+              ],
+              "valid_resamples": 1747
+            }
+          },
+          "threshold_accuracy": {
+            "parent": {
+              "value": 0.9142857142857143,
+              "ci95": [
+                0.8,
+                1.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.9428571428571428,
+              "ci95": [
+                0.8571428571428571,
+                1.0
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.02857142857142858,
+              "ci95": [
+                0.0,
+                0.08571428571428574
+              ],
+              "valid_resamples": 2000
+            }
+          }
+        }
+      }
+    }
+  },
+  "by_source": {
+    "sev_swarmtraces_response_policy": {
+      "always_permit_baseline": {
+        "n": 175,
+        "positives": 35,
+        "negatives": 140,
+        "confusion": {
+          "tp": 0,
+          "fp": 0,
+          "tn": 140,
+          "fn": 35
+        },
+        "recall": 0.0,
+        "false_alert_rate": 0.0,
+        "precision": null,
+        "threshold_accuracy": 0.8
+      },
+      "models": {
+        "parent": {
+          "questions": 175,
+          "groups": 35,
+          "raw": {
+            "n": 175,
+            "nll": 0.795502582803619,
+            "acc": 0.7257142857142858,
+            "ece": 0.1747005023605082,
+            "brier": 0.4505153855550131,
+            "mean_conf": 0.8840572965356658,
+            "confident_error_rate": 0.13714285714285715,
+            "coverage_at_0_9": 0.6457142857142857,
+            "accuracy_at_0_9": 0.7876106194690266,
+            "coverage_at_5pct_error": 0.02857142857142857,
+            "coverage_at_1pct_error": 0.02857142857142857,
+            "aurc": 0.1992292401228163,
+            "error_rate_at_0_9": 0.21238938053097345,
+            "confidence_bias": 0.15834301082138003,
+            "top_bins": {
+              "0.9": {
+                "n": 113,
+                "errors": 24,
+                "error_rate": 0.21238938053097345
+              },
+              "0.95": {
+                "n": 84,
+                "errors": 18,
+                "error_rate": 0.21428571428571427
+              },
+              "0.99": {
+                "n": 26,
+                "errors": 2,
+                "error_rate": 0.07692307692307693
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5028571428571429,
+                "accuracy": 0.7613636363636364,
+                "confidence_cutoff": 0.9457634807080788
+              },
+              "0.8": {
+                "coverage": 0.8,
                 "accuracy": 0.75,
-                "true_positive_n": 3,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "1e7cf0ee593015038b9cf7686c7db7604b7de8b52d0a8b556038b7795a59dd5a",
-                "n": 19,
-                "accuracy": 0.7894736842105263,
-                "true_positive_n": 4,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "23f95dce880a8574d254a19ac7f94f1284a556dfa61126a92844ba979083ede1",
-                "n": 2,
-                "accuracy": 0.5,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "2413f2cb2e9a29595c0278b868a30250d3573d487f99e35b3fe3836713d79e96",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "280f0d43bcdf087751b7a2b90739860bc02c2cccea845f81fae7555cdd9d7745",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "2af6f963a1301ac7cbf959fb0f63442e6b1b28be622e0d74c6b94c1e459c8bf0",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "2d4fd6af7767d8fbcfeddff36a76b5c993a49d5e45a515806ec30b1b53afa2ce",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3229724a48ff16d1ea5cff92fcbc1a29fb636760780576606a1f931cba64c7e6",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "33593514e9eb3b4a2f11fbff9a900e304d7f7088413e1a6ed36210de11e1a7cd",
+                "confidence_cutoff": 0.7938824081649869
+              }
+            }
+          },
+          "calibrated": {
+            "n": 175,
+            "nll": 0.6260474644745716,
+            "acc": 0.7257142857142858,
+            "ece": 0.10761147298493057,
+            "brier": 0.4111063010906543,
+            "mean_conf": 0.8140932667241441,
+            "confident_error_rate": 0.06857142857142857,
+            "coverage_at_0_9": 0.35428571428571426,
+            "accuracy_at_0_9": 0.8064516129032258,
+            "coverage_at_5pct_error": 0.02857142857142857,
+            "coverage_at_1pct_error": 0.02857142857142857,
+            "aurc": 0.1992292401228163,
+            "error_rate_at_0_9": 0.1935483870967742,
+            "confidence_bias": 0.0883789810098583,
+            "top_bins": {
+              "0.9": {
+                "n": 62,
+                "errors": 12,
+                "error_rate": 0.1935483870967742
+              },
+              "0.95": {
+                "n": 21,
+                "errors": 2,
+                "error_rate": 0.09523809523809523
+              },
+              "0.99": {
                 "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "357e4fcd5f01346ea07a7b6805f5e0e94f11cd39d870ebde997c06b3ca23403b",
-                "n": 20,
-                "accuracy": 0.4,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "36be30d9fffc6077ec297a0ccde6690eb34709163c58b4b5df24236dd44c0e91",
-                "n": 18,
-                "accuracy": 0.9444444444444444,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3788764c156c1d8107b5e65fc226f169b6cfeb28e92341b3776e0323894d1182",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "397381ff8e300cf10164a627a5a6e614e68a7fb2a4dbac6daa5c67e331248cb3",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3a3a67375cabf27ee46ffe17a8e3122e2bc0d77c22a4bc5fd3caf21af2fdf827",
-                "n": 9,
-                "accuracy": 0.8888888888888888,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "3ba2e76ba41ace83ac5fc70feb81f7bd790ebcc01cef912db9a520e0659301e6",
-                "n": 20,
-                "accuracy": 0.3,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3c519ebea078526e0c7724d67b71d0e9d928d62adbb68de9cc3ab45caeab0ba6",
-                "n": 20,
-                "accuracy": 0.35,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "42a20c36367ded75633c56b5468efe4dab76b5242f7fc0fafe5de10ae8b9a078",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "4f4986e7532d0ba637710cb0010206dd22720f9b35cc163fd2f833fc93c25334",
-                "n": 14,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "52ed406f643d4adbab570be68c48860e0e9f239fe00b7f1bd0c35e00868fc879",
-                "n": 4,
-                "accuracy": 0.5,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "557c22b05ccf851b94b1fc4571234d237e761ded29b151b3a3adcacd17f9ba00",
-                "n": 5,
-                "accuracy": 0.6,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "58f3e1e6f3eca7efe52c2f97b888733f6a541ca293e36d6e1a1f22c2332d05fb",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5d11eb324b736d8f810b14b708ea1982028cb405fb0fb2e3f20dab9ab5bb3fe1",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5d2fc2acbf5e2afa16d5ad87f1cce4b784cdd43ca2da54c3eea33521e52a5a81",
-                "n": 20,
-                "accuracy": 0.45,
-                "true_positive_n": 10,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5f21eec69194bcd2e80220b2701002f2da10e7cd2b04625674cdf69feb57dedd",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "61c9e0872a43ed30a90ab06f7a3e8b85d33f54355b964a7fd1ffa361531c5626",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "61e7fb88134413ce324703852a01f81dda0cb96d4ae07a44cf845c9b1de8baa0",
-                "n": 20,
-                "accuracy": 0.25,
-                "true_positive_n": 13,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6201b69130be0b440362fc95b4cc2ef6c4c856543d9beb9584d43899e72d0283",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "629dffdc159a2dbb566dab56a6f17419a63aa2b0815020338d686277ff90f623",
-                "n": 20,
-                "accuracy": 0.35,
-                "true_positive_n": 3,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "63b98074727eecae10e68ab106f9c5f2ec98bd2347e0c3eaa72d5557c1ae1b64",
-                "n": 7,
-                "accuracy": 0.42857142857142855,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "63deca8572354a01b85154c34b64122772c89023b59c7931064bb347d8389e7c",
+                "errors": 0,
+                "error_rate": 0.0
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5028571428571429,
+                "accuracy": 0.7613636363636364,
+                "confidence_cutoff": 0.8531718492565429
+              },
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.75,
+                "confidence_cutoff": 0.6963743924447515
+              }
+            }
+          },
+          "argmax_classes": {
+            "requires_approval": {
+              "n": 35,
+              "correct": 29,
+              "recall": 0.8285714285714286
+            },
+            "permitted": {
+              "n": 140,
+              "correct": 98,
+              "recall": 0.7
+            }
+          },
+          "threshold": 0.8875371629397495,
+          "n": 175,
+          "positives": 35,
+          "negatives": 140,
+          "confusion": {
+            "tp": 14,
+            "fp": 12,
+            "tn": 128,
+            "fn": 21
+          },
+          "recall": 0.4,
+          "false_alert_rate": 0.08571428571428572,
+          "precision": 0.5384615384615384,
+          "threshold_accuracy": 0.8114285714285714
+        },
+        "candidate": {
+          "questions": 175,
+          "groups": 35,
+          "raw": {
+            "n": 175,
+            "nll": 0.49844721765472333,
+            "acc": 0.84,
+            "ece": 0.11735638435435483,
+            "brier": 0.269672286103042,
+            "mean_conf": 0.9559888432162478,
+            "confident_error_rate": 0.10285714285714286,
+            "coverage_at_0_9": 0.8742857142857143,
+            "accuracy_at_0_9": 0.8823529411764706,
+            "coverage_at_5pct_error": 0.6114285714285714,
+            "coverage_at_1pct_error": 0.44,
+            "aurc": 0.044632779737602504,
+            "error_rate_at_0_9": 0.11764705882352941,
+            "confidence_bias": 0.1159888432162478,
+            "top_bins": {
+              "0.9": {
+                "n": 153,
+                "errors": 18,
+                "error_rate": 0.11764705882352941
+              },
+              "0.95": {
+                "n": 140,
+                "errors": 14,
+                "error_rate": 0.1
+              },
+              "0.99": {
+                "n": 92,
+                "errors": 3,
+                "error_rate": 0.03260869565217391
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5028571428571429,
+                "accuracy": 0.9772727272727273,
+                "confidence_cutoff": 0.9904485398763483
+              },
+              "0.8": {
+                "coverage": 0.8,
+                "accuracy": 0.9,
+                "confidence_cutoff": 0.9516673073011176
+              }
+            }
+          },
+          "calibrated": {
+            "n": 175,
+            "nll": 0.37433095036445935,
+            "acc": 0.84,
+            "ece": 0.06636288221880407,
+            "brier": 0.23938834727626476,
+            "mean_conf": 0.9048661883246669,
+            "confident_error_rate": 0.05142857142857143,
+            "coverage_at_0_9": 0.7142857142857143,
+            "accuracy_at_0_9": 0.928,
+            "coverage_at_5pct_error": 0.6114285714285714,
+            "coverage_at_1pct_error": 0.44,
+            "aurc": 0.044632779737602504,
+            "error_rate_at_0_9": 0.072,
+            "confidence_bias": 0.06486618832466695,
+            "top_bins": {
+              "0.9": {
+                "n": 125,
+                "errors": 9,
+                "error_rate": 0.072
+              },
+              "0.95": {
+                "n": 77,
+                "errors": 0,
+                "error_rate": 0.0
+              },
+              "0.99": {
                 "n": 14,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "659e4d7e826303db51376a34ed1ee1187305d8b4fcd2ba86de0f1df79cbe9d09",
-                "n": 20,
-                "accuracy": 0.95,
-                "true_positive_n": 20,
-                "true_positive_correct": 19
-              },
-              {
-                "group": "681a13deadc6006e0d4d996f2b006e35b1d71fdeaaebfe98db36b84ce9e1bb82",
-                "n": 9,
-                "accuracy": 0.8888888888888888,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6972efb6c1c8c5719fc446f6b4e9bbf812c8274c04ddce38ae95cf5b06e65e10",
-                "n": 12,
-                "accuracy": 0.9166666666666666,
-                "true_positive_n": 5,
-                "true_positive_correct": 5
-              },
-              {
-                "group": "6d2a8e5968c904fbeceb057dc6d374d5cd1ba38019b8d31da27cc658c12aa476",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6d5ea32830050e4ac2840eed05d6b46ec9d7a78f65fbf85d0742d83cc628fa9c",
-                "n": 20,
-                "accuracy": 0.35,
-                "true_positive_n": 7,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6d74b56adb5791668e05d491f03f083060be7b184f47660a3b9cc14d7a32b511",
-                "n": 13,
-                "accuracy": 1.0,
-                "true_positive_n": 13,
-                "true_positive_correct": 13
-              },
-              {
-                "group": "6e73fd09bd5175004650b33517e33671c77ebb649384e14ffd95309ac3e567a7",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6f07feecb02e317c2db269ed9ec22266aa9195e11612cf7362bcdd1f3c9a7d3c",
-                "n": 20,
+                "errors": 0,
+                "error_rate": 0.0
+              }
+            },
+            "selective": {
+              "0.5": {
+                "coverage": 0.5028571428571429,
+                "accuracy": 0.9772727272727273,
+                "confidence_cutoff": 0.9404661687170754
+              },
+              "0.8": {
+                "coverage": 0.8,
                 "accuracy": 0.9,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6fc0443881957355817ec4b23a2d3027d6ce1ba949d0c313ac5134c5a33d7752",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "725e8bad3bf121140d3f1e8bb8625fac12766a1a7ebea88a432b39f8af246111",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "745320acbe1a562a24246846ea4e7c92fe29683dd9e75636a6183142147be543",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "77af40ba5530d0abf3b141de72a4c48f4d594f8bc50515d71d71bb0ef57ba861",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "798e40bc3bfcfea7bb0935e395494ac4836e1d49f294b2ca803f7e234a478493",
-                "n": 4,
-                "accuracy": 0.5,
-                "true_positive_n": 4,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "79e0a2063b1c1395f9032d7bcca4a7987b25c134851f2b7087535b66973f8767",
-                "n": 3,
-                "accuracy": 0.6666666666666666,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "7c2c1e933c21cfd9f0534226991df5542463f0d2c40138378024cbbabc1f205f",
-                "n": 20,
-                "accuracy": 0.55,
-                "true_positive_n": 7,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "7d092ad55211811544dc5993b69d906906fce9350e187508760b25a549bbc9ad",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 5,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8269675fe1c1fcb6500b8262056237a7eb277d8d7822e63cc1b335c9b3587400",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 20,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8433ad4a51679bb9fc21396e090e351720fb331d25292de538c1e1a8ca524bab",
-                "n": 8,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8d353f585c30ae854c944e56bcab8cfff771273fefcde8d929e6faf7cb75488f",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8da5a2b78094becb52357364f9f1ea558c4052e869e9636c912fa88cce650666",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8e11912bbdeb4ce0b56ca82a5e0ad37a9159b78dd620607541dc93cb9ac8e02e",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8f019450f830cbc42ded96cce0b7b4c1ee38439a9e198406ffb38ba5e0524fd7",
-                "n": 4,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "90994dcfeb71c7a87a4784556f5bf0bdb0b542c15a5f6b92da96e8020c4f1bf1",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "94d54be3b552337f2fc772ce8ff09d2b668ec497130ded8b947506a120acfb4e",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 14,
-                "true_positive_correct": 14
-              },
-              {
-                "group": "97d7c3f3b10a157cdb2791ff320a99a7d1856f8bbdd7ee020ad9ddbe7ec14b8e",
-                "n": 13,
-                "accuracy": 0.0,
-                "true_positive_n": 13,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "97f8ab62aee5d5d53245a565b836e892a9a7cda46748be23097421abb76ee39a",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 6,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "99a049ff4aba4bc20df5f424b24efb9bab9118c192fd17f29b31ec910e497def",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "9b54c928a43ef6ff8b5b53cfeaad5e554106ed4d6718d08bc4f7440e968414af",
-                "n": 20,
-                "accuracy": 0.5,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9bcc91cc05b92cb30cead1b81acb706497144e4815da75642e99d3fd7ebfd154",
-                "n": 11,
-                "accuracy": 0.9090909090909091,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9d2f045465f59eb2bcb8a37818341ce3635a125bced7bfbcb7bf9acdd80c1a71",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 20,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9e08a63fef975cba0f7fe19b9931144a9480e43b5b78497f97cfe4079c916045",
-                "n": 10,
-                "accuracy": 0.6,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9e38ecba2d710c7a9e3c2bef6abd472773b0edae845c5af26f317a4f5a2a3bc2",
-                "n": 19,
-                "accuracy": 0.10526315789473684,
-                "true_positive_n": 7,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a04b59b050c050568baa9abccd205e73547f1e5111afaabdbcbe5949c3511ec6",
-                "n": 20,
-                "accuracy": 0.25,
-                "true_positive_n": 3,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a1e75cda474e69786231632ed64953c19d60658191ea63553964e1d27015a7ce",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a52745698cb48cb1a28ac9bf212d724174c320e4648c9bececd83422b3364c1d",
-                "n": 19,
-                "accuracy": 0.10526315789473684,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a5a3b2d9f1ca0af015ed0584c90b28830aa11e86e8311c7f8f32f26b6d4f2768",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a68f2269c6412f5885bdd26d3023dae3411476f5dd5b2e103355d4e1b0a41bb5",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a7b13e8d802fa5d584ad83d610714c858aab70e0c1f005324a2a91e7a369aed1",
-                "n": 16,
-                "accuracy": 0.0625,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a94db659229b95e557d461f6de9071da0762515528aa3a77d5cb870fcf0f914b",
-                "n": 20,
-                "accuracy": 0.75,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "aab21c537c79f9eee1a053d0d9f7fbea9eb1451e166156d59dd6c464104a7992",
-                "n": 10,
-                "accuracy": 0.7,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "abd52b07ebdf5d9235ca1239d9c3acbe23eb7cabdbc661e0dc93653479a095f0",
-                "n": 7,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "aefb87718e7fc3725adba69a4f789e9de7d638ec0a0d1f12fcff8a8735fb622c",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b095632bbe63fbdc009ed98e30a50899f6a8edc165e3be8a924a7f0f600be4c0",
-                "n": 20,
-                "accuracy": 0.4,
-                "true_positive_n": 18,
-                "true_positive_correct": 6
-              },
-              {
-                "group": "b1b47d1978d292c9b80319bd2b531d2f2af206e3bcabf9aa0c46472112638ba9",
-                "n": 6,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b7fa0bdd60d2f2321a27ad4ef1c84b2258bb60cb55372ea8aa56643d6d7a6d2b",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 15,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b8b4a7b5769be3d4db661d88d40336ade544fcac1e5bd2ec6606c09f83e5263c",
-                "n": 7,
-                "accuracy": 0.7142857142857143,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b9063c091cb8deaab4ea7fc341aebc169033dfb16792a7f0f253ce89ce556510",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "bc9c57f088052889f45c0b709137c494d747464872fca72f16cfdde538281d64",
-                "n": 15,
-                "accuracy": 0.8666666666666667,
-                "true_positive_n": 6,
-                "true_positive_correct": 4
-              },
-              {
-                "group": "bd4aa774324b8affeb489e7b796b6f32a9a2f46442394a8f5042c1967eac05dc",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "c0371b16a587352143c383ca9799634e07feb94a2df75ef8830adb86efe153c1",
-                "n": 7,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "c045d0efa4ee5ab2be1d352625f16f6fb9e5cfbe90e02574363e83cd27bc5109",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "c98451abd4304660daabdb977564d36fdad533b4458c467bd2b97d80a651955f",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "cb85febf11725dacf437b764d64bf097c9c2bab08e74ac2eeadb496ccd51b5e8",
-                "n": 16,
-                "accuracy": 0.875,
-                "true_positive_n": 3,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "cfe165abb396d68223741b950f77ec55bd59d4c200bd310d7e9a956548c1590e",
-                "n": 8,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d530b979376d67460a1a3dcafcc95c91e1a51ce2d97124dc2050c2030ae223b7",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 12,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d5496d95b9bcd56d2d382354f44af6098e30a6b132c6bd2ccbac429ee48d7d3b",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d6e9a943c91ab8d80d913c89d053cf89f2fe4bcd7df1963f3fd5a70bf56f3deb",
-                "n": 12,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d794d72182a0c6f8a3009e01f6d607875a60752bf7f05920823cc55729bcbf3c",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d82b9f42323202feb6db7ed45f9990070e72e41705aeb5aa64d60a3fd7549f17",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "dfc85bf0674a0000b0dfbcf621419502a19c4fffabf786b4a5e74a69776a1f9d",
-                "n": 5,
-                "accuracy": 0.4,
-                "true_positive_n": 3,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "e35b0e3bc4a28a3bcf49fd9fe5c6e3c6ece59e921ecabf1904c9459cb7b84ad1",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "e6e40718557ace0b1764c2b7ce7021ff2b42e61db43cd59c7a54132f69807b2d",
-                "n": 7,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "f5b769ca29423ada31602cdd07bba0ab057d937256622788fa0872f6fe095865",
-                "n": 4,
-                "accuracy": 0.75,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "f72e0a3b72341d46d7eb5ab932fcce07cf34702afa40d217c5e997bef74764be",
-                "n": 15,
-                "accuracy": 0.9333333333333333,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "f9476bc6e284b58a4d168d14b188b2ca1636ebe4a0adfcb0c40060bed41781c8",
-                "n": 3,
-                "accuracy": 0.3333333333333333,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "f97f37793bfaa1b487125bd7b1f34bff0fed50146a5606962c763220f17bb6de",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "fb39d2600c554fe3ac1eddb65704bc34c6102ca0fbb4a77744ac525869d35be4",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "fcd64091f029e53dd4a770300ecf446dbf5bca8cc267148f28b198584caeae46",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "fe703c5eb241d3542fcc57e11688b3641c4f1f0b9506a720b42a9b9031382154",
-                "n": 19,
-                "accuracy": 0.8947368421052632,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
+                "confidence_cutoff": 0.8547040294961724
               }
-            ],
-            "correct": 623
+            }
+          },
+          "argmax_classes": {
+            "requires_approval": {
+              "n": 35,
+              "correct": 35,
+              "recall": 1.0
+            },
+            "permitted": {
+              "n": 140,
+              "correct": 112,
+              "recall": 0.8
+            }
+          },
+          "threshold": 0.8449815374033091,
+          "n": 175,
+          "positives": 35,
+          "negatives": 140,
+          "confusion": {
+            "tp": 25,
+            "fp": 14,
+            "tn": 126,
+            "fn": 10
+          },
+          "recall": 0.7142857142857143,
+          "false_alert_rate": 0.1,
+          "precision": 0.6410256410256411,
+          "threshold_accuracy": 0.8628571428571429
+        }
+      },
+      "paired_threshold_intervals": {
+        "groups": 35,
+        "samples": 2000,
+        "seed": 20260927,
+        "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
+        "metrics": {
+          "recall": {
+            "parent": {
+              "value": 0.4,
+              "ci95": [
+                0.22857142857142856,
+                0.5714285714285714
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.7142857142857143,
+              "ci95": [
+                0.5714285714285714,
+                0.8571428571428571
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.3142857142857143,
+              "ci95": [
+                0.17142857142857143,
+                0.4857142857142857
+              ],
+              "valid_resamples": 2000
+            }
+          },
+          "false_alert_rate": {
+            "parent": {
+              "value": 0.08571428571428572,
+              "ci95": [
+                0.05,
+                0.12857142857142856
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.1,
+              "ci95": [
+                0.06428571428571428,
+                0.14285714285714285
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.01428571428571429,
+              "ci95": [
+                -0.014285714285714277,
+                0.04285714285714286
+              ],
+              "valid_resamples": 2000
+            }
+          },
+          "precision": {
+            "parent": {
+              "value": 0.5384615384615384,
+              "ci95": [
+                0.3665909090909091,
+                0.70979262672811
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.6410256410256411,
+              "ci95": [
+                0.55,
+                0.7435897435897436
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.10256410256410264,
+              "ci95": [
+                0.005701872682569551,
+                0.21591595300834426
+              ],
+              "valid_resamples": 2000
+            }
+          },
+          "threshold_accuracy": {
+            "parent": {
+              "value": 0.8114285714285714,
+              "ci95": [
+                0.7655714285714286,
+                0.8628571428571429
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate": {
+              "value": 0.8628571428571429,
+              "ci95": [
+                0.8228571428571428,
+                0.9028571428571428
+              ],
+              "valid_resamples": 2000
+            },
+            "candidate_minus_parent": {
+              "value": 0.05142857142857149,
+              "ci95": [
+                0.02285714285714291,
+                0.08000000000000007
+              ],
+              "valid_resamples": 2000
+            }
+          }
+        }
+      }
+    }
+  },
+  "retained_development_by_source": {
+    "parent": {
+      "excytin_file": {
+        "questions": 80,
+        "groups": 1,
+        "raw": {
+          "n": 80,
+          "nll": 0.0001342531538525746,
+          "acc": 1.0,
+          "ece": 0.0001340575450615722,
+          "brier": 7.773831743030022e-07,
+          "mean_conf": 0.9998659424549384,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.0001340575450615722,
+          "top_bins": {
+            "0.9": {
+              "n": 80,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 80,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 80,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9999710759158646
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9999066849313115
+            }
           }
         },
-        "swarm_static": {
-          "raw": {
-            "n": 354,
-            "accuracy": 0.9152542372881356,
-            "nll": 0.23980562862108395,
-            "brier": 0.11952531272784916,
-            "correct": 324
-          },
-          "global_calibrated": {
-            "n": 354,
-            "accuracy": 0.9152542372881356,
-            "nll": 0.23046829278993344,
-            "brier": 0.11775650997940981,
-            "correct": 324
-          },
-          "families": {
-            "binding": {
-              "raw": {
-                "n": 164,
-                "accuracy": 0.8292682926829268,
-                "nll": 0.4973668964039057,
-                "brier": 0.24305378453073645,
-                "correct": 136
-              },
-              "global_calibrated": {
-                "n": 164,
-                "accuracy": 0.8292682926829268,
-                "nll": 0.47756760916281,
-                "brier": 0.24123129045340497,
-                "correct": 136
-              }
+        "calibrated": {
+          "n": 80,
+          "nll": 0.0030639977354718724,
+          "acc": 1.0,
+          "ece": 0.0030486898704090715,
+          "brier": 5.723205099455454e-05,
+          "mean_conf": 0.9969513101295909,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.0030486898704090715,
+          "top_bins": {
+            "0.9": {
+              "n": 80,
+              "errors": 0,
+              "error_rate": 0.0
             },
-            "direct": {
-              "raw": {
-                "n": 134,
-                "accuracy": 1.0,
-                "nll": 0.0006701158955142708,
-                "brier": 3.3165775411476116e-05,
-                "correct": 134
-              },
-              "global_calibrated": {
-                "n": 134,
-                "accuracy": 1.0,
-                "nll": 0.0034830280523404688,
-                "brier": 0.0001921529648151749,
-                "correct": 134
-              }
+            "0.95": {
+              "n": 80,
+              "errors": 0,
+              "error_rate": 0.0
             },
-            "flow": {
-              "raw": {
-                "n": 56,
-                "accuracy": 0.9642857142857143,
-                "nll": 0.05773617842186206,
-                "brier": 0.04369099694129809,
-                "correct": 54
-              },
-              "global_calibrated": {
-                "n": 56,
-                "accuracy": 0.9642857142857143,
-                "nll": 0.04996360689146422,
-                "brier": 0.037466507161918496,
-                "correct": 54
-              }
+            "0.99": {
+              "n": 79,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9980559839388836
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9963993201639499
+            }
+          }
+        }
+      },
+      "excytin_parent_join": {
+        "questions": 78,
+        "groups": 1,
+        "raw": {
+          "n": 78,
+          "nll": 0.0019727127979591126,
+          "acc": 1.0,
+          "ece": 0.0019615703387183547,
+          "brier": 4.023721192309001e-05,
+          "mean_conf": 0.9980384296612816,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.0019615703387183547,
+          "top_bins": {
+            "0.9": {
+              "n": 78,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 78,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 76,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9996318703080018
+            },
+            "0.8": {
+              "coverage": 0.8076923076923077,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9974582021431176
             }
           }
         },
-        "manual": {
-          "raw": {
-            "n": 83,
-            "accuracy": 0.6987951807228916,
-            "nll": 1.2611921240040693,
-            "brier": 0.5153410721147468,
-            "correct": 58
-          },
-          "global_calibrated": {
-            "n": 83,
-            "accuracy": 0.6987951807228916,
-            "nll": 1.0302111739617255,
-            "brier": 0.48339882880272644,
-            "correct": 58
-          },
-          "families": {
-            "binding": {
-              "raw": {
-                "n": 40,
-                "accuracy": 0.625,
-                "nll": 1.3389280289714236,
-                "brier": 0.6296514707199694,
-                "correct": 25
-              },
-              "global_calibrated": {
-                "n": 40,
-                "accuracy": 0.625,
-                "nll": 1.1010460949559158,
-                "brier": 0.5735795406676166,
-                "correct": 25
-              }
+        "calibrated": {
+          "n": 78,
+          "nll": 0.01903091706838521,
+          "acc": 1.0,
+          "ece": 0.018568486423778917,
+          "brier": 0.0013900388516520708,
+          "mean_conf": 0.9814315135762209,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9871794871794872,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.018568486423779063,
+          "top_bins": {
+            "0.9": {
+              "n": 77,
+              "errors": 0,
+              "error_rate": 0.0
             },
-            "flow": {
-              "raw": {
-                "n": 40,
-                "accuracy": 0.75,
-                "nll": 1.2780391309680283,
-                "brier": 0.4396812526809053,
-                "correct": 30
-              },
-              "global_calibrated": {
-                "n": 40,
-                "accuracy": 0.75,
-                "nll": 1.0365517358773104,
-                "brier": 0.4294728191968137,
-                "correct": 30
-              }
+            "0.95": {
+              "n": 69,
+              "errors": 0,
+              "error_rate": 0.0
             },
-            "prerequisite": {
-              "raw": {
-                "n": 3,
-                "accuracy": 1.0,
-                "nll": 8.663158656148827e-05,
-                "brier": 1.649633063680738e-08,
-                "correct": 3
-              },
-              "global_calibrated": {
-                "n": 3,
-                "accuracy": 1.0,
-                "nll": 0.0012047351647270432,
-                "brier": 2.7986830271292114e-06,
-                "correct": 3
-              }
+            "0.99": {
+              "n": 46,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9916928553567559
+            },
+            "0.8": {
+              "coverage": 0.8076923076923077,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9693893850709232
+            }
+          }
+        }
+      },
+      "excytin_process": {
+        "questions": 120,
+        "groups": 1,
+        "raw": {
+          "n": 120,
+          "nll": 0.0005180266626209783,
+          "acc": 1.0,
+          "ece": 0.0005174316239614507,
+          "brier": 2.3119235778779676e-06,
+          "mean_conf": 0.9994825683760385,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.0005174316239614507,
+          "top_bins": {
+            "0.9": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9999134057097643
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.99901689132107
             }
           }
         },
-        "sysmon": {
-          "raw": {
-            "n": 64,
-            "accuracy": 0.96875,
-            "nll": 0.08465266759433944,
-            "brier": 0.044922383555011054,
-            "correct": 62
-          },
-          "global_calibrated": {
-            "n": 64,
-            "accuracy": 0.96875,
-            "nll": 0.10719175042876139,
-            "brier": 0.048442258840311225,
-            "correct": 62
+        "calibrated": {
+          "n": 120,
+          "nll": 0.007587462315793864,
+          "acc": 1.0,
+          "ece": 0.007523761830370401,
+          "brier": 0.00023474191911792246,
+          "mean_conf": 0.9924762381696296,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.007523761830370401,
+          "top_bins": {
+            "0.9": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 119,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 83,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9964206554980828
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9859276562719884
+            }
+          }
+        }
+      },
+      "excytin_signin": {
+        "questions": 120,
+        "groups": 1,
+        "raw": {
+          "n": 120,
+          "nll": 0.0013203461429938826,
+          "acc": 1.0,
+          "ece": 0.0012737143193113606,
+          "brier": 0.00013106658103070147,
+          "mean_conf": 0.9987262856806887,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9916666666666667,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.0012737143193113365,
+          "top_bins": {
+            "0.9": {
+              "n": 119,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 119,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 119,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9998413534198065
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9996110203225037
+            }
           }
         },
-        "general": {
-          "raw": {
-            "n": 105,
-            "accuracy": 0.819047619047619,
-            "nll": 0.5016458133964976,
-            "brier": 0.2771177005204226,
-            "correct": 86
-          },
-          "global_calibrated": {
-            "n": 105,
-            "accuracy": 0.819047619047619,
-            "nll": 0.4265669988063534,
-            "brier": 0.24785251439683742,
-            "correct": 86
+        "calibrated": {
+          "n": 120,
+          "nll": 0.009411479962491658,
+          "acc": 1.0,
+          "ece": 0.00903361007075822,
+          "brier": 0.0010010974637190823,
+          "mean_conf": 0.9909663899292418,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9916666666666667,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.009033610070758202,
+          "top_bins": {
+            "0.9": {
+              "n": 119,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 119,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 105,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9950297195670218
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9914608088731087
+            }
           }
         }
-      }
-    },
-    "candidate": {
-      "temperature": 1.6245047927124707,
-      "domains": {
-        "excytin": {
-          "raw": {
-            "n": 398,
-            "accuracy": 1.0,
-            "nll": 0.0009678808724190956,
-            "brier": 4.8256567178328636e-05,
-            "correct": 398
-          },
-          "global_calibrated": {
-            "n": 398,
-            "accuracy": 1.0,
-            "nll": 0.009470865385844369,
-            "brier": 0.0006565384935901168,
-            "correct": 398
+      },
+      "sev_guide_detector": {
+        "questions": 1064,
+        "groups": 114,
+        "raw": {
+          "n": 1064,
+          "nll": 0.8276776090882748,
+          "acc": 0.599624060150376,
+          "ece": 0.10621007339433168,
+          "brier": 0.48019952548397726,
+          "mean_conf": 0.7041447618951532,
+          "confident_error_rate": 0.012218045112781954,
+          "coverage_at_0_9": 0.32706766917293234,
+          "accuracy_at_0_9": 0.9626436781609196,
+          "coverage_at_5pct_error": 0.3524436090225564,
+          "coverage_at_1pct_error": 0.0,
+          "aurc": 0.19304798592127517,
+          "error_rate_at_0_9": 0.03735632183908046,
+          "confidence_bias": 0.10452070174477723,
+          "top_bins": {
+            "0.9": {
+              "n": 348,
+              "errors": 13,
+              "error_rate": 0.03735632183908046
+            },
+            "0.95": {
+              "n": 327,
+              "errors": 13,
+              "error_rate": 0.039755351681957186
+            },
+            "0.99": {
+              "n": 319,
+              "errors": 12,
+              "error_rate": 0.03761755485893417
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5075187969924813,
+              "accuracy": 0.8074074074074075,
+              "confidence_cutoff": 0.6273336504002459
+            },
+            "0.8": {
+              "coverage": 0.8016917293233082,
+              "accuracy": 0.6518171160609613,
+              "confidence_cutoff": 0.48816800774395525
+            }
           }
         },
-        "guide": {
-          "raw": {
-            "n": 231,
-            "accuracy": 0.5627705627705628,
-            "nll": 0.8563265185203435,
-            "brier": 0.5255142828500304,
-            "macro_f1": 0.4613423970886119,
-            "class_order": [
-              "BenignPositive",
-              "FalsePositive",
-              "TruePositive"
-            ],
-            "per_class": {
-              "BenignPositive": {
-                "precision": 0.3333333333333333,
-                "recall": 0.4915254237288136,
-                "f1-score": 0.3972602739726027,
-                "support": 59.0
-              },
-              "FalsePositive": {
-                "precision": 0.7175572519083969,
-                "recall": 0.6962962962962963,
-                "f1-score": 0.706766917293233,
-                "support": 135.0
-              },
-              "TruePositive": {
-                "precision": 0.5384615384615384,
-                "recall": 0.1891891891891892,
-                "f1-score": 0.28,
-                "support": 37.0
-              }
+        "calibrated": {
+          "n": 1064,
+          "nll": 0.785071927880167,
+          "acc": 0.599624060150376,
+          "ece": 0.05266334727942813,
+          "brier": 0.46075730837789786,
+          "mean_conf": 0.6304966279496853,
+          "confident_error_rate": 0.011278195488721804,
+          "coverage_at_0_9": 0.3007518796992481,
+          "accuracy_at_0_9": 0.9625,
+          "coverage_at_5pct_error": 0.3524436090225564,
+          "coverage_at_1pct_error": 0.0,
+          "aurc": 0.19295849854549843,
+          "error_rate_at_0_9": 0.0375,
+          "confidence_bias": 0.03087256779930936,
+          "top_bins": {
+            "0.9": {
+              "n": 320,
+              "errors": 12,
+              "error_rate": 0.0375
             },
-            "confusion": [
-              [
-                29,
-                27,
-                3
-              ],
-              [
-                38,
-                94,
-                3
-              ],
-              [
-                20,
-                10,
-                7
-              ]
-            ],
-            "equal_organization_accuracy": 0.6510869565217392,
-            "organization_results": [
-              {
-                "group": "0d1b7eba9c4d5a5852ef2805e5f1c1018c827e36bfb922fefa4d0a92d76f322c",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "108482877637fd1bbe809b2faddd12c9affef276cdb170667c0234a670019cd4",
-                "n": 20,
-                "accuracy": 0.45,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "13a6e1ec428dca68e3cc729ac008e19870e5caa60b3f2d8d013eeb5a4f25aa71",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "17515941d0c83a2c0515166a1503db5e3d24c63b53c0398a9c4f3345d3ef5eda",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "22e3ca4ff2df3a13f6db322d247b7b7f8742181d456be68cb84fb4dfbda7b993",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5667ea75698bc671a6165fb7604bf6d7b7874ba2ef04005f82edf2811b406b53",
-                "n": 20,
-                "accuracy": 0.65,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "5cd724f6f8c622cf83c9ee56eeb3e956e4c0428f4bb398608581d51b9f3bd4cf",
-                "n": 7,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5fd9d56568dd7e69c4871b023d3df17203f2eb6d9aaf7a808371a391407f9026",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "66288757ecaf36bf368cd8d4c43f454658be762a12957c52eae438ca3e727ad8",
-                "n": 8,
-                "accuracy": 0.375,
-                "true_positive_n": 6,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "6d53f455fa642b810781930d63db6b35443da067812f1bf329a37de938179aed",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "79707d8a1f7240a758d67ba5212ab21ef286647a4b71ba71c892f88341f3f4e6",
-                "n": 20,
-                "accuracy": 0.5,
-                "true_positive_n": 4,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "7b4cb8f8c017bd94da14e9460c49cdf44fa47f6eab9726918b70ebb7b2695ee1",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "7f55e216b81a67b2fcebf627b534d1d18d9d734f31cb41477725aaa041f6d42b",
-                "n": 20,
-                "accuracy": 0.6,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "974cf500c91fb4e80273489c70f89803fe8eaf2fc9d4e68a6aac899d0e580551",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 18,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "97dbafe0dc6d23395bb2008b78ccf190cf0710c18154cbbfc664b3d388086baf",
-                "n": 5,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a756d4fa124cf854b444faed605f6fee522c5418d960b8cd60a8a06c44ef56b0",
-                "n": 5,
-                "accuracy": 0.6,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "adcffe87f88661c654325a95ccbc252777e27ff83f947ff85ea77cf167ad57c0",
-                "n": 5,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b4edbf01a09fe17e2fc4eba076213094ab09b40f0ff73c9d3e5c9592f0f59353",
-                "n": 7,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "bfe24bf96507627d9c2e0eb183e0cb607aa50a5195abded30b612c252216e7b0",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "cbdba8017701da737b2f23ee11598f766b99054550311935a5730acb5f7e57d9",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "cc3696bd6f2c8a2d65d8376338102471db2565326ab124811ce53c428e54425e",
-                "n": 4,
-                "accuracy": 0.5,
-                "true_positive_n": 4,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "d592c0767849012441fe14ab7451f4371421ba152f26c3587463adcf58b911e5",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "f562314a5da425b50c820fa25a6756b09f54b042ff1a3a4b580647dd4c20aa7d",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              }
-            ],
-            "correct": 130
-          },
-          "global_calibrated": {
-            "n": 231,
-            "accuracy": 0.5627705627705628,
-            "nll": 0.7793526965750157,
-            "brier": 0.4718856823661165,
-            "macro_f1": 0.4613423970886119,
-            "class_order": [
-              "BenignPositive",
-              "FalsePositive",
-              "TruePositive"
-            ],
-            "per_class": {
-              "BenignPositive": {
-                "precision": 0.3333333333333333,
-                "recall": 0.4915254237288136,
-                "f1-score": 0.3972602739726027,
-                "support": 59.0
-              },
-              "FalsePositive": {
-                "precision": 0.7175572519083969,
-                "recall": 0.6962962962962963,
-                "f1-score": 0.706766917293233,
-                "support": 135.0
-              },
-              "TruePositive": {
-                "precision": 0.5384615384615384,
-                "recall": 0.1891891891891892,
-                "f1-score": 0.28,
-                "support": 37.0
-              }
+            "0.95": {
+              "n": 213,
+              "errors": 11,
+              "error_rate": 0.051643192488262914
             },
-            "confusion": [
-              [
-                29,
-                27,
-                3
-              ],
-              [
-                38,
-                94,
-                3
-              ],
-              [
-                20,
-                10,
-                7
-              ]
-            ],
-            "equal_organization_accuracy": 0.6510869565217392,
-            "organization_results": [
-              {
-                "group": "0d1b7eba9c4d5a5852ef2805e5f1c1018c827e36bfb922fefa4d0a92d76f322c",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "108482877637fd1bbe809b2faddd12c9affef276cdb170667c0234a670019cd4",
-                "n": 20,
-                "accuracy": 0.45,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "13a6e1ec428dca68e3cc729ac008e19870e5caa60b3f2d8d013eeb5a4f25aa71",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "17515941d0c83a2c0515166a1503db5e3d24c63b53c0398a9c4f3345d3ef5eda",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "22e3ca4ff2df3a13f6db322d247b7b7f8742181d456be68cb84fb4dfbda7b993",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5667ea75698bc671a6165fb7604bf6d7b7874ba2ef04005f82edf2811b406b53",
-                "n": 20,
-                "accuracy": 0.65,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "5cd724f6f8c622cf83c9ee56eeb3e956e4c0428f4bb398608581d51b9f3bd4cf",
-                "n": 7,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5fd9d56568dd7e69c4871b023d3df17203f2eb6d9aaf7a808371a391407f9026",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "66288757ecaf36bf368cd8d4c43f454658be762a12957c52eae438ca3e727ad8",
-                "n": 8,
-                "accuracy": 0.375,
-                "true_positive_n": 6,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "6d53f455fa642b810781930d63db6b35443da067812f1bf329a37de938179aed",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "79707d8a1f7240a758d67ba5212ab21ef286647a4b71ba71c892f88341f3f4e6",
-                "n": 20,
-                "accuracy": 0.5,
-                "true_positive_n": 4,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "7b4cb8f8c017bd94da14e9460c49cdf44fa47f6eab9726918b70ebb7b2695ee1",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "7f55e216b81a67b2fcebf627b534d1d18d9d734f31cb41477725aaa041f6d42b",
-                "n": 20,
-                "accuracy": 0.6,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "974cf500c91fb4e80273489c70f89803fe8eaf2fc9d4e68a6aac899d0e580551",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 18,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "97dbafe0dc6d23395bb2008b78ccf190cf0710c18154cbbfc664b3d388086baf",
-                "n": 5,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a756d4fa124cf854b444faed605f6fee522c5418d960b8cd60a8a06c44ef56b0",
-                "n": 5,
-                "accuracy": 0.6,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "adcffe87f88661c654325a95ccbc252777e27ff83f947ff85ea77cf167ad57c0",
-                "n": 5,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b4edbf01a09fe17e2fc4eba076213094ab09b40f0ff73c9d3e5c9592f0f59353",
-                "n": 7,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "bfe24bf96507627d9c2e0eb183e0cb607aa50a5195abded30b612c252216e7b0",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "cbdba8017701da737b2f23ee11598f766b99054550311935a5730acb5f7e57d9",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "cc3696bd6f2c8a2d65d8376338102471db2565326ab124811ce53c428e54425e",
-                "n": 4,
-                "accuracy": 0.5,
-                "true_positive_n": 4,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "d592c0767849012441fe14ab7451f4371421ba152f26c3587463adcf58b911e5",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "f562314a5da425b50c820fa25a6756b09f54b042ff1a3a4b580647dd4c20aa7d",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              }
-            ],
-            "correct": 130
+            "0.99": {
+              "n": 0,
+              "errors": 0,
+              "error_rate": null
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5075187969924813,
+              "accuracy": 0.8074074074074075,
+              "confidence_cutoff": 0.5144221593760727
+            },
+            "0.8": {
+              "coverage": 0.8016917293233082,
+              "accuracy": 0.6553341148886284,
+              "confidence_cutoff": 0.4322968853489006
+            }
+          }
+        }
+      },
+      "sev_guide_triage": {
+        "questions": 231,
+        "groups": 23,
+        "raw": {
+          "n": 231,
+          "nll": 0.8563265185203435,
+          "acc": 0.5627705627705628,
+          "ece": 0.1895007716411903,
+          "brier": 0.5255142828500304,
+          "mean_conf": 0.7444269666893102,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.35064935064935066,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.36363636363636365,
+          "coverage_at_1pct_error": 0.36363636363636365,
+          "aurc": 0.22468166386345326,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": 0.18165640391874738,
+          "top_bins": {
+            "0.9": {
+              "n": 81,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 80,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 79,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5021645021645021,
+              "accuracy": 0.75,
+              "confidence_cutoff": 0.7894512944300596
+            },
+            "0.8": {
+              "coverage": 0.8095238095238095,
+              "accuracy": 0.5775401069518716,
+              "confidence_cutoff": 0.49009219177135965
+            }
           }
         },
-        "native": {
-          "raw": {
-            "n": 410,
-            "accuracy": 0.9975609756097561,
-            "nll": 0.008457485545022507,
-            "brier": 0.0035344210430228854,
-            "correct": 409
-          },
-          "global_calibrated": {
-            "n": 410,
-            "accuracy": 0.9975609756097561,
-            "nll": 0.028769043116345983,
-            "brier": 0.006284804103393485,
-            "correct": 409
+        "calibrated": {
+          "n": 231,
+          "nll": 0.7793526965750157,
+          "acc": 0.5627705627705628,
+          "ece": 0.1664752401619534,
+          "brier": 0.4718856823661165,
+          "mean_conf": 0.6653812407670776,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.3463203463203463,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.36363636363636365,
+          "coverage_at_1pct_error": 0.36363636363636365,
+          "aurc": 0.22610167628363687,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": 0.1026106779965148,
+          "top_bins": {
+            "0.9": {
+              "n": 80,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 14,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 0,
+              "errors": 0,
+              "error_rate": null
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5021645021645021,
+              "accuracy": 0.75,
+              "confidence_cutoff": 0.6343620681380909
+            },
+            "0.8": {
+              "coverage": 0.8008658008658008,
+              "accuracy": 0.5675675675675675,
+              "confidence_cutoff": 0.4381830283799813
+            }
+          }
+        }
+      },
+      "sev_swarmtraces_static": {
+        "questions": 354,
+        "groups": 176,
+        "raw": {
+          "n": 354,
+          "nll": 0.06671770193368179,
+          "acc": 0.9745762711864406,
+          "ece": 0.008417033573491902,
+          "brier": 0.03589778460324393,
+          "mean_conf": 0.9740214622331307,
+          "confident_error_rate": 0.002824858757062147,
+          "coverage_at_0_9": 0.9322033898305084,
+          "accuracy_at_0_9": 0.996969696969697,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 0.9745762711864406,
+          "aurc": 0.0009317569638462815,
+          "error_rate_at_0_9": 0.0030303030303030303,
+          "confidence_bias": -0.0005548089533099487,
+          "top_bins": {
+            "0.9": {
+              "n": 330,
+              "errors": 1,
+              "error_rate": 0.0030303030303030303
+            },
+            "0.95": {
+              "n": 317,
+              "errors": 1,
+              "error_rate": 0.0031545741324921135
+            },
+            "0.99": {
+              "n": 292,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9998280676315141
+            },
+            "0.8": {
+              "coverage": 0.8022598870056498,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9930714713128618
+            }
           }
         },
-        "guide_detector": {
-          "raw": {
-            "n": 1064,
-            "accuracy": 0.599624060150376,
-            "nll": 0.8276776090882748,
-            "brier": 0.48019952548397726,
-            "macro_f1": 0.5681608349992006,
-            "class_order": [
-              "BenignPositive",
-              "FalsePositive",
-              "TruePositive"
-            ],
-            "per_class": {
-              "BenignPositive": {
-                "precision": 0.44772727272727275,
-                "recall": 0.701067615658363,
-                "f1-score": 0.5464632454923717,
-                "support": 281.0
-              },
-              "FalsePositive": {
-                "precision": 0.7606263982102909,
-                "recall": 0.6746031746031746,
-                "f1-score": 0.7150368033648791,
-                "support": 504.0
-              },
-              "TruePositive": {
-                "precision": 0.5706214689265536,
-                "recall": 0.36200716845878134,
-                "f1-score": 0.44298245614035087,
-                "support": 279.0
-              }
+        "calibrated": {
+          "n": 354,
+          "nll": 0.08603305952328294,
+          "acc": 0.9745762711864406,
+          "ece": 0.031999170395289905,
+          "brier": 0.03970483443698841,
+          "mean_conf": 0.9467552592594386,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.8615819209039548,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 0.9745762711864406,
+          "aurc": 0.0009516834613723162,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.027821011927002037,
+          "top_bins": {
+            "0.9": {
+              "n": 305,
+              "errors": 0,
+              "error_rate": 0.0
             },
-            "confusion": [
-              [
-                197,
-                55,
-                29
-              ],
-              [
-                117,
-                340,
-                47
-              ],
-              [
-                126,
-                52,
-                101
-              ]
-            ],
-            "equal_organization_accuracy": 0.6743365093994473,
-            "organization_results": [
-              {
-                "group": "0228d088db03da24994da8d12bb80e2925bf424387c12a0cab440661f61e53db",
-                "n": 2,
-                "accuracy": 0.5,
-                "true_positive_n": 2,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "03b2cff809e7692ca15f889834563f866b356cc60a41023c8611dd2b60bcf51a",
-                "n": 2,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "04eb81bd28a99e8d254c7a0a9c07234f55af1f9e4fe167269f504109495c29a5",
-                "n": 4,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "06646577f2484818ed5044762571197f87d3e79272459df5462e193f43ddd530",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "06f295d348152073a6bf3ba5485ca147a0c14bb94baf88391d465d7ea7e6eca9",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "08b767258688d27f07153aa89f0a2c2c6e8e426289043614eb48c2c881591192",
-                "n": 17,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "09890e550c1a9a583cf9aaa2a997c1bbc61162a5d36c8a891bbbc1a837dffc36",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "0b8649cde9e63fe45d17a7941ef90f01d1b3068c464c013d3ff7e21047164b9c",
-                "n": 5,
-                "accuracy": 0.8,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "10f18e39bc3c8e8713158ad66e666edf9118b8b83fb7eb21e5e39476a75a3125",
-                "n": 12,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "14919c2e5da07951c425b98441bab1107d02cbcb0496a9148b184b1fea1eb4d2",
-                "n": 3,
-                "accuracy": 0.6666666666666666,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "14a068f7500ef993a7f2e4cfcd5e194c57217338de4e72cf848d710c20392e91",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 18,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "16cb7259665afd83462953cc90bf025352bab6f7210a59e5d86d49561f03fcbe",
-                "n": 3,
-                "accuracy": 0.3333333333333333,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "16fe8b7b9a0987bf861cee078b8763e8f75ec46733a6a59b61ffa8e2bd6e49f7",
-                "n": 5,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "17cd31c2cd2d2ccbabdfab902aee6e4e759167d126e1bcaf4d638c23b93272ac",
-                "n": 17,
-                "accuracy": 0.9411764705882353,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "18aa7b3932a87bfdfc7e3f032e91b38999f82e037146754bd8bba044870170ac",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "191618c5651b68d639db0f90f67315430ae08a9467ebc761ca10a54847034de5",
-                "n": 20,
-                "accuracy": 0.65,
-                "true_positive_n": 3,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "1e7cf0ee593015038b9cf7686c7db7604b7de8b52d0a8b556038b7795a59dd5a",
-                "n": 19,
-                "accuracy": 0.9473684210526315,
-                "true_positive_n": 4,
-                "true_positive_correct": 4
-              },
-              {
-                "group": "23f95dce880a8574d254a19ac7f94f1284a556dfa61126a92844ba979083ede1",
-                "n": 2,
-                "accuracy": 0.5,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "2413f2cb2e9a29595c0278b868a30250d3573d487f99e35b3fe3836713d79e96",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "280f0d43bcdf087751b7a2b90739860bc02c2cccea845f81fae7555cdd9d7745",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "2af6f963a1301ac7cbf959fb0f63442e6b1b28be622e0d74c6b94c1e459c8bf0",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "2d4fd6af7767d8fbcfeddff36a76b5c993a49d5e45a515806ec30b1b53afa2ce",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3229724a48ff16d1ea5cff92fcbc1a29fb636760780576606a1f931cba64c7e6",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "33593514e9eb3b4a2f11fbff9a900e304d7f7088413e1a6ed36210de11e1a7cd",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "357e4fcd5f01346ea07a7b6805f5e0e94f11cd39d870ebde997c06b3ca23403b",
-                "n": 20,
-                "accuracy": 0.3,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "36be30d9fffc6077ec297a0ccde6690eb34709163c58b4b5df24236dd44c0e91",
-                "n": 18,
-                "accuracy": 0.9444444444444444,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3788764c156c1d8107b5e65fc226f169b6cfeb28e92341b3776e0323894d1182",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "397381ff8e300cf10164a627a5a6e614e68a7fb2a4dbac6daa5c67e331248cb3",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3a3a67375cabf27ee46ffe17a8e3122e2bc0d77c22a4bc5fd3caf21af2fdf827",
-                "n": 9,
-                "accuracy": 0.8888888888888888,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "3ba2e76ba41ace83ac5fc70feb81f7bd790ebcc01cef912db9a520e0659301e6",
-                "n": 20,
-                "accuracy": 0.25,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3c519ebea078526e0c7724d67b71d0e9d928d62adbb68de9cc3ab45caeab0ba6",
-                "n": 20,
-                "accuracy": 0.35,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "42a20c36367ded75633c56b5468efe4dab76b5242f7fc0fafe5de10ae8b9a078",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "4f4986e7532d0ba637710cb0010206dd22720f9b35cc163fd2f833fc93c25334",
-                "n": 14,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "52ed406f643d4adbab570be68c48860e0e9f239fe00b7f1bd0c35e00868fc879",
-                "n": 4,
-                "accuracy": 0.5,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "557c22b05ccf851b94b1fc4571234d237e761ded29b151b3a3adcacd17f9ba00",
-                "n": 5,
-                "accuracy": 0.6,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "58f3e1e6f3eca7efe52c2f97b888733f6a541ca293e36d6e1a1f22c2332d05fb",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5d11eb324b736d8f810b14b708ea1982028cb405fb0fb2e3f20dab9ab5bb3fe1",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5d2fc2acbf5e2afa16d5ad87f1cce4b784cdd43ca2da54c3eea33521e52a5a81",
-                "n": 20,
-                "accuracy": 0.45,
-                "true_positive_n": 10,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5f21eec69194bcd2e80220b2701002f2da10e7cd2b04625674cdf69feb57dedd",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "61c9e0872a43ed30a90ab06f7a3e8b85d33f54355b964a7fd1ffa361531c5626",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "61e7fb88134413ce324703852a01f81dda0cb96d4ae07a44cf845c9b1de8baa0",
-                "n": 20,
-                "accuracy": 0.7,
-                "true_positive_n": 13,
-                "true_positive_correct": 9
-              },
-              {
-                "group": "6201b69130be0b440362fc95b4cc2ef6c4c856543d9beb9584d43899e72d0283",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "629dffdc159a2dbb566dab56a6f17419a63aa2b0815020338d686277ff90f623",
-                "n": 20,
-                "accuracy": 0.4,
-                "true_positive_n": 3,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "63b98074727eecae10e68ab106f9c5f2ec98bd2347e0c3eaa72d5557c1ae1b64",
-                "n": 7,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "63deca8572354a01b85154c34b64122772c89023b59c7931064bb347d8389e7c",
-                "n": 14,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "659e4d7e826303db51376a34ed1ee1187305d8b4fcd2ba86de0f1df79cbe9d09",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 20,
-                "true_positive_correct": 20
-              },
-              {
-                "group": "681a13deadc6006e0d4d996f2b006e35b1d71fdeaaebfe98db36b84ce9e1bb82",
-                "n": 9,
-                "accuracy": 0.8888888888888888,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6972efb6c1c8c5719fc446f6b4e9bbf812c8274c04ddce38ae95cf5b06e65e10",
-                "n": 12,
-                "accuracy": 0.9166666666666666,
-                "true_positive_n": 5,
-                "true_positive_correct": 5
-              },
-              {
-                "group": "6d2a8e5968c904fbeceb057dc6d374d5cd1ba38019b8d31da27cc658c12aa476",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6d5ea32830050e4ac2840eed05d6b46ec9d7a78f65fbf85d0742d83cc628fa9c",
-                "n": 20,
-                "accuracy": 0.4,
-                "true_positive_n": 7,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6d74b56adb5791668e05d491f03f083060be7b184f47660a3b9cc14d7a32b511",
-                "n": 13,
-                "accuracy": 1.0,
-                "true_positive_n": 13,
-                "true_positive_correct": 13
-              },
-              {
-                "group": "6e73fd09bd5175004650b33517e33671c77ebb649384e14ffd95309ac3e567a7",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6f07feecb02e317c2db269ed9ec22266aa9195e11612cf7362bcdd1f3c9a7d3c",
-                "n": 20,
-                "accuracy": 0.95,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6fc0443881957355817ec4b23a2d3027d6ce1ba949d0c313ac5134c5a33d7752",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "725e8bad3bf121140d3f1e8bb8625fac12766a1a7ebea88a432b39f8af246111",
-                "n": 20,
-                "accuracy": 0.95,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "745320acbe1a562a24246846ea4e7c92fe29683dd9e75636a6183142147be543",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "77af40ba5530d0abf3b141de72a4c48f4d594f8bc50515d71d71bb0ef57ba861",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "798e40bc3bfcfea7bb0935e395494ac4836e1d49f294b2ca803f7e234a478493",
-                "n": 4,
-                "accuracy": 0.5,
-                "true_positive_n": 4,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "79e0a2063b1c1395f9032d7bcca4a7987b25c134851f2b7087535b66973f8767",
-                "n": 3,
-                "accuracy": 0.3333333333333333,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "7c2c1e933c21cfd9f0534226991df5542463f0d2c40138378024cbbabc1f205f",
-                "n": 20,
-                "accuracy": 0.55,
-                "true_positive_n": 7,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "7d092ad55211811544dc5993b69d906906fce9350e187508760b25a549bbc9ad",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 5,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8269675fe1c1fcb6500b8262056237a7eb277d8d7822e63cc1b335c9b3587400",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 20,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8433ad4a51679bb9fc21396e090e351720fb331d25292de538c1e1a8ca524bab",
-                "n": 8,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8d353f585c30ae854c944e56bcab8cfff771273fefcde8d929e6faf7cb75488f",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8da5a2b78094becb52357364f9f1ea558c4052e869e9636c912fa88cce650666",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8e11912bbdeb4ce0b56ca82a5e0ad37a9159b78dd620607541dc93cb9ac8e02e",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8f019450f830cbc42ded96cce0b7b4c1ee38439a9e198406ffb38ba5e0524fd7",
-                "n": 4,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "90994dcfeb71c7a87a4784556f5bf0bdb0b542c15a5f6b92da96e8020c4f1bf1",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "94d54be3b552337f2fc772ce8ff09d2b668ec497130ded8b947506a120acfb4e",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 14,
-                "true_positive_correct": 14
-              },
-              {
-                "group": "97d7c3f3b10a157cdb2791ff320a99a7d1856f8bbdd7ee020ad9ddbe7ec14b8e",
-                "n": 13,
-                "accuracy": 0.15384615384615385,
-                "true_positive_n": 13,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "97f8ab62aee5d5d53245a565b836e892a9a7cda46748be23097421abb76ee39a",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 6,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "99a049ff4aba4bc20df5f424b24efb9bab9118c192fd17f29b31ec910e497def",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "9b54c928a43ef6ff8b5b53cfeaad5e554106ed4d6718d08bc4f7440e968414af",
-                "n": 20,
-                "accuracy": 0.5,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9bcc91cc05b92cb30cead1b81acb706497144e4815da75642e99d3fd7ebfd154",
-                "n": 11,
-                "accuracy": 0.9090909090909091,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9d2f045465f59eb2bcb8a37818341ce3635a125bced7bfbcb7bf9acdd80c1a71",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 20,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9e08a63fef975cba0f7fe19b9931144a9480e43b5b78497f97cfe4079c916045",
-                "n": 10,
-                "accuracy": 0.5,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9e38ecba2d710c7a9e3c2bef6abd472773b0edae845c5af26f317a4f5a2a3bc2",
-                "n": 19,
-                "accuracy": 0.42105263157894735,
-                "true_positive_n": 7,
-                "true_positive_correct": 6
-              },
-              {
-                "group": "a04b59b050c050568baa9abccd205e73547f1e5111afaabdbcbe5949c3511ec6",
-                "n": 20,
-                "accuracy": 0.05,
-                "true_positive_n": 3,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a1e75cda474e69786231632ed64953c19d60658191ea63553964e1d27015a7ce",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a52745698cb48cb1a28ac9bf212d724174c320e4648c9bececd83422b3364c1d",
-                "n": 19,
-                "accuracy": 0.10526315789473684,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a5a3b2d9f1ca0af015ed0584c90b28830aa11e86e8311c7f8f32f26b6d4f2768",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a68f2269c6412f5885bdd26d3023dae3411476f5dd5b2e103355d4e1b0a41bb5",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a7b13e8d802fa5d584ad83d610714c858aab70e0c1f005324a2a91e7a369aed1",
-                "n": 16,
-                "accuracy": 0.0625,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a94db659229b95e557d461f6de9071da0762515528aa3a77d5cb870fcf0f914b",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "aab21c537c79f9eee1a053d0d9f7fbea9eb1451e166156d59dd6c464104a7992",
-                "n": 10,
-                "accuracy": 0.5,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "abd52b07ebdf5d9235ca1239d9c3acbe23eb7cabdbc661e0dc93653479a095f0",
-                "n": 7,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "aefb87718e7fc3725adba69a4f789e9de7d638ec0a0d1f12fcff8a8735fb622c",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b095632bbe63fbdc009ed98e30a50899f6a8edc165e3be8a924a7f0f600be4c0",
-                "n": 20,
-                "accuracy": 0.45,
-                "true_positive_n": 18,
-                "true_positive_correct": 7
-              },
-              {
-                "group": "b1b47d1978d292c9b80319bd2b531d2f2af206e3bcabf9aa0c46472112638ba9",
-                "n": 6,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b7fa0bdd60d2f2321a27ad4ef1c84b2258bb60cb55372ea8aa56643d6d7a6d2b",
-                "n": 20,
-                "accuracy": 0.05,
-                "true_positive_n": 15,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b8b4a7b5769be3d4db661d88d40336ade544fcac1e5bd2ec6606c09f83e5263c",
-                "n": 7,
-                "accuracy": 0.7142857142857143,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b9063c091cb8deaab4ea7fc341aebc169033dfb16792a7f0f253ce89ce556510",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "bc9c57f088052889f45c0b709137c494d747464872fca72f16cfdde538281d64",
-                "n": 15,
-                "accuracy": 1.0,
-                "true_positive_n": 6,
-                "true_positive_correct": 6
-              },
-              {
-                "group": "bd4aa774324b8affeb489e7b796b6f32a9a2f46442394a8f5042c1967eac05dc",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "c0371b16a587352143c383ca9799634e07feb94a2df75ef8830adb86efe153c1",
-                "n": 7,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "c045d0efa4ee5ab2be1d352625f16f6fb9e5cfbe90e02574363e83cd27bc5109",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "c98451abd4304660daabdb977564d36fdad533b4458c467bd2b97d80a651955f",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "cb85febf11725dacf437b764d64bf097c9c2bab08e74ac2eeadb496ccd51b5e8",
-                "n": 16,
-                "accuracy": 0.875,
-                "true_positive_n": 3,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "cfe165abb396d68223741b950f77ec55bd59d4c200bd310d7e9a956548c1590e",
-                "n": 8,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d530b979376d67460a1a3dcafcc95c91e1a51ce2d97124dc2050c2030ae223b7",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 12,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d5496d95b9bcd56d2d382354f44af6098e30a6b132c6bd2ccbac429ee48d7d3b",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d6e9a943c91ab8d80d913c89d053cf89f2fe4bcd7df1963f3fd5a70bf56f3deb",
-                "n": 12,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d794d72182a0c6f8a3009e01f6d607875a60752bf7f05920823cc55729bcbf3c",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d82b9f42323202feb6db7ed45f9990070e72e41705aeb5aa64d60a3fd7549f17",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "dfc85bf0674a0000b0dfbcf621419502a19c4fffabf786b4a5e74a69776a1f9d",
-                "n": 5,
-                "accuracy": 0.4,
-                "true_positive_n": 3,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "e35b0e3bc4a28a3bcf49fd9fe5c6e3c6ece59e921ecabf1904c9459cb7b84ad1",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "e6e40718557ace0b1764c2b7ce7021ff2b42e61db43cd59c7a54132f69807b2d",
-                "n": 7,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "f5b769ca29423ada31602cdd07bba0ab057d937256622788fa0872f6fe095865",
-                "n": 4,
-                "accuracy": 0.75,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "f72e0a3b72341d46d7eb5ab932fcce07cf34702afa40d217c5e997bef74764be",
-                "n": 15,
-                "accuracy": 0.6666666666666666,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "f9476bc6e284b58a4d168d14b188b2ca1636ebe4a0adfcb0c40060bed41781c8",
-                "n": 3,
-                "accuracy": 0.3333333333333333,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "f97f37793bfaa1b487125bd7b1f34bff0fed50146a5606962c763220f17bb6de",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "fb39d2600c554fe3ac1eddb65704bc34c6102ca0fbb4a77744ac525869d35be4",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "fcd64091f029e53dd4a770300ecf446dbf5bca8cc267148f28b198584caeae46",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "fe703c5eb241d3542fcc57e11688b3641c4f1f0b9506a720b42a9b9031382154",
-                "n": 19,
-                "accuracy": 0.7368421052631579,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
-              }
-            ],
-            "correct": 638
-          },
-          "global_calibrated": {
-            "n": 1064,
-            "accuracy": 0.599624060150376,
-            "nll": 0.785071927880167,
-            "brier": 0.46075730837789786,
-            "macro_f1": 0.5681608349992006,
-            "class_order": [
-              "BenignPositive",
-              "FalsePositive",
-              "TruePositive"
-            ],
-            "per_class": {
-              "BenignPositive": {
-                "precision": 0.44772727272727275,
-                "recall": 0.701067615658363,
-                "f1-score": 0.5464632454923717,
-                "support": 281.0
-              },
-              "FalsePositive": {
-                "precision": 0.7606263982102909,
-                "recall": 0.6746031746031746,
-                "f1-score": 0.7150368033648791,
-                "support": 504.0
-              },
-              "TruePositive": {
-                "precision": 0.5706214689265536,
-                "recall": 0.36200716845878134,
-                "f1-score": 0.44298245614035087,
-                "support": 279.0
-              }
-            },
-            "confusion": [
-              [
-                197,
-                55,
-                29
-              ],
-              [
-                117,
-                340,
-                47
-              ],
-              [
-                126,
-                52,
-                101
-              ]
-            ],
-            "equal_organization_accuracy": 0.6743365093994473,
-            "organization_results": [
-              {
-                "group": "0228d088db03da24994da8d12bb80e2925bf424387c12a0cab440661f61e53db",
-                "n": 2,
-                "accuracy": 0.5,
-                "true_positive_n": 2,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "03b2cff809e7692ca15f889834563f866b356cc60a41023c8611dd2b60bcf51a",
-                "n": 2,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "04eb81bd28a99e8d254c7a0a9c07234f55af1f9e4fe167269f504109495c29a5",
-                "n": 4,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "06646577f2484818ed5044762571197f87d3e79272459df5462e193f43ddd530",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "06f295d348152073a6bf3ba5485ca147a0c14bb94baf88391d465d7ea7e6eca9",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "08b767258688d27f07153aa89f0a2c2c6e8e426289043614eb48c2c881591192",
-                "n": 17,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "09890e550c1a9a583cf9aaa2a997c1bbc61162a5d36c8a891bbbc1a837dffc36",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "0b8649cde9e63fe45d17a7941ef90f01d1b3068c464c013d3ff7e21047164b9c",
-                "n": 5,
-                "accuracy": 0.8,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "10f18e39bc3c8e8713158ad66e666edf9118b8b83fb7eb21e5e39476a75a3125",
-                "n": 12,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "14919c2e5da07951c425b98441bab1107d02cbcb0496a9148b184b1fea1eb4d2",
-                "n": 3,
-                "accuracy": 0.6666666666666666,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "14a068f7500ef993a7f2e4cfcd5e194c57217338de4e72cf848d710c20392e91",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 18,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "16cb7259665afd83462953cc90bf025352bab6f7210a59e5d86d49561f03fcbe",
-                "n": 3,
-                "accuracy": 0.3333333333333333,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "16fe8b7b9a0987bf861cee078b8763e8f75ec46733a6a59b61ffa8e2bd6e49f7",
-                "n": 5,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "17cd31c2cd2d2ccbabdfab902aee6e4e759167d126e1bcaf4d638c23b93272ac",
-                "n": 17,
-                "accuracy": 0.9411764705882353,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "18aa7b3932a87bfdfc7e3f032e91b38999f82e037146754bd8bba044870170ac",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "191618c5651b68d639db0f90f67315430ae08a9467ebc761ca10a54847034de5",
-                "n": 20,
-                "accuracy": 0.65,
-                "true_positive_n": 3,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "1e7cf0ee593015038b9cf7686c7db7604b7de8b52d0a8b556038b7795a59dd5a",
-                "n": 19,
-                "accuracy": 0.9473684210526315,
-                "true_positive_n": 4,
-                "true_positive_correct": 4
-              },
-              {
-                "group": "23f95dce880a8574d254a19ac7f94f1284a556dfa61126a92844ba979083ede1",
-                "n": 2,
-                "accuracy": 0.5,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "2413f2cb2e9a29595c0278b868a30250d3573d487f99e35b3fe3836713d79e96",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "280f0d43bcdf087751b7a2b90739860bc02c2cccea845f81fae7555cdd9d7745",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "2af6f963a1301ac7cbf959fb0f63442e6b1b28be622e0d74c6b94c1e459c8bf0",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "2d4fd6af7767d8fbcfeddff36a76b5c993a49d5e45a515806ec30b1b53afa2ce",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3229724a48ff16d1ea5cff92fcbc1a29fb636760780576606a1f931cba64c7e6",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "33593514e9eb3b4a2f11fbff9a900e304d7f7088413e1a6ed36210de11e1a7cd",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "357e4fcd5f01346ea07a7b6805f5e0e94f11cd39d870ebde997c06b3ca23403b",
-                "n": 20,
-                "accuracy": 0.3,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "36be30d9fffc6077ec297a0ccde6690eb34709163c58b4b5df24236dd44c0e91",
-                "n": 18,
-                "accuracy": 0.9444444444444444,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3788764c156c1d8107b5e65fc226f169b6cfeb28e92341b3776e0323894d1182",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "397381ff8e300cf10164a627a5a6e614e68a7fb2a4dbac6daa5c67e331248cb3",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3a3a67375cabf27ee46ffe17a8e3122e2bc0d77c22a4bc5fd3caf21af2fdf827",
-                "n": 9,
-                "accuracy": 0.8888888888888888,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "3ba2e76ba41ace83ac5fc70feb81f7bd790ebcc01cef912db9a520e0659301e6",
-                "n": 20,
-                "accuracy": 0.25,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "3c519ebea078526e0c7724d67b71d0e9d928d62adbb68de9cc3ab45caeab0ba6",
-                "n": 20,
-                "accuracy": 0.35,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "42a20c36367ded75633c56b5468efe4dab76b5242f7fc0fafe5de10ae8b9a078",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "4f4986e7532d0ba637710cb0010206dd22720f9b35cc163fd2f833fc93c25334",
-                "n": 14,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "52ed406f643d4adbab570be68c48860e0e9f239fe00b7f1bd0c35e00868fc879",
-                "n": 4,
-                "accuracy": 0.5,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "557c22b05ccf851b94b1fc4571234d237e761ded29b151b3a3adcacd17f9ba00",
-                "n": 5,
-                "accuracy": 0.6,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "58f3e1e6f3eca7efe52c2f97b888733f6a541ca293e36d6e1a1f22c2332d05fb",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5d11eb324b736d8f810b14b708ea1982028cb405fb0fb2e3f20dab9ab5bb3fe1",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5d2fc2acbf5e2afa16d5ad87f1cce4b784cdd43ca2da54c3eea33521e52a5a81",
-                "n": 20,
-                "accuracy": 0.45,
-                "true_positive_n": 10,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "5f21eec69194bcd2e80220b2701002f2da10e7cd2b04625674cdf69feb57dedd",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "61c9e0872a43ed30a90ab06f7a3e8b85d33f54355b964a7fd1ffa361531c5626",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "61e7fb88134413ce324703852a01f81dda0cb96d4ae07a44cf845c9b1de8baa0",
-                "n": 20,
-                "accuracy": 0.7,
-                "true_positive_n": 13,
-                "true_positive_correct": 9
-              },
-              {
-                "group": "6201b69130be0b440362fc95b4cc2ef6c4c856543d9beb9584d43899e72d0283",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "629dffdc159a2dbb566dab56a6f17419a63aa2b0815020338d686277ff90f623",
-                "n": 20,
-                "accuracy": 0.4,
-                "true_positive_n": 3,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "63b98074727eecae10e68ab106f9c5f2ec98bd2347e0c3eaa72d5557c1ae1b64",
-                "n": 7,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "63deca8572354a01b85154c34b64122772c89023b59c7931064bb347d8389e7c",
-                "n": 14,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "659e4d7e826303db51376a34ed1ee1187305d8b4fcd2ba86de0f1df79cbe9d09",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 20,
-                "true_positive_correct": 20
-              },
-              {
-                "group": "681a13deadc6006e0d4d996f2b006e35b1d71fdeaaebfe98db36b84ce9e1bb82",
-                "n": 9,
-                "accuracy": 0.8888888888888888,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6972efb6c1c8c5719fc446f6b4e9bbf812c8274c04ddce38ae95cf5b06e65e10",
-                "n": 12,
-                "accuracy": 0.9166666666666666,
-                "true_positive_n": 5,
-                "true_positive_correct": 5
-              },
-              {
-                "group": "6d2a8e5968c904fbeceb057dc6d374d5cd1ba38019b8d31da27cc658c12aa476",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6d5ea32830050e4ac2840eed05d6b46ec9d7a78f65fbf85d0742d83cc628fa9c",
-                "n": 20,
-                "accuracy": 0.4,
-                "true_positive_n": 7,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6d74b56adb5791668e05d491f03f083060be7b184f47660a3b9cc14d7a32b511",
-                "n": 13,
-                "accuracy": 1.0,
-                "true_positive_n": 13,
-                "true_positive_correct": 13
-              },
-              {
-                "group": "6e73fd09bd5175004650b33517e33671c77ebb649384e14ffd95309ac3e567a7",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6f07feecb02e317c2db269ed9ec22266aa9195e11612cf7362bcdd1f3c9a7d3c",
-                "n": 20,
-                "accuracy": 0.95,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "6fc0443881957355817ec4b23a2d3027d6ce1ba949d0c313ac5134c5a33d7752",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "725e8bad3bf121140d3f1e8bb8625fac12766a1a7ebea88a432b39f8af246111",
-                "n": 20,
-                "accuracy": 0.95,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "745320acbe1a562a24246846ea4e7c92fe29683dd9e75636a6183142147be543",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "77af40ba5530d0abf3b141de72a4c48f4d594f8bc50515d71d71bb0ef57ba861",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "798e40bc3bfcfea7bb0935e395494ac4836e1d49f294b2ca803f7e234a478493",
-                "n": 4,
-                "accuracy": 0.5,
-                "true_positive_n": 4,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "79e0a2063b1c1395f9032d7bcca4a7987b25c134851f2b7087535b66973f8767",
-                "n": 3,
-                "accuracy": 0.3333333333333333,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "7c2c1e933c21cfd9f0534226991df5542463f0d2c40138378024cbbabc1f205f",
-                "n": 20,
-                "accuracy": 0.55,
-                "true_positive_n": 7,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "7d092ad55211811544dc5993b69d906906fce9350e187508760b25a549bbc9ad",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 5,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8269675fe1c1fcb6500b8262056237a7eb277d8d7822e63cc1b335c9b3587400",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 20,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8433ad4a51679bb9fc21396e090e351720fb331d25292de538c1e1a8ca524bab",
-                "n": 8,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8d353f585c30ae854c944e56bcab8cfff771273fefcde8d929e6faf7cb75488f",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8da5a2b78094becb52357364f9f1ea558c4052e869e9636c912fa88cce650666",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8e11912bbdeb4ce0b56ca82a5e0ad37a9159b78dd620607541dc93cb9ac8e02e",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "8f019450f830cbc42ded96cce0b7b4c1ee38439a9e198406ffb38ba5e0524fd7",
-                "n": 4,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "90994dcfeb71c7a87a4784556f5bf0bdb0b542c15a5f6b92da96e8020c4f1bf1",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "94d54be3b552337f2fc772ce8ff09d2b668ec497130ded8b947506a120acfb4e",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 14,
-                "true_positive_correct": 14
-              },
-              {
-                "group": "97d7c3f3b10a157cdb2791ff320a99a7d1856f8bbdd7ee020ad9ddbe7ec14b8e",
-                "n": 13,
-                "accuracy": 0.15384615384615385,
-                "true_positive_n": 13,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "97f8ab62aee5d5d53245a565b836e892a9a7cda46748be23097421abb76ee39a",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 6,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "99a049ff4aba4bc20df5f424b24efb9bab9118c192fd17f29b31ec910e497def",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "9b54c928a43ef6ff8b5b53cfeaad5e554106ed4d6718d08bc4f7440e968414af",
-                "n": 20,
-                "accuracy": 0.5,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9bcc91cc05b92cb30cead1b81acb706497144e4815da75642e99d3fd7ebfd154",
-                "n": 11,
-                "accuracy": 0.9090909090909091,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9d2f045465f59eb2bcb8a37818341ce3635a125bced7bfbcb7bf9acdd80c1a71",
-                "n": 20,
-                "accuracy": 0.0,
-                "true_positive_n": 20,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9e08a63fef975cba0f7fe19b9931144a9480e43b5b78497f97cfe4079c916045",
-                "n": 10,
-                "accuracy": 0.5,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "9e38ecba2d710c7a9e3c2bef6abd472773b0edae845c5af26f317a4f5a2a3bc2",
-                "n": 19,
-                "accuracy": 0.42105263157894735,
-                "true_positive_n": 7,
-                "true_positive_correct": 6
-              },
-              {
-                "group": "a04b59b050c050568baa9abccd205e73547f1e5111afaabdbcbe5949c3511ec6",
-                "n": 20,
-                "accuracy": 0.05,
-                "true_positive_n": 3,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a1e75cda474e69786231632ed64953c19d60658191ea63553964e1d27015a7ce",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a52745698cb48cb1a28ac9bf212d724174c320e4648c9bececd83422b3364c1d",
-                "n": 19,
-                "accuracy": 0.10526315789473684,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a5a3b2d9f1ca0af015ed0584c90b28830aa11e86e8311c7f8f32f26b6d4f2768",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a68f2269c6412f5885bdd26d3023dae3411476f5dd5b2e103355d4e1b0a41bb5",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a7b13e8d802fa5d584ad83d610714c858aab70e0c1f005324a2a91e7a369aed1",
-                "n": 16,
-                "accuracy": 0.0625,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "a94db659229b95e557d461f6de9071da0762515528aa3a77d5cb870fcf0f914b",
-                "n": 20,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "aab21c537c79f9eee1a053d0d9f7fbea9eb1451e166156d59dd6c464104a7992",
-                "n": 10,
-                "accuracy": 0.5,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "abd52b07ebdf5d9235ca1239d9c3acbe23eb7cabdbc661e0dc93653479a095f0",
-                "n": 7,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "aefb87718e7fc3725adba69a4f789e9de7d638ec0a0d1f12fcff8a8735fb622c",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b095632bbe63fbdc009ed98e30a50899f6a8edc165e3be8a924a7f0f600be4c0",
-                "n": 20,
-                "accuracy": 0.45,
-                "true_positive_n": 18,
-                "true_positive_correct": 7
-              },
-              {
-                "group": "b1b47d1978d292c9b80319bd2b531d2f2af206e3bcabf9aa0c46472112638ba9",
-                "n": 6,
-                "accuracy": 0.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b7fa0bdd60d2f2321a27ad4ef1c84b2258bb60cb55372ea8aa56643d6d7a6d2b",
-                "n": 20,
-                "accuracy": 0.05,
-                "true_positive_n": 15,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b8b4a7b5769be3d4db661d88d40336ade544fcac1e5bd2ec6606c09f83e5263c",
-                "n": 7,
-                "accuracy": 0.7142857142857143,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "b9063c091cb8deaab4ea7fc341aebc169033dfb16792a7f0f253ce89ce556510",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "bc9c57f088052889f45c0b709137c494d747464872fca72f16cfdde538281d64",
-                "n": 15,
-                "accuracy": 1.0,
-                "true_positive_n": 6,
-                "true_positive_correct": 6
-              },
-              {
-                "group": "bd4aa774324b8affeb489e7b796b6f32a9a2f46442394a8f5042c1967eac05dc",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "c0371b16a587352143c383ca9799634e07feb94a2df75ef8830adb86efe153c1",
-                "n": 7,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "c045d0efa4ee5ab2be1d352625f16f6fb9e5cfbe90e02574363e83cd27bc5109",
-                "n": 1,
-                "accuracy": 0.0,
-                "true_positive_n": 1,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "c98451abd4304660daabdb977564d36fdad533b4458c467bd2b97d80a651955f",
-                "n": 3,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "cb85febf11725dacf437b764d64bf097c9c2bab08e74ac2eeadb496ccd51b5e8",
-                "n": 16,
-                "accuracy": 0.875,
-                "true_positive_n": 3,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "cfe165abb396d68223741b950f77ec55bd59d4c200bd310d7e9a956548c1590e",
-                "n": 8,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d530b979376d67460a1a3dcafcc95c91e1a51ce2d97124dc2050c2030ae223b7",
-                "n": 20,
-                "accuracy": 0.1,
-                "true_positive_n": 12,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d5496d95b9bcd56d2d382354f44af6098e30a6b132c6bd2ccbac429ee48d7d3b",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d6e9a943c91ab8d80d913c89d053cf89f2fe4bcd7df1963f3fd5a70bf56f3deb",
-                "n": 12,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d794d72182a0c6f8a3009e01f6d607875a60752bf7f05920823cc55729bcbf3c",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "d82b9f42323202feb6db7ed45f9990070e72e41705aeb5aa64d60a3fd7549f17",
-                "n": 2,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "dfc85bf0674a0000b0dfbcf621419502a19c4fffabf786b4a5e74a69776a1f9d",
-                "n": 5,
-                "accuracy": 0.4,
-                "true_positive_n": 3,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "e35b0e3bc4a28a3bcf49fd9fe5c6e3c6ece59e921ecabf1904c9459cb7b84ad1",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "e6e40718557ace0b1764c2b7ce7021ff2b42e61db43cd59c7a54132f69807b2d",
-                "n": 7,
-                "accuracy": 0.5714285714285714,
-                "true_positive_n": 1,
-                "true_positive_correct": 1
-              },
-              {
-                "group": "f5b769ca29423ada31602cdd07bba0ab057d937256622788fa0872f6fe095865",
-                "n": 4,
-                "accuracy": 0.75,
-                "true_positive_n": 2,
-                "true_positive_correct": 2
-              },
-              {
-                "group": "f72e0a3b72341d46d7eb5ab932fcce07cf34702afa40d217c5e997bef74764be",
-                "n": 15,
-                "accuracy": 0.6666666666666666,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "f9476bc6e284b58a4d168d14b188b2ca1636ebe4a0adfcb0c40060bed41781c8",
-                "n": 3,
-                "accuracy": 0.3333333333333333,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "f97f37793bfaa1b487125bd7b1f34bff0fed50146a5606962c763220f17bb6de",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "fb39d2600c554fe3ac1eddb65704bc34c6102ca0fbb4a77744ac525869d35be4",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "fcd64091f029e53dd4a770300ecf446dbf5bca8cc267148f28b198584caeae46",
-                "n": 1,
-                "accuracy": 1.0,
-                "true_positive_n": 0,
-                "true_positive_correct": 0
-              },
-              {
-                "group": "fe703c5eb241d3542fcc57e11688b3641c4f1f0b9506a720b42a9b9031382154",
-                "n": 19,
-                "accuracy": 0.7368421052631579,
-                "true_positive_n": 2,
-                "true_positive_correct": 0
-              }
-            ],
-            "correct": 638
+            "0.95": {
+              "n": 277,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 194,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9929179700515357
+            },
+            "0.8": {
+              "coverage": 0.8022598870056498,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9423198026275108
+            }
+          }
+        }
+      },
+      "sev_sysmon_native_file": {
+        "questions": 62,
+        "groups": 4,
+        "raw": {
+          "n": 62,
+          "nll": 0.010792211612586852,
+          "acc": 1.0,
+          "ece": 0.009954006393107204,
+          "brier": 0.002287040487090685,
+          "mean_conf": 0.9900459936068928,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9838709677419355,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.009954006393107173,
+          "top_bins": {
+            "0.9": {
+              "n": 61,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 59,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 54,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9986117420701618
+            },
+            "0.8": {
+              "coverage": 0.8064516129032258,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9936233886125292
+            }
           }
         },
-        "swarm_static": {
-          "raw": {
-            "n": 354,
-            "accuracy": 0.9745762711864406,
-            "nll": 0.06671770193368179,
-            "brier": 0.03589778460324393,
-            "correct": 345
-          },
-          "global_calibrated": {
-            "n": 354,
-            "accuracy": 0.9745762711864406,
-            "nll": 0.08603305952328294,
-            "brier": 0.03970483443698841,
-            "correct": 345
-          },
-          "families": {
-            "binding": {
-              "raw": {
-                "n": 164,
-                "accuracy": 0.9512195121951219,
-                "nll": 0.12986993405061373,
-                "brier": 0.06690797882642434,
-                "correct": 156
-              },
-              "global_calibrated": {
-                "n": 164,
-                "accuracy": 0.9512195121951219,
-                "nll": 0.17000841292483287,
-                "brier": 0.07668135319778648,
-                "correct": 156
-              }
+        "calibrated": {
+          "n": 62,
+          "nll": 0.04080521144437412,
+          "acc": 1.0,
+          "ece": 0.03730996348948351,
+          "brier": 0.008675036111100168,
+          "mean_conf": 0.9626900365105165,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9516129032258065,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.037309963489483455,
+          "top_bins": {
+            "0.9": {
+              "n": 59,
+              "errors": 0,
+              "error_rate": 0.0
             },
-            "direct": {
-              "raw": {
-                "n": 134,
-                "accuracy": 1.0,
-                "nll": 0.0002463532494033971,
-                "brier": 5.956582865744346e-06,
-                "correct": 134
-              },
-              "global_calibrated": {
-                "n": 134,
-                "accuracy": 1.0,
-                "nll": 0.003880609414537556,
-                "brier": 0.00015479616373253736,
-                "correct": 134
-              }
+            "0.95": {
+              "n": 48,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 23,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9803392906572079
+            },
+            "0.8": {
+              "coverage": 0.8064516129032258,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9477741924481837
+            }
+          }
+        }
+      },
+      "sev_sysmon_native_join": {
+        "questions": 120,
+        "groups": 3,
+        "raw": {
+          "n": 120,
+          "nll": 0.0008810538663341702,
+          "acc": 1.0,
+          "ece": 0.0008800625080441771,
+          "brier": 3.5216739059086607e-06,
+          "mean_conf": 0.9991199374919558,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.0008800625080441771,
+          "top_bins": {
+            "0.9": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9994848628990232
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.998641259167216
+            }
+          }
+        },
+        "calibrated": {
+          "n": 120,
+          "nll": 0.013824573239669817,
+          "acc": 1.0,
+          "ece": 0.013685808798396426,
+          "brier": 0.00044603592410257366,
+          "mean_conf": 0.9863141912016036,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.013685808798396426,
+          "top_bins": {
+            "0.9": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 118,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 55,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9888123032356468
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.980357236597309
+            }
+          }
+        }
+      },
+      "sev_sysmon_native_other_effect": {
+        "questions": 88,
+        "groups": 4,
+        "raw": {
+          "n": 88,
+          "nll": 0.007295006715728625,
+          "acc": 1.0,
+          "ece": 0.006888792886758961,
+          "brier": 0.001385599246512425,
+          "mean_conf": 0.9931112071132411,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9886363636363636,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.006888792886758921,
+          "top_bins": {
+            "0.9": {
+              "n": 87,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 86,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 79,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9993634031772723
+            },
+            "0.8": {
+              "coverage": 0.8068181818181818,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9952397949435474
+            }
+          }
+        },
+        "calibrated": {
+          "n": 88,
+          "nll": 0.03669008514897713,
+          "acc": 1.0,
+          "ece": 0.03458520458981973,
+          "brier": 0.006053607090618847,
+          "mean_conf": 0.9654147954101805,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9431818181818182,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.03458520458981951,
+          "top_bins": {
+            "0.9": {
+              "n": 83,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 67,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 41,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9865664502206583
+            },
+            "0.8": {
+              "coverage": 0.8068181818181818,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9454838788991825
+            }
+          }
+        }
+      },
+      "sev_sysmon_native_process": {
+        "questions": 74,
+        "groups": 4,
+        "raw": {
+          "n": 74,
+          "nll": 0.005360095133334598,
+          "acc": 1.0,
+          "ece": 0.005183972707978367,
+          "brier": 0.0006452896028548641,
+          "mean_conf": 0.9948160272920217,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9864864864864865,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.005183972707978324,
+          "top_bins": {
+            "0.9": {
+              "n": 73,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 72,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 67,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9998754065974452
+            },
+            "0.8": {
+              "coverage": 0.8108108108108109,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9964080071736937
+            }
+          }
+        },
+        "calibrated": {
+          "n": 74,
+          "nll": 0.02430026482825407,
+          "acc": 1.0,
+          "ece": 0.023069255552173623,
+          "brier": 0.004038757822471681,
+          "mean_conf": 0.9769307444478267,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9324324324324325,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.023069255552173318,
+          "top_bins": {
+            "0.9": {
+              "n": 69,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 65,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 43,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9959686665627235
+            },
+            "0.8": {
+              "coverage": 0.8108108108108109,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9671912008530428
+            }
+          }
+        }
+      },
+      "sev_sysmon_native_registry": {
+        "questions": 66,
+        "groups": 4,
+        "raw": {
+          "n": 66,
+          "nll": 0.02506239179799794,
+          "acc": 0.9848484848484849,
+          "ece": 0.005781042813278618,
+          "brier": 0.017230444731313608,
+          "mean_conf": 0.9868495288218273,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9696969696969697,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 0.9696969696969697,
+          "aurc": 0.00046266864448682633,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": 0.0020010439733424423,
+          "top_bins": {
+            "0.9": {
+              "n": 64,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 64,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 61,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9996958591947809
+            },
+            "0.8": {
+              "coverage": 0.803030303030303,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9967594448763499
+            }
+          }
+        },
+        "calibrated": {
+          "n": 66,
+          "nll": 0.03908316195438624,
+          "acc": 0.9848484848484849,
+          "ece": 0.018983776220710902,
+          "brier": 0.01748193378444616,
+          "mean_conf": 0.9706927611865057,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9545454545454546,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 0.9696969696969697,
+          "aurc": 0.00046266864448682633,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.014155723661979147,
+          "top_bins": {
+            "0.9": {
+              "n": 63,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 58,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 36,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9919306919122572
+            },
+            "0.8": {
+              "coverage": 0.803030303030303,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9688350009585398
+            }
+          }
+        }
+      }
+    },
+    "candidate": {
+      "excytin_file": {
+        "questions": 80,
+        "groups": 1,
+        "raw": {
+          "n": 80,
+          "nll": 8.416562133899781e-05,
+          "acc": 1.0,
+          "ece": 8.407504508700647e-05,
+          "brier": 3.608306194877327e-07,
+          "mean_conf": 0.999915924954913,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -8.407504508700647e-05,
+          "top_bins": {
+            "0.9": {
+              "n": 80,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 80,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 80,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9999870465451609
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9999445146924221
+            }
+          }
+        },
+        "calibrated": {
+          "n": 80,
+          "nll": 0.0025800477441721577,
+          "acc": 1.0,
+          "ece": 0.0025682388226171415,
+          "brier": 4.4624375778219304e-05,
+          "mean_conf": 0.9974317611773829,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.0025682388226171415,
+          "top_bins": {
+            "0.9": {
+              "n": 80,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 80,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 79,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9984996790217291
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9968816753727368
+            }
+          }
+        }
+      },
+      "excytin_parent_join": {
+        "questions": 78,
+        "groups": 1,
+        "raw": {
+          "n": 78,
+          "nll": 0.0010931532029286512,
+          "acc": 1.0,
+          "ece": 0.0010890559164937752,
+          "brier": 1.524968968159761e-05,
+          "mean_conf": 0.9989109440835062,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.0010890559164937752,
+          "top_bins": {
+            "0.9": {
+              "n": 78,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 78,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 77,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9997808486535813
+            },
+            "0.8": {
+              "coverage": 0.8076923076923077,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9985521003494622
+            }
+          }
+        },
+        "calibrated": {
+          "n": 78,
+          "nll": 0.014932533018017379,
+          "acc": 1.0,
+          "ece": 0.014655484687648102,
+          "brier": 0.0008621856358382366,
+          "mean_conf": 0.9853445153123519,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.014655484687648102,
+          "top_bins": {
+            "0.9": {
+              "n": 78,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 73,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 49,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9924833351572758
+            },
+            "0.8": {
+              "coverage": 0.8076923076923077,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9749372136312703
+            }
+          }
+        }
+      },
+      "excytin_process": {
+        "questions": 120,
+        "groups": 1,
+        "raw": {
+          "n": 120,
+          "nll": 0.00038924968205708544,
+          "acc": 1.0,
+          "ece": 0.00038892368944332123,
+          "brier": 1.2813112890567744e-06,
+          "mean_conf": 0.9996110763105567,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.00038892368944332123,
+          "top_bins": {
+            "0.9": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9999547557532524
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9992749850755753
+            }
+          }
+        },
+        "calibrated": {
+          "n": 120,
+          "nll": 0.007136704365819422,
+          "acc": 1.0,
+          "ece": 0.007078397430680172,
+          "brier": 0.00021856930435733632,
+          "mean_conf": 0.9929216025693198,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.007078397430680172,
+          "top_bins": {
+            "0.9": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 119,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 84,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9971072222116408
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9865275469490526
+            }
+          }
+        }
+      },
+      "excytin_signin": {
+        "questions": 120,
+        "groups": 1,
+        "raw": {
+          "n": 120,
+          "nll": 0.0009466221977245799,
+          "acc": 1.0,
+          "ece": 0.0009222842581878554,
+          "brier": 7.013154556645113e-05,
+          "mean_conf": 0.9990777157418121,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.0009222842581878554,
+          "top_bins": {
+            "0.9": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 119,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 119,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9999016374480455
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9997216681533668
+            }
+          }
+        },
+        "calibrated": {
+          "n": 120,
+          "nll": 0.00847100086283192,
+          "acc": 1.0,
+          "ece": 0.008170507143524745,
+          "brier": 0.0008193865392318428,
+          "mean_conf": 0.9918294928564755,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9916666666666667,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.008170507143524519,
+          "top_bins": {
+            "0.9": {
+              "n": 119,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 119,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 105,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9955464217668417
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9921417222813483
+            }
+          }
+        }
+      },
+      "sev_guide_detector": {
+        "questions": 1064,
+        "groups": 114,
+        "raw": {
+          "n": 1064,
+          "nll": 0.8605571101399087,
+          "acc": 0.6005639097744361,
+          "ece": 0.14734385206430384,
+          "brier": 0.5008109312564021,
+          "mean_conf": 0.7382136284198534,
+          "confident_error_rate": 0.017857142857142856,
+          "coverage_at_0_9": 0.33270676691729323,
+          "accuracy_at_0_9": 0.9463276836158192,
+          "coverage_at_5pct_error": 0.3082706766917293,
+          "coverage_at_1pct_error": 0.0,
+          "aurc": 0.19350209200158913,
+          "error_rate_at_0_9": 0.05367231638418079,
+          "confidence_bias": 0.1376497186454173,
+          "top_bins": {
+            "0.9": {
+              "n": 354,
+              "errors": 19,
+              "error_rate": 0.05367231638418079
+            },
+            "0.95": {
+              "n": 327,
+              "errors": 13,
+              "error_rate": 0.039755351681957186
+            },
+            "0.99": {
+              "n": 319,
+              "errors": 12,
+              "error_rate": 0.03761755485893417
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5028195488721805,
+              "accuracy": 0.8205607476635514,
+              "confidence_cutoff": 0.7059991149306561
+            },
+            "0.8": {
+              "coverage": 0.8035714285714286,
+              "accuracy": 0.6596491228070176,
+              "confidence_cutoff": 0.5473316703985754
+            }
+          }
+        },
+        "calibrated": {
+          "n": 1064,
+          "nll": 0.7966730081647482,
+          "acc": 0.6005639097744361,
+          "ece": 0.06373435814748656,
+          "brier": 0.46843654552918773,
+          "mean_conf": 0.6455031255616036,
+          "confident_error_rate": 0.011278195488721804,
+          "coverage_at_0_9": 0.299812030075188,
+          "accuracy_at_0_9": 0.9623824451410659,
+          "coverage_at_5pct_error": 0.3082706766917293,
+          "coverage_at_1pct_error": 0.0,
+          "aurc": 0.1938629191063446,
+          "error_rate_at_0_9": 0.03761755485893417,
+          "confidence_bias": 0.04493921578716753,
+          "top_bins": {
+            "0.9": {
+              "n": 319,
+              "errors": 12,
+              "error_rate": 0.03761755485893417
+            },
+            "0.95": {
+              "n": 0,
+              "errors": 0,
+              "error_rate": null
+            },
+            "0.99": {
+              "n": 0,
+              "errors": 0,
+              "error_rate": null
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5028195488721805,
+              "accuracy": 0.8205607476635514,
+              "confidence_cutoff": 0.5612814391210313
+            },
+            "0.8": {
+              "coverage": 0.8007518796992481,
+              "accuracy": 0.6619718309859155,
+              "confidence_cutoff": 0.4656043556130704
+            }
+          }
+        }
+      },
+      "sev_guide_triage": {
+        "questions": 231,
+        "groups": 23,
+        "raw": {
+          "n": 231,
+          "nll": 0.8396428451107271,
+          "acc": 0.5670995670995671,
+          "ece": 0.1889188281653689,
+          "brier": 0.5162927524049745,
+          "mean_conf": 0.7421494516023849,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.3593073593073593,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.3722943722943723,
+          "coverage_at_1pct_error": 0.36363636363636365,
+          "aurc": 0.20987091476964936,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": 0.17504988450281778,
+          "top_bins": {
+            "0.9": {
+              "n": 83,
+              "errors": 0,
+              "error_rate": 0.0
             },
-            "flow": {
-              "raw": {
-                "n": 56,
-                "accuracy": 0.9821428571428571,
-                "nll": 0.040828320800047294,
-                "brier": 0.030966232855549104,
-                "correct": 55
-              },
-              "global_calibrated": {
-                "n": 56,
-                "accuracy": 0.9821428571428571,
-                "nll": 0.03668431589324172,
-                "brier": 0.02605404964851352,
-                "correct": 55
-              }
+            "0.95": {
+              "n": 80,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 14,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5627705627705628,
+              "accuracy": 0.7153846153846154,
+              "confidence_cutoff": 0.7564145922660828
+            },
+            "0.8": {
+              "coverage": 0.8051948051948052,
+              "accuracy": 0.5806451612903226,
+              "confidence_cutoff": 0.51023405030021
             }
           }
         },
-        "manual": {
-          "raw": {
-            "n": 83,
-            "accuracy": 0.7831325301204819,
-            "nll": 1.1015953331868638,
-            "brier": 0.38672041228910636,
-            "correct": 65
-          },
-          "global_calibrated": {
-            "n": 83,
-            "accuracy": 0.7831325301204819,
-            "nll": 0.8197108651023933,
-            "brier": 0.3693842986364894,
-            "correct": 65
-          },
-          "families": {
-            "binding": {
-              "raw": {
-                "n": 40,
-                "accuracy": 0.8,
-                "nll": 0.906880605987712,
-                "brier": 0.3722274315061049,
-                "correct": 32
-              },
-              "global_calibrated": {
-                "n": 40,
-                "accuracy": 0.8,
-                "nll": 0.7146033829092314,
-                "brier": 0.3514071574500127,
-                "correct": 32
-              }
+        "calibrated": {
+          "n": 231,
+          "nll": 0.7761249864817235,
+          "acc": 0.5670995670995671,
+          "ece": 0.16797778410789055,
+          "brier": 0.46407372894941223,
+          "mean_conf": 0.6510059283006658,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.341991341991342,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 0.37662337662337664,
+          "coverage_at_1pct_error": 0.36363636363636365,
+          "aurc": 0.20742662880964652,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": 0.08390636120109873,
+          "top_bins": {
+            "0.9": {
+              "n": 79,
+              "errors": 0,
+              "error_rate": 0.0
             },
-            "flow": {
-              "raw": {
-                "n": 40,
-                "accuracy": 0.75,
-                "nll": 1.3789279125484732,
-                "brier": 0.43021742389448914,
-                "correct": 30
-              },
-              "global_calibrated": {
-                "n": 40,
-                "accuracy": 0.75,
-                "nll": 0.9861678633098097,
-                "brier": 0.4150648418384071,
-                "correct": 30
-              }
+            "0.95": {
+              "n": 14,
+              "errors": 0,
+              "error_rate": 0.0
             },
-            "prerequisite": {
-              "raw": {
-                "n": 3,
-                "accuracy": 1.0,
-                "nll": 2.3971020760583907e-05,
-                "brier": 1.3240221828953951e-09,
-                "correct": 3
-              },
-              "global_calibrated": {
-                "n": 3,
-                "accuracy": 1.0,
-                "nll": 0.0017173182456664555,
-                "brier": 5.60509727845392e-06,
-                "correct": 3
-              }
+            "0.99": {
+              "n": 0,
+              "errors": 0,
+              "error_rate": null
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5584415584415584,
+              "accuracy": 0.7209302325581395,
+              "confidence_cutoff": 0.6016800953517527
+            },
+            "0.8": {
+              "coverage": 0.8095238095238095,
+              "accuracy": 0.5989304812834224,
+              "confidence_cutoff": 0.4481415540755688
             }
           }
-        },
-        "sysmon": {
-          "raw": {
-            "n": 64,
-            "accuracy": 0.96875,
-            "nll": 0.08322377416424186,
-            "brier": 0.045695299016738855,
-            "correct": 62
-          },
-          "global_calibrated": {
-            "n": 64,
-            "accuracy": 0.96875,
-            "nll": 0.11024750472054996,
-            "brier": 0.04954806836760281,
-            "correct": 62
+        }
+      },
+      "sev_swarmtraces_static": {
+        "questions": 354,
+        "groups": 176,
+        "raw": {
+          "n": 354,
+          "nll": 0.04944532768977841,
+          "acc": 0.980225988700565,
+          "ece": 0.01487544035693287,
+          "brier": 0.024608535342050798,
+          "mean_conf": 0.9802643779708107,
+          "confident_error_rate": 0.002824858757062147,
+          "coverage_at_0_9": 0.9463276836158192,
+          "accuracy_at_0_9": 0.9970149253731343,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 0.9858757062146892,
+          "aurc": 0.0005032049402250012,
+          "error_rate_at_0_9": 0.0029850746268656717,
+          "confidence_bias": 3.8389270245686724e-05,
+          "top_bins": {
+            "0.9": {
+              "n": 335,
+              "errors": 1,
+              "error_rate": 0.0029850746268656717
+            },
+            "0.95": {
+              "n": 330,
+              "errors": 1,
+              "error_rate": 0.0030303030303030303
+            },
+            "0.99": {
+              "n": 308,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9999276209511099
+            },
+            "0.8": {
+              "coverage": 0.8022598870056498,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.997796467038603
+            }
           }
         },
-        "general": {
-          "raw": {
-            "n": 105,
-            "accuracy": 0.819047619047619,
-            "nll": 0.5574328669578856,
-            "brier": 0.29243113448917374,
-            "correct": 86
-          },
-          "global_calibrated": {
-            "n": 105,
-            "accuracy": 0.819047619047619,
-            "nll": 0.42292499037031256,
-            "brier": 0.24649933996243034,
-            "correct": 86
+        "calibrated": {
+          "n": 354,
+          "nll": 0.06588860080731399,
+          "acc": 0.980225988700565,
+          "ece": 0.029025934670617953,
+          "brier": 0.028907916842541088,
+          "mean_conf": 0.9580211141684397,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.8898305084745762,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 0.9887005649717514,
+          "aurc": 0.0005463830895640532,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.022204874532125274,
+          "top_bins": {
+            "0.9": {
+              "n": 315,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 297,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 221,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9952110216327251
+            },
+            "0.8": {
+              "coverage": 0.8022598870056498,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9673839577051934
+            }
           }
         }
-      }
-    }
-  },
-  "admissions": {
-    "parent": {
-      "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-swarmtraces-4b-v2/00-checkpoint",
-      "files": {
-        "result.json": "46b3ee2a68f250ba1d433c25612083bb8235a03e617d90707fa796f86ae2cbf4",
-        "calibration/rows.json": "9a33b44bee9cb63a635ac1516bcab9f67fbe56b15a72995a8c2b54a321ad4dd6",
-        "development/rows.json": "d882c91290398ba96cf39de9128ea5a973cf8d79d20ca84f5249f9f1c1cd5300",
-        "provenance.json": "39898e5e6ef87112f653436a9b6b80555da10a01b0d203b3f5eb29000dddae3a",
-        "calibration/report.json": "15dc07e79e0121e7345ece52b028220fbbf9123609a2eee5b331c9d372370383",
-        "development/report.json": "02fc22461d5b44f88173b90e0fffe97905b4eeb132eec79088c9a8396ce73134",
-        "transfer/rows.json": "5b624c0b0f91940332045523703dd5d52761a5ff931f2c9f1751185883976368",
-        "transfer/report.json": "f44bc7be71a3b37ded9e35f6fa382c1907c70fc959c15bc712b76f024ac726cc"
       },
-      "provenance": {
-        "config": {},
-        "config_sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
-        "suite_sha256": "61e3c1898b8a03eb178355d3c8c5050e85bd217285c700df9b4506ec9e3680f8",
-        "source_hashes": {
-          "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-          "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb",
-          "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582",
-          "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a",
-          "kev/benchmark.py": "8d486659a9036102386b077fcbd197fc0d85ae7752a4711666d6b3a95622b26f",
-          "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519",
-          "kev/checkpoint.py": "9f1dbc5d3d7d3050aca4ee34e0578c9ad76b7a2a7467dfc7a4ad65e3c471db59",
-          "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435",
-          "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536",
-          "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf",
-          "kev/data.py": "a5a5e200f56b60e4bf30a195a4cc87a74afddae15999ec48c7543a4f9b59f71c",
-          "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4",
-          "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c",
-          "kev/experiment.py": "1a5ca47d2b3d236e53c87e8c4197704f162c6692aa008521d31ea491945693f0",
-          "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243",
-          "kev/metrics.py": "db0930d470c0ce16f16dcaf726d49a7a103973a5296cefc5259d1e68707d882b",
-          "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909",
-          "kev/model.py": "eac0f56c807b85b5c4077483d922e6bb03f07890de2835efd9372cf20cd01b8e",
-          "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02",
-          "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3",
-          "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa",
-          "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735",
-          "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3",
-          "kev/suite.py": "c5e5e27f2fed871b547a021d180b0a7f4a0542a488cce4ee02ad936f433441db",
-          "kev/train.py": "a2d85632d73760fb3efe51627ff4fde6af3100ed1c34e75d290c1b00c9899aa9",
-          "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763",
-          "modal_app.py": "639577258f4d581daefa51dad639961280ce92de9da256316ab4124f005a53fb",
-          "pyproject.toml": "39ec40f14c88598dee7ac98fb859a5a44e22337235a3064ebd5e89036ce81e30",
-          "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466"
+      "sev_sysmon_native_file": {
+        "questions": 62,
+        "groups": 4,
+        "raw": {
+          "n": 62,
+          "nll": 0.008428780795422771,
+          "acc": 1.0,
+          "ece": 0.007810434776248104,
+          "brier": 0.0017443201233063612,
+          "mean_conf": 0.9921895652237519,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9838709677419355,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.0078104347762480986,
+          "top_bins": {
+            "0.9": {
+              "n": 61,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 59,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 57,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9991686260052061
+            },
+            "0.8": {
+              "coverage": 0.8064516129032258,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9962482122843107
+            }
+          }
         },
-        "git_commit": "449101a53d4385bac3aef7df4fee5a0d93248953",
-        "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36",
-        "torch": "2.8.0+cu128",
-        "device": "cuda",
-        "gpu": "NVIDIA H100 80GB HBM3",
-        "legacy_checkpoint": true,
-        "measured_checkpoint": {
-          "requested": "/runs/sev-r2-curriculum-repair-4b-v1/02-trial-2/checkpoint",
-          "resolved": "/runs/sev-r2-curriculum-repair-4b-v1/02-trial-2/checkpoint",
-          "head_sha256": "5a91ed82f11c6d0c8ab0a910e0c85770119e0034039379aa601d17a3ec7664d6",
-          "adapter_sha256": "0452010abdcc58f7c39a47a1562a1c812bf85f025d4a6c105c93d486051765de",
-          "inference_temperature": 1.0
+        "calibrated": {
+          "n": 62,
+          "nll": 0.03509983435601263,
+          "acc": 1.0,
+          "ece": 0.032160212545650484,
+          "brier": 0.007419485561600889,
+          "mean_conf": 0.9678397874543494,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9516129032258065,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.0321602125456506,
+          "top_bins": {
+            "0.9": {
+              "n": 59,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 55,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 26,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9838915386931907
+            },
+            "0.8": {
+              "coverage": 0.8064516129032258,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9583052893470861
+            }
+          }
         }
       },
-      "experiment_gates": {
-        "passed": true,
-        "checks": {
-          "complete_coverage": true,
-          "isolation_and_packing": true,
-          "transfer_complete": true,
-          "heldout_pairs_at_least_70pct": true,
-          "transfer_confident_errors_below_10pct": true
-        },
-        "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release."
-      },
-      "weight_verification": "Recorded checkpoint digests; model weights were not opened by this report"
-    },
-    "candidate": {
-      "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-swarmtraces-4b-retention-v1/00-trial-0",
-      "files": {
-        "result.json": "9e6143f4cf6055c8644458f6920b7138a1a348cc254422e9d3173c268b97c4c6",
-        "calibration/rows.json": "5dc5633f59c9da863561e1ae34fbc1c87199053cf1397d4592b921ec5ff69790",
-        "development/rows.json": "9c969c91604dbe733626246efcdecee0c3706d84d5e7504d07f586ee2b518542",
-        "provenance.json": "5712631ff496a274abe8abecc84f8f259c8e58d6bf1f4a73638d969f2cce8227",
-        "calibration/report.json": "4f0cae5eb45b35029fe8b65241b5a8976c6a11a7bb5453b8111a6da995c4c40f",
-        "development/report.json": "f12c5bd74787f4c0daac0f93807f6f2e7c537509a6cc05fb98828e31a58aab20",
-        "checkpoint/training_config.json": "daf81a93394c6cd9e38cc4b0e049f54910048ffebf4de34f6ded2339d85478cb",
-        "transfer/rows.json": "2987ed490118f383c96f590852e4a4a4ef72721d50887f8fe4ff7e406670ba37",
-        "transfer/report.json": "31345b7b6f60a2036039d233f0c484dc7abac1285e1b5db0fe7444d82c9014e4"
-      },
-      "provenance": {
-        "config": {
-          "epochs": 1,
-          "seed": 4,
-          "lr": 2.5e-06,
-          "lora": 16,
-          "accum": 2,
-          "batch": 4,
-          "perm_kl": 0,
-          "perm_frac": 0.3,
-          "ord_w": 0.0,
-          "p_none": 0,
-          "p_none_distract": 0,
-          "p_distract": 0,
-          "p_none_pair": 0,
-          "synthetic_repeat": 1,
-          "public_frac": 1,
-          "head_lr": 0.0,
-          "weight_decay": 0.01,
-          "anchor_w": 0.0,
-          "label_smoothing": 0.0,
-          "brier_w": 0.0,
-          "focal_gamma": 0.0,
-          "base": "Qwen/Qwen3.5-4B-Base",
-          "base_revision": "1001bb4d826a52d1f399e183466143f4da7b741b",
-          "init_from": "/runs/sev-r2-curriculum-repair-4b-v1/02-trial-2/checkpoint",
-          "lora_targets": "all",
-          "head_dim": 256,
-          "weights_dtype": "fp32",
-          "dtype": "bf16",
-          "checkpointing": 1,
-          "option_isolation": 0,
-          "special_embeddings": 0
-        },
-        "config_sha256": "958d0b20b3ffd1c572e0abf221383a450e0f8e8d3fff3cb745235a371e6a025d",
-        "suite_sha256": "61e3c1898b8a03eb178355d3c8c5050e85bd217285c700df9b4506ec9e3680f8",
-        "source_hashes": {
-          "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-          "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb",
-          "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582",
-          "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a",
-          "kev/benchmark.py": "8d486659a9036102386b077fcbd197fc0d85ae7752a4711666d6b3a95622b26f",
-          "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519",
-          "kev/checkpoint.py": "9f1dbc5d3d7d3050aca4ee34e0578c9ad76b7a2a7467dfc7a4ad65e3c471db59",
-          "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435",
-          "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536",
-          "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf",
-          "kev/data.py": "a5a5e200f56b60e4bf30a195a4cc87a74afddae15999ec48c7543a4f9b59f71c",
-          "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4",
-          "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c",
-          "kev/experiment.py": "1a5ca47d2b3d236e53c87e8c4197704f162c6692aa008521d31ea491945693f0",
-          "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243",
-          "kev/metrics.py": "db0930d470c0ce16f16dcaf726d49a7a103973a5296cefc5259d1e68707d882b",
-          "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909",
-          "kev/model.py": "eac0f56c807b85b5c4077483d922e6bb03f07890de2835efd9372cf20cd01b8e",
-          "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02",
-          "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3",
-          "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa",
-          "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735",
-          "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3",
-          "kev/suite.py": "c5e5e27f2fed871b547a021d180b0a7f4a0542a488cce4ee02ad936f433441db",
-          "kev/train.py": "a2d85632d73760fb3efe51627ff4fde6af3100ed1c34e75d290c1b00c9899aa9",
-          "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763",
-          "modal_app.py": "639577258f4d581daefa51dad639961280ce92de9da256316ab4124f005a53fb",
-          "pyproject.toml": "39ec40f14c88598dee7ac98fb859a5a44e22337235a3064ebd5e89036ce81e30",
-          "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466"
+      "sev_sysmon_native_join": {
+        "questions": 120,
+        "groups": 3,
+        "raw": {
+          "n": 120,
+          "nll": 0.00039767294618876087,
+          "acc": 1.0,
+          "ece": 0.0003974740966455892,
+          "brier": 7.310901229709924e-07,
+          "mean_conf": 0.9996025259033544,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.0003974740966455892,
+          "top_bins": {
+            "0.9": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9997660582574378
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9994606035447323
+            }
+          }
         },
-        "git_commit": "76e180880160721082a065bf1882495e756a643a",
-        "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36",
-        "torch": "2.8.0+cu128",
-        "device": "cuda",
-        "gpu": "NVIDIA H100 80GB HBM3",
-        "legacy_checkpoint": false,
-        "measured_checkpoint": {
-          "requested": "/runs/sev-r2-swarmtraces-4b-retention-v1/00-trial-0/checkpoint",
-          "resolved": "/runs/sev-r2-swarmtraces-4b-retention-v1/00-trial-0/checkpoint",
-          "head_sha256": "b29abc817f2340aa510fcd4b2c813455ef38102508120ffc35f7917ce359c27e",
-          "adapter_sha256": "b5afd73584d1098cccb0a1bd42afc40e2de3c3bc3074a5173a642b6d4b880420",
-          "inference_temperature": 1.0
+        "calibrated": {
+          "n": 120,
+          "nll": 0.009879536093519296,
+          "acc": 1.0,
+          "ece": 0.009811175359927704,
+          "brier": 0.00022722202758135087,
+          "mean_conf": 0.9901888246400723,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 1.0,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.009811175359927704,
+          "top_bins": {
+            "0.9": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 120,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 67,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9917343480609009
+            },
+            "0.8": {
+              "coverage": 0.8,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9855433361281362
+            }
+          }
         }
       },
-      "experiment_gates": {
-        "passed": true,
-        "checks": {
-          "complete_coverage": true,
-          "isolation_and_packing": true,
-          "transfer_complete": true,
-          "heldout_pairs_at_least_70pct": true,
-          "transfer_confident_errors_below_10pct": true
-        },
-        "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release."
-      },
-      "weight_verification": "Recorded checkpoint digests; model weights were not opened by this report"
-    }
-  },
-  "training_resources": {
-    "wall_seconds": 1043.7473661899567,
-    "records_seen": 6438,
-    "requested_records": 6438,
-    "truncated_records": 0,
-    "rejected_records": 0,
-    "optimizer_steps": 805,
-    "forward_tokens": 1447855,
-    "peak_device_bytes": 22227358208,
-    "device": "cuda",
-    "dtype": "bf16",
-    "batch": 4,
-    "peak_rss_bytes": 31557873664
-  },
-  "paired_raw": {
-    "excytin": {
-      "groups": 1,
-      "ci95": null,
-      "point_deltas": {
-        "acc": 0.0,
-        "nll": -0.0010619269814233934,
-        "brier": -8.441910898222422e-05
-      },
-      "reason": "One incident group; no generalization interval estimated"
-    },
-    "guide": {
-      "acc": {
-        "micro_acc_delta": 0.012987012987012991,
-        "ci95": [
-          -0.005102694924123447,
-          0.028847543352601156
-        ],
-        "samples": 2000,
-        "groups": 23,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      },
-      "nll": {
-        "micro_nll_delta": 0.0806574651079075,
-        "ci95": [
-          -0.016061369620481,
-          0.1993407432639239
-        ],
-        "samples": 2000,
-        "groups": 23,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      },
-      "brier": {
-        "micro_brier_delta": 0.04629405344979143,
-        "ci95": [
-          -0.005474368002895286,
-          0.10610974848164327
-        ],
-        "samples": 2000,
-        "groups": 23,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      }
-    },
-    "native": {
-      "acc": {
-        "micro_acc_delta": 0.0,
-        "ci95": [
-          0.0,
-          0.0
-        ],
-        "samples": 2000,
-        "groups": 4,
-        "aggregation": "micro",
-        "unit": "Whole capture group across source families",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      },
-      "nll": {
-        "micro_nll_delta": -0.004541798865699442,
-        "ci95": [
-          -0.007275272740407906,
-          -0.003258894034076542
-        ],
-        "samples": 2000,
+      "sev_sysmon_native_other_effect": {
+        "questions": 88,
         "groups": 4,
-        "aggregation": "micro",
-        "unit": "Whole capture group across source families",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
+        "raw": {
+          "n": 88,
+          "nll": 0.005466714567741097,
+          "acc": 1.0,
+          "ece": 0.005048372154544871,
+          "brier": 0.0014126265728775101,
+          "mean_conf": 0.994951627845455,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9886363636363636,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.005048372154545011,
+          "top_bins": {
+            "0.9": {
+              "n": 87,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 86,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 83,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9997677646417501
+            },
+            "0.8": {
+              "coverage": 0.8068181818181818,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9978176224669177
+            }
+          }
+        },
+        "calibrated": {
+          "n": 88,
+          "nll": 0.02789592785139411,
+          "acc": 1.0,
+          "ece": 0.026282565295571752,
+          "brier": 0.004783993285232044,
+          "mean_conf": 0.9737174347044284,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9545454545454546,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.026282565295571603,
+          "top_bins": {
+            "0.9": {
+              "n": 84,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 75,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 46,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9913758403771589
+            },
+            "0.8": {
+              "coverage": 0.8068181818181818,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9627441989296704
+            }
+          }
+        }
       },
-      "brier": {
-        "micro_brier_delta": -0.0006570378773440486,
-        "ci95": [
-          -0.0018239016582102339,
-          -0.00012501403854909632
-        ],
-        "samples": 2000,
+      "sev_sysmon_native_process": {
+        "questions": 74,
         "groups": 4,
-        "aggregation": "micro",
-        "unit": "Whole capture group across source families",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      }
-    },
-    "guide_detector": {
-      "acc": {
-        "micro_acc_delta": 0.014097744360902276,
-        "ci95": [
-          -0.010733268142241958,
-          0.04307115301628897
-        ],
-        "samples": 2000,
-        "groups": 114,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      },
-      "nll": {
-        "micro_nll_delta": 0.010318565978466143,
-        "ci95": [
-          -0.02562492366637801,
-          0.044691969172975764
-        ],
-        "samples": 2000,
-        "groups": 114,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      },
-      "brier": {
-        "micro_brier_delta": 0.005962989482191983,
-        "ci95": [
-          -0.01664672866743052,
-          0.028271086382067975
-        ],
-        "samples": 2000,
-        "groups": 114,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      }
-    },
-    "swarm_static": {
-      "acc": {
-        "micro_acc_delta": 0.05932203389830504,
-        "ci95": [
-          0.034480288509040564,
-          0.08547197640117996
-        ],
-        "samples": 2000,
-        "groups": 176,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      },
-      "nll": {
-        "micro_nll_delta": -0.17308792668740217,
-        "ci95": [
-          -0.23906732648623316,
-          -0.11508206848561314
-        ],
-        "samples": 2000,
-        "groups": 176,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      },
-      "brier": {
-        "micro_brier_delta": -0.08362752812460522,
-        "ci95": [
-          -0.11395797861253039,
-          -0.05612652240109466
-        ],
-        "samples": 2000,
-        "groups": 176,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      }
-    },
-    "manual": {
-      "acc": {
-        "micro_acc_delta": 0.0843373493975903,
-        "ci95": [
-          0.034628911857593994,
-          0.14634146341463417
-        ],
-        "samples": 2000,
-        "groups": 40,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      },
-      "nll": {
-        "micro_nll_delta": -0.15959679081720557,
-        "ci95": [
-          -0.29781254216292613,
-          -0.037512233422527574
-        ],
-        "samples": 2000,
-        "groups": 40,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      },
-      "brier": {
-        "micro_brier_delta": -0.12862065982564036,
-        "ci95": [
-          -0.20555506872375667,
-          -0.0596051281217685
-        ],
-        "samples": 2000,
-        "groups": 40,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      }
-    },
-    "sysmon": {
-      "acc": {
-        "micro_acc_delta": 0.0,
-        "ci95": [
-          0.0,
-          0.0
-        ],
-        "samples": 2000,
-        "groups": 8,
-        "aggregation": "micro",
-        "unit": "Whole capture group across source families",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      },
-      "nll": {
-        "micro_nll_delta": -0.0014288934300975553,
-        "ci95": [
-          -0.011110020952993583,
-          0.013753809583785515
-        ],
-        "samples": 2000,
-        "groups": 8,
-        "aggregation": "micro",
-        "unit": "Whole capture group across source families",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      },
-      "brier": {
-        "micro_brier_delta": 0.0007729154617278078,
-        "ci95": [
-          -0.002342110229997109,
-          0.005453425939115126
-        ],
-        "samples": 2000,
-        "groups": 8,
-        "aggregation": "micro",
-        "unit": "Whole capture group across source families",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      }
-    },
-    "general": {
-      "acc": {
-        "micro_acc_delta": 0.0,
-        "ci95": [
-          0.0,
-          0.0
-        ],
-        "samples": 2000,
-        "groups": 64,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      },
-      "nll": {
-        "micro_nll_delta": 0.05578705356138791,
-        "ci95": [
-          0.02406632980677795,
-          0.08982514373984912
-        ],
-        "samples": 2000,
-        "groups": 64,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      },
-      "brier": {
-        "micro_brier_delta": 0.015313433968751122,
-        "ci95": [
-          0.004495110348257341,
-          0.02583795985424143
-        ],
-        "samples": 2000,
-        "groups": 64,
-        "aggregation": "micro",
-        "unit": "Whole source/group; all original variants included",
-        "method": "paired cluster percentile bootstrap; full statistic recomputed in each resample"
-      }
-    }
-  },
-  "paired_guide_decisions": {
-    "guide": {
-      "samples": 2000,
-      "seed": 20260927,
-      "groups": 23,
-      "unit": "Whole organization, sampled with replacement; all incidents remain together.",
-      "method": "Paired percentile bootstrap; fixed three-class macro-F1, absent-class F1 zero. TP recall excludes draws with no true positives.",
-      "statistics": {
-        "macro_f1": {
-          "delta": 0.013536133799551708,
-          "ci95": [
-            -0.016836824631839598,
-            0.05791991746908926
-          ],
-          "valid_samples": 2000
-        },
-        "true_positive_recall": {
-          "delta": -0.02702702702702703,
-          "ci95": [
-            -0.047619047619047616,
-            0.0
-          ],
-          "valid_samples": 2000
+        "raw": {
+          "n": 74,
+          "nll": 0.004798899209615666,
+          "acc": 1.0,
+          "ece": 0.004644321057477101,
+          "brier": 0.0005716684001551928,
+          "mean_conf": 0.995355678942523,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9864864864864865,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.00464432105747703,
+          "top_bins": {
+            "0.9": {
+              "n": 73,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 72,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 68,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9999268900168323
+            },
+            "0.8": {
+              "coverage": 0.8108108108108109,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9973453419434639
+            }
+          }
         },
-        "equal_organization_accuracy": {
-          "delta": 0.006521739130434745,
-          "ci95": [
-            -0.0021739130434783593,
-            0.015217391304347794
-          ],
-          "valid_samples": 2000
+        "calibrated": {
+          "n": 74,
+          "nll": 0.023528453108139967,
+          "acc": 1.0,
+          "ece": 0.022317812691888844,
+          "brier": 0.004027925361032504,
+          "mean_conf": 0.9776821873081113,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9324324324324325,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "aurc": 0.0,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.022317812691888705,
+          "top_bins": {
+            "0.9": {
+              "n": 69,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 65,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 45,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9964760714345572
+            },
+            "0.8": {
+              "coverage": 0.8108108108108109,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9703686880072409
+            }
+          }
         }
-      }
-    },
-    "guide_detector": {
-      "samples": 2000,
-      "seed": 20260927,
-      "groups": 114,
-      "unit": "Whole organization, sampled with replacement; all incidents remain together.",
-      "method": "Paired percentile bootstrap; fixed three-class macro-F1, absent-class F1 zero. TP recall excludes draws with no true positives.",
-      "statistics": {
-        "macro_f1": {
-          "delta": 0.0266170267250998,
-          "ci95": [
-            -0.005222855290995668,
-            0.06622974286948383
-          ],
-          "valid_samples": 2000
-        },
-        "true_positive_recall": {
-          "delta": 0.10035842293906805,
-          "ci95": [
-            0.033210332103321055,
-            0.194811225925777
-          ],
-          "valid_samples": 2000
+      },
+      "sev_sysmon_native_registry": {
+        "questions": 66,
+        "groups": 4,
+        "raw": {
+          "n": 66,
+          "nll": 0.024015813931122083,
+          "acc": 0.9848484848484849,
+          "ece": 0.016913465896023366,
+          "brier": 0.01690246747159733,
+          "mean_conf": 0.9866786138448916,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9696969696969697,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 0.9696969696969697,
+          "aurc": 0.00046266864448682633,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": 0.0018301289964067768,
+          "top_bins": {
+            "0.9": {
+              "n": 64,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 64,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 62,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9998656142719909
+            },
+            "0.8": {
+              "coverage": 0.803030303030303,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9982153954384291
+            }
+          }
         },
-        "equal_organization_accuracy": {
-          "delta": 0.024447391002711205,
-          "ci95": [
-            -0.0024186565385442365,
-            0.05694784972373788
-          ],
-          "valid_samples": 2000
+        "calibrated": {
+          "n": 66,
+          "nll": 0.03582391955947498,
+          "acc": 0.9848484848484849,
+          "ece": 0.01540577138447916,
+          "brier": 0.016990289728223678,
+          "mean_conf": 0.9730644954445979,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.9696969696969697,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 0.9696969696969697,
+          "aurc": 0.00046266864448682633,
+          "error_rate_at_0_9": 0.0,
+          "confidence_bias": -0.011783989403886919,
+          "top_bins": {
+            "0.9": {
+              "n": 64,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.95": {
+              "n": 59,
+              "errors": 0,
+              "error_rate": 0.0
+            },
+            "0.99": {
+              "n": 40,
+              "errors": 0,
+              "error_rate": 0.0
+            }
+          },
+          "selective": {
+            "0.5": {
+              "coverage": 0.5,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.9940913990982823
+            },
+            "0.8": {
+              "coverage": 0.803030303030303,
+              "accuracy": 1.0,
+              "confidence_cutoff": 0.975504713794386
+            }
+          }
         }
       }
     }
   },
-  "gates": {
-    "passed": false,
-    "checks": {
-      "manual_correct_gain": {
-        "actual": 7,
-        "bound": 3,
-        "operator": ">=",
-        "passed": true
-      },
-      "native_correct": {
-        "actual": 409,
-        "bound": 398,
-        "operator": ">=",
-        "passed": true
-      },
-      "excytin_correct": {
-        "actual": 398,
-        "bound": 391,
-        "operator": ">=",
-        "passed": true
-      },
-      "sysmon_correct": {
-        "actual": 62,
-        "bound": 60,
-        "operator": ">=",
-        "passed": true
-      },
-      "general_correct": {
-        "actual": 86,
-        "bound": 85,
-        "operator": ">=",
-        "passed": true
-      },
-      "manual_raw_nll": {
-        "actual": 1.1015953331868638,
-        "bound": 1.3111921240040694,
-        "operator": "<=",
-        "passed": true
-      },
-      "manual_raw_brier": {
-        "actual": 0.38672041228910636,
-        "bound": 0.5353410721147468,
-        "operator": "<=",
-        "passed": true
-      },
-      "manual_global_calibrated_nll": {
-        "actual": 0.8197108651023933,
-        "bound": 1.0802111739617255,
-        "operator": "<=",
-        "passed": true
-      },
-      "manual_global_calibrated_brier": {
-        "actual": 0.3693842986364894,
-        "bound": 0.5033988288027265,
-        "operator": "<=",
-        "passed": true
-      },
-      "native_raw_nll": {
-        "actual": 0.008457485545022507,
-        "bound": 0.06299928441072196,
-        "operator": "<=",
-        "passed": true
-      },
-      "native_raw_brier": {
-        "actual": 0.0035344210430228854,
-        "bound": 0.024191458920366934,
-        "operator": "<=",
-        "passed": true
-      },
-      "native_global_calibrated_nll": {
-        "actual": 0.028769043116345983,
-        "bound": 0.08020084130636222,
-        "operator": "<=",
-        "passed": true
-      },
-      "native_global_calibrated_brier": {
-        "actual": 0.006284804103393485,
-        "bound": 0.02669768749600822,
-        "operator": "<=",
-        "passed": true
-      },
-      "excytin_raw_nll": {
-        "actual": 0.0009678808724190956,
-        "bound": 0.052029807853842494,
-        "operator": "<=",
-        "passed": true
-      },
-      "excytin_raw_brier": {
-        "actual": 4.8256567178328636e-05,
-        "bound": 0.020132675676160554,
-        "operator": "<=",
-        "passed": true
-      },
-      "excytin_global_calibrated_nll": {
-        "actual": 0.009470865385844369,
-        "bound": 0.05820502390654013,
-        "operator": "<=",
-        "passed": true
-      },
-      "excytin_global_calibrated_brier": {
-        "actual": 0.0006565384935901168,
-        "bound": 0.02062640684122579,
-        "operator": "<=",
-        "passed": true
-      },
-      "sysmon_raw_nll": {
-        "actual": 0.08322377416424186,
-        "bound": 0.13465266759433944,
-        "operator": "<=",
-        "passed": true
-      },
-      "sysmon_raw_brier": {
-        "actual": 0.045695299016738855,
-        "bound": 0.06492238355501105,
-        "operator": "<=",
-        "passed": true
-      },
-      "sysmon_global_calibrated_nll": {
-        "actual": 0.11024750472054996,
-        "bound": 0.15719175042876138,
-        "operator": "<=",
-        "passed": true
-      },
-      "sysmon_global_calibrated_brier": {
-        "actual": 0.04954806836760281,
-        "bound": 0.06844225884031123,
-        "operator": "<=",
-        "passed": true
-      },
-      "general_raw_nll": {
-        "actual": 0.5574328669578856,
-        "bound": 0.5516458133964977,
-        "operator": "<=",
-        "passed": false
-      },
-      "general_raw_brier": {
-        "actual": 0.29243113448917374,
-        "bound": 0.29711770052042263,
-        "operator": "<=",
-        "passed": true
-      },
-      "general_global_calibrated_nll": {
-        "actual": 0.42292499037031256,
-        "bound": 0.4765669988063534,
-        "operator": "<=",
-        "passed": true
-      },
-      "general_global_calibrated_brier": {
-        "actual": 0.24649933996243034,
-        "bound": 0.26785251439683744,
-        "operator": "<=",
-        "passed": true
-      },
-      "guide_raw_nll": {
-        "actual": 0.8563265185203435,
-        "bound": 0.8256690534124361,
-        "operator": "<=",
-        "passed": false
-      },
-      "guide_raw_brier": {
-        "actual": 0.5255142828500304,
-        "bound": 0.499220229400239,
-        "operator": "<=",
-        "passed": false
-      },
-      "guide_global_calibrated_nll": {
-        "actual": 0.7793526965750157,
-        "bound": 0.8008521265465799,
-        "operator": "<=",
-        "passed": true
-      },
-      "guide_global_calibrated_brier": {
-        "actual": 0.4718856823661165,
-        "bound": 0.4766593103734014,
-        "operator": "<=",
-        "passed": true
-      },
-      "guide_detector_raw_nll": {
-        "actual": 0.8276776090882748,
-        "bound": 0.8673590431098086,
-        "operator": "<=",
-        "passed": true
-      },
-      "guide_detector_raw_brier": {
-        "actual": 0.48019952548397726,
-        "bound": 0.49423653600178535,
-        "operator": "<=",
-        "passed": true
-      },
-      "guide_detector_global_calibrated_nll": {
-        "actual": 0.785071927880167,
-        "bound": 0.8476013493654482,
-        "operator": "<=",
-        "passed": true
-      },
-      "guide_detector_global_calibrated_brier": {
-        "actual": 0.46075730837789786,
-        "bound": 0.4889190783897773,
-        "operator": "<=",
-        "passed": true
-      },
-      "guide_macro_f1": {
-        "actual": 0.4613423970886119,
-        "bound": 0.4278062632890602,
-        "operator": ">=",
-        "passed": true
-      },
-      "guide_true_positive_correct": {
-        "actual": 7,
-        "bound": 6,
-        "operator": ">=",
-        "passed": true
-      },
-      "guide_detector_macro_f1": {
-        "actual": 0.5681608349992006,
-        "bound": 0.5215438082741007,
-        "operator": ">=",
-        "passed": true
-      },
-      "guide_detector_true_positive_correct": {
-        "actual": 101,
-        "bound": 71,
-        "operator": ">=",
-        "passed": true
-      }
-    },
-    "interpretation": "Fixed descriptive engineering gates for a research continuation; not statistical noninferiority or a field detection claim."
+  "policy": {
+    "positive": "Requires approval under the supplied policy",
+    "negative": "Permitted under the supplied policy",
+    "threshold_split": "calibration",
+    "temperature": "reuse saved calibration receipt, or benchmark inference temperature; no refit",
+    "population": "all five conditional policy questions per admitted source program",
+    "bootstrap": "percentile intervals conditional on the fixed calibration threshold; no threshold reselection"
   },
-  "protocol_sha256": "acc38c3ec612344b8bf732f8bacf60f3ebd58815d735982ad25c748be3480f6a",
-  "scorer_sha256": "967b5bf646812d95167d3d88e1b4df7405da6d74983f8aceefb283512bc29285",
-  "protocol": {
-    "version": 3,
-    "registered_date": "2026-09-27",
-    "study": "sev-r2-swarmtraces-4b-retention-v1",
-    "suite": "evals/sev/swarmtraces-curriculum-v1",
-    "plan": "experiments/sev-swarmtraces-4b-retention-v1.json",
-    "transfer": "evals/sev/swarmtraces-transfer-v1",
-    "original_transfer": "evals/sev/security-transfer-v1",
-    "checkpoints": {
-      "parent": "/runs/sev-r2-curriculum-repair-4b-v1/02-trial-2/checkpoint",
-      "candidate": "/runs/sev-r2-swarmtraces-4b-retention-v1/00-trial-0/checkpoint"
-    },
-    "parent_checkpoint_hashes": {
-      "head_sha256": "5a91ed82f11c6d0c8ab0a910e0c85770119e0034039379aa601d17a3ec7664d6",
-      "adapter_sha256": "0452010abdcc58f7c39a47a1562a1c812bf85f025d4a6c105c93d486051765de"
-    },
-    "panels": {
-      "excytin": {
-        "suite": "evals/sev/excytin-native-v1"
-      },
-      "guide": {
-        "suite": "evals/sev/guide-v1"
-      },
-      "native": {
-        "suite": "evals/sev/sysmon-native-v2"
-      },
-      "guide_detector": {
-        "suite": "evals/sev/guide-detector-v1"
-      },
-      "swarm_static": {
-        "suite": "evals/sev/swarmtraces-static-v1"
-      },
-      "manual": {
-        "suite": "evals/sev/swarmtraces-manual-v1"
-      }
-    },
-    "gates": {
-      "manual_min_correct_gain": 3,
-      "minimum_correct": {
-        "native": 398,
-        "excytin": 391,
-        "sysmon": 60,
-        "general": 85
-      },
-      "probability_retention_domains": [
-        "manual",
-        "native",
-        "excytin",
-        "sysmon",
-        "general",
-        "guide",
-        "guide_detector"
-      ],
-      "nll_margin": 0.05,
-      "brier_margin": 0.02,
-      "guide_macro_f1_margin": 0.02,
-      "guide_tp_count_margin": 2
-    },
-    "calibration": "One micro-NLL temperature for each model on identical union calibration; unchanged on all evaluation panels. Raw and calibrated metrics reported separately.",
-    "primary": "Independent manually annotated complete-program panel. Development screen, not final field validation.",
-    "secondary": "Parser-derived held-out programs, grouped by exact text, source reconstruction and whole-program AST templates.",
-    "interpretation": "Continuation adoption criteria only. Prior GUIDE shortcomings remain separate; this experiment does not establish a general release. No final test is used.",
-    "bootstrap": {
-      "samples": 2000,
-      "seed": 20260927
-    },
-    "timeout_seconds": 3600,
-    "maximum_reservation_usd": "5.68",
-    "input_sha256": {
-      "docs/reviews/sev-swarmtraces-curriculum-review.md": "5ba6f9682dbfe1782309c934e05c487825731c544ce23f293d131927b059c309",
-      "docs/reviews/sev-swarmtraces-scorer-v1.md": "d63c433da881fd9a03a2b3d065533dd19f68f0f905bbe7c3aa042d0ccfb63447",
-      "docs/sev-swarmtraces-round-v1.md": "bdddd3fe1c8f4776710bff45e1f257c66e48f2bb198ca641aaa51f0db053bf35",
-      "evals/sev/curriculum-repair-v1/manifest.json": "5ae5dbfd48e9a2717397c0197ed8db016db87d69abc30d51813878ef734beba3",
-      "evals/sev/excytin-native-v1/manifest.json": "1a537e295dce0df278b7648160674bdabcec5d1e6d8adc80c203c1dae9dcaf5c",
-      "evals/sev/guide-detector-v1/manifest.json": "507cefca384f2a88aa7172f54757dc06bfd6f815e9501c522fcbf071f05be714",
-      "evals/sev/guide-v1/manifest.json": "50245db125492792319f4fc477ed5b356819f0bf32567bfc059f5285ae58b069",
-      "evals/sev/security-transfer-v1/manifest.json": "6b2d546ac337946bfce4c85c7634b6f749f98bff0dc84e6c7f2317a4f6eddaea",
-      "evals/sev/swarmtraces-curriculum-v1/calibration.jsonl": "8504531b423590392325f9e1d5fd291280122e6ca038483268e05bfe94970f77",
-      "evals/sev/swarmtraces-curriculum-v1/development.jsonl": "69d4aec321f985901ab0ca3ecccf072c66d60479924111d663d5f71d7e877c84",
-      "evals/sev/swarmtraces-curriculum-v1/manifest.json": "61e3c1898b8a03eb178355d3c8c5050e85bd217285c700df9b4506ec9e3680f8",
-      "evals/sev/swarmtraces-curriculum-v1/train.jsonl": "d6faa54db510a9285d4c2c2bc1f0c9095108a42b71460ad7ce83bccfcf199b92",
-      "evals/sev/swarmtraces-manual-v1/annotations.json": "e5f3412722153a5de7e4ee6ee3aed9844f0c062a8afc028b396e65c794314206",
-      "evals/sev/swarmtraces-manual-v1/calibration.jsonl": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-      "evals/sev/swarmtraces-manual-v1/development.jsonl": "d0e5832305f1f04e82a16fe3f643a3518061817858736297b6cc3f7423ad4055",
-      "evals/sev/swarmtraces-manual-v1/manifest.json": "6c1f892e39bd722dea54042f2139514720021faf33698aba9eb7cd399b67324e",
-      "evals/sev/swarmtraces-manual-v1/protection.json": "abd1e0ebf71e0edd2d94ded564d52f1555667a3fbf6fb828f3e44e1d7feb173d",
-      "evals/sev/swarmtraces-manual-v1/train.jsonl": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-      "evals/sev/swarmtraces-manual-v1/validation.json": "b4302a4fa05444ac1714c91ad0daf4e63dc66e8e4065fbc52e78bb6d79de0fea",
-      "evals/sev/swarmtraces-static-v1/calibration.jsonl": "7754bac053cd7a2bfb2303d9e1a591381a2a25925f6f728d7995a6fbb343dc5d",
-      "evals/sev/swarmtraces-static-v1/development.jsonl": "1847a598a428615a416c2c11d5bce0f6a129bd926684cbbe3bd5ccd7b4ec1321",
-      "evals/sev/swarmtraces-static-v1/manifest.json": "1f304228dbbcad5b4aab5d9ce582738b8b7109214c1fdfd56c617fa5f1db583b",
-      "evals/sev/swarmtraces-static-v1/train.jsonl": "c09de8a213c5cd6b3506c9b3ccea60074115052617ce71307a24d99d067ef8bd",
-      "evals/sev/swarmtraces-transfer-v1/calibration.jsonl": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-      "evals/sev/swarmtraces-transfer-v1/development.jsonl": "e312086e75f6c3b8367be5b949e163f9d48a5634a1f304f555d9f063685c9190",
-      "evals/sev/swarmtraces-transfer-v1/manifest.json": "1caf2255850930113bc56b708d0881ed22d3df9ba1868e966f9454595c2cefa7",
-      "evals/sev/swarmtraces-transfer-v1/train.jsonl": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-      "evals/sev/sysmon-native-v2/manifest.json": "e8c3b5dd306525e88e17b70f410892744bf99f2ed18fb14f42b910db52e558b1",
-      "experiments/sev-swarmtraces-4b-v1.json": "87e70bce1e76ccddd4a60866f3eb93e946a6f950de062cb2290db58e4ca1aee1",
-      "experiments/sev-swarmtraces-assembly-v1.json": "1915555716b0c993e1dfebc8928c1a8188265f1869db1884873f28c85d363851",
-      "runs/sev-curriculum-repair-v1/comparison.json": "623c2c4b2e348eab8a5975e48a94a4e9b0d2c98e7f7a43d6024a87779485b042",
-      "runs/sev-swarmtraces-review-20260927/draft4-admission.json": "ff1bcfbfb7448d4c60b4d86038ec5c7b6057c3e9e833a3b51d01ee66337cb318",
-      "runs/sev-swarmtraces-review-20260927/final-admission-receipt.json": "5f037732f74e09e0e46bbebbe95ad45146f89060d96ad9cbccfa59568ac5fad2",
-      "runs/sev-swarmtraces-review-20260927/panel-protection.json": "abd1e0ebf71e0edd2d94ded564d52f1555667a3fbf6fb828f3e44e1d7feb173d",
-      "runs/sev-swarmtraces-review-20260927/shared-fingerprints.jsonl": "452985f0efabc3b97a1f40a1e6a3eec57dd881ebeb448049f2de9120b491055b",
-      "scripts/assemble_sev_curriculum.py": "147b9f2a5133761d3a30acb08c90ba65cf9f4a278fed0e6310d39b5922eb551a",
-      "scripts/build_sev_swarmtraces.py": "08d60fcdfd9557633b5293e2b5e3b69758756f50ecedfa3e997ef310d396267c",
-      "scripts/integrate_sev_swarmtraces.py": "08ec5f630da2e8987eae7d6f93cd0cf7c674a0f528e3c2597f6886d01075a8ed",
-      "scripts/score_sev_curriculum_repair.py": "f9456677fbfc5f97106ce16c8bd31ab8818513778ab527e366a808ba655506b5",
-      "scripts/score_sev_guide.py": "1fc0d5e94af1fb456c571e48f312b8ff833dc2db53a5d53db1ba1f5b00cb8527",
-      "scripts/score_sev_mixed_public.py": "c2bbbef53c567065ee7c694f355b26af7b43696cf3244d89bac835dba4081192",
-      "scripts/score_sev_security.py": "83351d2299aa308611e14679e4bb77845e40519a8a01248bc651547f844f172b",
-      "scripts/score_sev_swarmtraces.py": "967b5bf646812d95167d3d88e1b4df7405da6d74983f8aceefb283512bc29285",
-      "scripts/swarmtraces_ast.mjs": "1c1035149778b411eda4ca95a5cf822afd6a0117064c6f6687130fc113ea81e3",
-      "modal_app.py": "639577258f4d581daefa51dad639961280ce92de9da256316ab4124f005a53fb",
-      "tests/test_modal_input_mount.py": "6fcc9ccc9da13f72646538e1342728b7e935bf7ebb156af704f0ad0f4843d3e3",
-      "runs/sev-swarmtraces-intake-20260927/failed-launch-v1.json": "6702c11219de850dba1723cbad354276c0db691bc5ba9125ce49ca1f67e5edfe",
-      "experiments/sev-swarmtraces-4b-retention-v1.json": "8dea92d4ab89f7aa09bcdae6c1299ea06a8875cfc09d657601b64331ac4377a4",
-      "docs/sev-swarmtraces-retention-v1.md": "39b57384b91a850420cc624d0081b2d1de7551539145e19af764854ed31b059c",
-      "experiments/sev-swarmtraces-protocol-v2.json": "62680e8f7965b981efe8d9281fc0b0dbd5b8f0759a871bad68e249365377538a",
-      "runs/sev-swarmtraces-v2/comparison.json": "df7a4433b1320e98f5a2cec59f792c2ac641aa74fa69a6e5306b7341f3219b57",
-      "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/provenance.json": "39898e5e6ef87112f653436a9b6b80555da10a01b0d203b3f5eb29000dddae3a",
-      "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/result.json": "46b3ee2a68f250ba1d433c25612083bb8235a03e617d90707fa796f86ae2cbf4",
-      "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/calibration/rows.json": "9a33b44bee9cb63a635ac1516bcab9f67fbe56b15a72995a8c2b54a321ad4dd6",
-      "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/calibration/report.json": "15dc07e79e0121e7345ece52b028220fbbf9123609a2eee5b331c9d372370383",
-      "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/development/rows.json": "d882c91290398ba96cf39de9128ea5a973cf8d79d20ca84f5249f9f1c1cd5300",
-      "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/development/report.json": "02fc22461d5b44f88173b90e0fffe97905b4eeb132eec79088c9a8396ce73134",
-      "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/transfer/rows.json": "5b624c0b0f91940332045523703dd5d52761a5ff931f2c9f1751185883976368",
-      "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint/transfer/report.json": "f44bc7be71a3b37ded9e35f6fa382c1907c70fc959c15bc712b76f024ac726cc",
-      "docs/reviews/sev-swarmtraces-results-v2.md": "bbfc48b93b09dafbdac5d44576d7c852fa07fcb021e587560109c5c1516313f8",
-      "runs/sev-swarmtraces-review-20260927/results-v2-final-receipt.json": "6d474088bbd9bbe725ea7b0fb17f4e82d31ec2f95bfa07b3a6994b7a93a5ce52",
-      "runs/sev-swarmtraces-review-20260927/results-v2-review.json": "dc5f2a90997f3d4c13fc65a63e1eb8939b8d9cae8dc278ff5c1707f8e588f397",
-      "runs/sev-swarmtraces-review-20260927/audit-results-v2.py": "5e9966bd8a51312e85a162e17d3ce71da87b2b9a7000641a407fa56ccb93be87"
-    },
-    "retained_suite": "evals/sev/curriculum-repair-v1",
-    "admission_limitations": [
-      "No earlier-response credential examples survive automatic admission.",
-      "Five execution-flow and eleven unavailable-context examples are training-only; automatic development does not test these classes.",
-      "The manual panel is a fixed development screen, not a locked final test or field detection evaluation."
-    ],
-    "maximum_total_reservations_including_failed_launch_usd": "28.40",
-    "preceding_development_screen": {
-      "protocol": "experiments/sev-swarmtraces-protocol-v2.json",
-      "protocol_sha256": "62680e8f7965b981efe8d9281fc0b0dbd5b8f0759a871bad68e249365377538a",
-      "result": "runs/sev-swarmtraces-v2/comparison.json",
-      "result_sha256": "df7a4433b1320e98f5a2cec59f792c2ac641aa74fa69a6e5306b7341f3219b57",
-      "decision": "Keep the same parent, data and gates. Test one continuation at half the learning rate after the older GUIDE panel failed raw NLL, raw Brier and true-positive retention checks. The manual panel is reused development data, not fresh confirmation."
-    },
-    "reused_parent_evaluation": "runs/sev-r2-swarmtraces-4b-v2/00-checkpoint",
-    "expected_training_resources": {
-      "records_seen": 6438,
-      "optimizer_steps": 805,
-      "forward_tokens": 1447855,
-      "truncated_records": 0,
-      "rejected_records": 0
-    }
-  }
+  "limitations": [
+    "Authored permissions and static callback dataflow, not historical maliciousness or execution evidence.",
+    "The selected conditional panel does not estimate deployment prevalence.",
+    "Actor origin is not measured. No locked test is read and this report does not promote a model.",
+    "The calibration false-alert budget is empirical, not a deployment guarantee.",
+    "Rare-operation rates are null without the relevant class; bootstrap draws without that class are omitted."
+  ]
 }