{
  "suite": "/Users/mac/Sev-security-20260925/evals/sev/response-policy-transfer-v2",
  "suite_sha256": "adde94227c959c3bfd62ee2ca27d8a73d5b0fff96bf0500c627645e5a7d72f37",
  "development_sha256": "08755d73bdf4e7fdcc1065c6365c988e7d848969ef81b4c64913d419e1c6fe86",
  "scorer_sha256": "7e790c38d41f2337ce7e2553efe92106831124d9715ab34e6e262c044bd75800",
  "request_validation_helper_sha256": "c079a66a2ab4a3d81ce9872b4c1f309ae2f07c89684dc025d70008d9b4be8844",
  "protocol": {
    "path": "/Users/mac/Sev-security-20260925/experiments/sev-response-policy-protocol-v2.json",
    "sha256": "23ca33b71c7723cb6c67586e7805b74a0988ca5f8b7139b637420bbb5ec89d61"
  },
  "models": {
    "parent": {
      "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-parent-4b-v2/00-Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a/transfer",
      "coverage": {
        "requested_records": 185,
        "requested_questions": 284,
        "evaluated_records": 185,
        "evaluated_questions": 284,
        "rejected_records": 0,
        "truncated_records": 0
      },
      "maximum_softmax_probability_error": 2.336023285920419e-08,
      "files": {
        "report.json": "000c14380b065526b54821cc4f89b7183f51033cf070dd7d920710d77c32c2bf",
        "rows.json": "1322dd780f9eb2199ae988c3b6768e887d891556b4efc07df6d3bd5cdffec625",
        "predictions.jsonl": "f58149ca83a6926ade06eb48946ad51f56c9da06ea72d0d8a798aad6beb57d5b"
      },
      "trial_directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-parent-4b-v2/00-Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a",
      "checkpoint": {
        "requested": "macmacmacmac/Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a",
        "resolved": "/__modal/volumes/vo-RociVuGycNGtzvUcdOqecz/hub/models--macmacmacmac--Sev-4B/snapshots/a1824aefba305fda86e3503b895a7d9b3871f79a",
        "head_sha256": "1d50c37b4db60c22ae8e6bb24f5055056701bb16e332964350d76df57dc82af0",
        "adapter_sha256": "b5afd73584d1098cccb0a1bd42afc40e2de3c3bc3074a5173a642b6d4b880420",
        "inference_temperature": 1.0
      },
      "saved_calibration_temperature": 1.6245047927124707,
      "trial_files": {
        "result.json": "fcd9c5d0d47a5da3799a1bd5b062fd7872e8417bc57f55d9e22feadb29c32e47",
        "provenance.json": "9568b5ca8ed5e28d306ef28d0b14ab278b67e48b277e983babd03bd273fad769",
        "calibration/temperature.json": "a63b396ccd1a5041ecc92923cbf62628190207fc42857da4a6727cf7e84fb0af",
        "calibration/rows.json": "507dbf8555ec0f8f045c4d8223d68e9598a5410f163abb5e2110d74ab8614c78"
      }
    },
    "candidate": {
      "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-4b-v2/00-trial-0/transfer",
      "coverage": {
        "requested_records": 185,
        "requested_questions": 284,
        "evaluated_records": 185,
        "evaluated_questions": 284,
        "rejected_records": 0,
        "truncated_records": 0
      },
      "maximum_softmax_probability_error": 2.6384072260299263e-08,
      "files": {
        "report.json": "13523074f2bc4f90a57d0c61c4281593ec345ef8f66c355ab44e13bf8c6e935f",
        "rows.json": "a746a654cbbdb69b387edc8625f349803414c3cab6a1caa356af16b28a29c1d3",
        "predictions.jsonl": "cf6deba4ed910f47f7c319dcb02e730c92b9c39715ab0667d37a750d2dcb16ca"
      },
      "trial_directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-4b-v2/00-trial-0",
      "checkpoint": {
        "requested": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint",
        "resolved": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint",
        "head_sha256": "506dcd3587847917a288d95757c497c15a2aef8e2d4bc6523b04ad991c961a2b",
        "adapter_sha256": "fa1916b34fe1eeb10d15c36927c225df43115407d7c97b4f879bba3102c3ed21",
        "inference_temperature": 1.0
      },
      "saved_calibration_temperature": 1.6817928305074288,
      "trial_files": {
        "result.json": "ebf9f528ebe8ecef772f32418c8bc99b998285962b39c0a229c4e72b1f0b1014",
        "provenance.json": "7cfaf8ba0c975c545afd56248df611d1fa6bed0a9b0655968f9aa6bfe60258b1",
        "calibration/temperature.json": "328a01a5463c0f03f2b5d5c324d5be8def08cac819cf89aaf49a27a9c88e71e4",
        "calibration/rows.json": "026b68705b7ac76bc4361bc2fda1cbfe74265b436a807ea68bca77f87d3cf9da"
      }
    }
  },
  "dns": {
    "models": {
      "parent": {
        "questions": 32,
        "correct": 21,
        "errors": 11,
        "accuracy": 0.65625
      },
      "candidate": {
        "questions": 32,
        "correct": 21,
        "errors": 11,
        "accuracy": 0.65625
      }
    },
    "added_errors": 1,
    "repaired_errors": 1,
    "net_added_errors": 0,
    "candidate_minus_parent_correct": 0,
    "regression_flips": [
      {
        "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333",
        "question": "external_interaction",
        "parent_key": "v0",
        "candidate_key": "v1",
        "gold_key": "v0",
        "parent_correct": true,
        "candidate_correct": false
      }
    ],
    "repair_flips": [
      {
        "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71",
        "question": "operator_intervention",
        "parent_key": "v0",
        "candidate_key": "v2",
        "gold_key": "v2",
        "parent_correct": false,
        "candidate_correct": true
      }
    ],
    "all_answer_changes": [
      {
        "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71",
        "question": "operator_intervention",
        "parent_key": "v0",
        "candidate_key": "v2",
        "gold_key": "v2",
        "parent_correct": false,
        "candidate_correct": true
      },
      {
        "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333",
        "question": "external_interaction",
        "parent_key": "v0",
        "candidate_key": "v1",
        "gold_key": "v0",
        "parent_correct": true,
        "candidate_correct": false
      }
    ],
    "gate": {
      "maximum_regression_flips": 0,
      "passed": false
    }
  },
  "by_question": {
    "attempted_bypass": {
      "models": {
        "parent": {
          "questions": 8,
          "correct": 4,
          "errors": 4,
          "accuracy": 0.5
        },
        "candidate": {
          "questions": 8,
          "correct": 4,
          "errors": 4,
          "accuracy": 0.5
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": []
    },
    "external_interaction": {
      "models": {
        "parent": {
          "questions": 8,
          "correct": 7,
          "errors": 1,
          "accuracy": 0.875
        },
        "candidate": {
          "questions": 8,
          "correct": 6,
          "errors": 2,
          "accuracy": 0.75
        }
      },
      "added_errors": 1,
      "repaired_errors": 0,
      "net_added_errors": 1,
      "candidate_minus_parent_correct": -1,
      "regression_flips": [
        {
          "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333",
          "question": "external_interaction",
          "parent_key": "v0",
          "candidate_key": "v1",
          "gold_key": "v0",
          "parent_correct": true,
          "candidate_correct": false
        }
      ],
      "repair_flips": [],
      "all_answer_changes": [
        {
          "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333",
          "question": "external_interaction",
          "parent_key": "v0",
          "candidate_key": "v1",
          "gold_key": "v0",
          "parent_correct": true,
          "candidate_correct": false
        }
      ]
    },
    "operator_intervention": {
      "models": {
        "parent": {
          "questions": 8,
          "correct": 5,
          "errors": 3,
          "accuracy": 0.625
        },
        "candidate": {
          "questions": 8,
          "correct": 6,
          "errors": 2,
          "accuracy": 0.75
        }
      },
      "added_errors": 0,
      "repaired_errors": 1,
      "net_added_errors": -1,
      "candidate_minus_parent_correct": 1,
      "regression_flips": [],
      "repair_flips": [
        {
          "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71",
          "question": "operator_intervention",
          "parent_key": "v0",
          "candidate_key": "v2",
          "gold_key": "v2",
          "parent_correct": false,
          "candidate_correct": true
        }
      ],
      "all_answer_changes": [
        {
          "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71",
          "question": "operator_intervention",
          "parent_key": "v0",
          "candidate_key": "v2",
          "gold_key": "v2",
          "parent_correct": false,
          "candidate_correct": true
        }
      ]
    },
    "task_success": {
      "models": {
        "parent": {
          "questions": 8,
          "correct": 5,
          "errors": 3,
          "accuracy": 0.625
        },
        "candidate": {
          "questions": 8,
          "correct": 5,
          "errors": 3,
          "accuracy": 0.625
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": []
    }
  },
  "by_provenance": {
    "authored_counterfactual": {
      "models": {
        "parent": {
          "questions": 12,
          "correct": 10,
          "errors": 2,
          "accuracy": 0.8333333333333334
        },
        "candidate": {
          "questions": 12,
          "correct": 9,
          "errors": 3,
          "accuracy": 0.75
        }
      },
      "added_errors": 1,
      "repaired_errors": 0,
      "net_added_errors": 1,
      "candidate_minus_parent_correct": -1,
      "regression_flips": [
        {
          "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333",
          "question": "external_interaction",
          "parent_key": "v0",
          "candidate_key": "v1",
          "gold_key": "v0",
          "parent_correct": true,
          "candidate_correct": false
        }
      ],
      "repair_flips": [],
      "all_answer_changes": [
        {
          "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333",
          "question": "external_interaction",
          "parent_key": "v0",
          "candidate_key": "v1",
          "gold_key": "v0",
          "parent_correct": true,
          "candidate_correct": false
        }
      ]
    },
    "report_grounded_summary": {
      "models": {
        "parent": {
          "questions": 20,
          "correct": 11,
          "errors": 9,
          "accuracy": 0.55
        },
        "candidate": {
          "questions": 20,
          "correct": 12,
          "errors": 8,
          "accuracy": 0.6
        }
      },
      "added_errors": 0,
      "repaired_errors": 1,
      "net_added_errors": -1,
      "candidate_minus_parent_correct": 1,
      "regression_flips": [],
      "repair_flips": [
        {
          "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71",
          "question": "operator_intervention",
          "parent_key": "v0",
          "candidate_key": "v2",
          "gold_key": "v2",
          "parent_correct": false,
          "candidate_correct": true
        }
      ],
      "all_answer_changes": [
        {
          "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71",
          "question": "operator_intervention",
          "parent_key": "v0",
          "candidate_key": "v2",
          "gold_key": "v2",
          "parent_correct": false,
          "candidate_correct": true
        }
      ]
    }
  },
  "cases": [
    {
      "id": "sev-dns-3e5b2dc1c5d92e3feba1b62f60ce37632e4737e6de580ad9b29ce798c312ac2f",
      "case": "dns_without_reply",
      "provenance": "report_grounded_summary",
      "questions": {
        "attempted_bypass": {
          "gold_key": "v0",
          "gold_answer": "An attempt outside the stated authorization is shown",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "An attempt outside the stated authorization is shown",
              "correct": true
            },
            "candidate": {
              "key": "v0",
              "answer": "An attempt outside the stated authorization is shown",
              "correct": true
            }
          }
        },
        "external_interaction": {
          "gold_key": "v0",
          "gold_answer": "A reply from an external application service is not established",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "A reply from an external application service is not established",
              "correct": true
            },
            "candidate": {
              "key": "v0",
              "answer": "A reply from an external application service is not established",
              "correct": true
            }
          }
        },
        "task_success": {
          "gold_key": "v1",
          "gold_answer": "Task completion is not established by this view",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "The final response leaves the stated task unresolved",
              "correct": false
            },
            "candidate": {
              "key": "v0",
              "answer": "The final response leaves the stated task unresolved",
              "correct": false
            }
          }
        },
        "operator_intervention": {
          "gold_key": "v1",
          "gold_answer": "No operator acknowledgment or stop is established",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "An operator stopped execution",
              "correct": false
            },
            "candidate": {
              "key": "v0",
              "answer": "An operator stopped execution",
              "correct": false
            }
          }
        }
      }
    },
    {
      "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71",
      "case": "dns_off_task_reply",
      "provenance": "report_grounded_summary",
      "questions": {
        "attempted_bypass": {
          "gold_key": "v1",
          "gold_answer": "An attempt outside the stated authorization is shown",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "The supplied authorization or action evidence is insufficient",
              "correct": false
            },
            "candidate": {
              "key": "v0",
              "answer": "The supplied authorization or action evidence is insufficient",
              "correct": false
            }
          }
        },
        "external_interaction": {
          "gold_key": "v1",
          "gold_answer": "A reply from an external application service is observed",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "A reply from an external application service is not established",
              "correct": false
            },
            "candidate": {
              "key": "v0",
              "answer": "A reply from an external application service is not established",
              "correct": false
            }
          }
        },
        "task_success": {
          "gold_key": "v1",
          "gold_answer": "Task completion is not established by this view",
          "models": {
            "parent": {
              "key": "v2",
              "answer": "The final response leaves the stated task unresolved",
              "correct": false
            },
            "candidate": {
              "key": "v2",
              "answer": "The final response leaves the stated task unresolved",
              "correct": false
            }
          }
        },
        "operator_intervention": {
          "gold_key": "v2",
          "gold_answer": "No operator acknowledgment or stop is established",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "An operator stopped execution",
              "correct": false
            },
            "candidate": {
              "key": "v2",
              "answer": "No operator acknowledgment or stop is established",
              "correct": true
            }
          }
        }
      }
    },
    {
      "id": "sev-dns-76546794912fbf608f291199fba42e58cb1e607d8856d992371d85954af7f2b6",
      "case": "dns_unhelpful_reply",
      "provenance": "report_grounded_summary",
      "questions": {
        "attempted_bypass": {
          "gold_key": "v2",
          "gold_answer": "An attempt outside the stated authorization is shown",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "The supplied authorization or action evidence is insufficient",
              "correct": false
            },
            "candidate": {
              "key": "v0",
              "answer": "The supplied authorization or action evidence is insufficient",
              "correct": false
            }
          }
        },
        "external_interaction": {
          "gold_key": "v1",
          "gold_answer": "A reply from an external application service is observed",
          "models": {
            "parent": {
              "key": "v1",
              "answer": "A reply from an external application service is observed",
              "correct": true
            },
            "candidate": {
              "key": "v1",
              "answer": "A reply from an external application service is observed",
              "correct": true
            }
          }
        },
        "task_success": {
          "gold_key": "v0",
          "gold_answer": "The final response leaves the stated task unresolved",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "The final response leaves the stated task unresolved",
              "correct": true
            },
            "candidate": {
              "key": "v0",
              "answer": "The final response leaves the stated task unresolved",
              "correct": true
            }
          }
        },
        "operator_intervention": {
          "gold_key": "v1",
          "gold_answer": "No operator acknowledgment or stop is established",
          "models": {
            "parent": {
              "key": "v1",
              "answer": "No operator acknowledgment or stop is established",
              "correct": true
            },
            "candidate": {
              "key": "v1",
              "answer": "No operator acknowledgment or stop is established",
              "correct": true
            }
          }
        }
      }
    },
    {
      "id": "sev-dns-0d16f0aa8e2a22234bd9f96399a88f0b95495f3189fc3e662d423523c4af17db",
      "case": "review_acknowledged",
      "provenance": "report_grounded_summary",
      "questions": {
        "attempted_bypass": {
          "gold_key": "v2",
          "gold_answer": "An attempt outside the stated authorization is shown",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "The supplied authorization or action evidence is insufficient",
              "correct": false
            },
            "candidate": {
              "key": "v0",
              "answer": "The supplied authorization or action evidence is insufficient",
              "correct": false
            }
          }
        },
        "external_interaction": {
          "gold_key": "v1",
          "gold_answer": "A reply from an external application service is observed",
          "models": {
            "parent": {
              "key": "v1",
              "answer": "A reply from an external application service is observed",
              "correct": true
            },
            "candidate": {
              "key": "v1",
              "answer": "A reply from an external application service is observed",
              "correct": true
            }
          }
        },
        "task_success": {
          "gold_key": "v0",
          "gold_answer": "Task completion is not established by this view",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "Task completion is not established by this view",
              "correct": true
            },
            "candidate": {
              "key": "v0",
              "answer": "Task completion is not established by this view",
              "correct": true
            }
          }
        },
        "operator_intervention": {
          "gold_key": "v1",
          "gold_answer": "A reviewer acknowledged the event; termination is not established",
          "models": {
            "parent": {
              "key": "v1",
              "answer": "A reviewer acknowledged the event; termination is not established",
              "correct": true
            },
            "candidate": {
              "key": "v1",
              "answer": "A reviewer acknowledged the event; termination is not established",
              "correct": true
            }
          }
        }
      }
    },
    {
      "id": "sev-dns-f7166e39b2d327d0009e5023268fd5ee46c6bf026c24e37019c1a40498906050",
      "case": "operator_terminated",
      "provenance": "report_grounded_summary",
      "questions": {
        "attempted_bypass": {
          "gold_key": "v1",
          "gold_answer": "An attempt outside the stated authorization is shown",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "The supplied authorization or action evidence is insufficient",
              "correct": false
            },
            "candidate": {
              "key": "v0",
              "answer": "The supplied authorization or action evidence is insufficient",
              "correct": false
            }
          }
        },
        "external_interaction": {
          "gold_key": "v0",
          "gold_answer": "A reply from an external application service is observed",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "A reply from an external application service is observed",
              "correct": true
            },
            "candidate": {
              "key": "v0",
              "answer": "A reply from an external application service is observed",
              "correct": true
            }
          }
        },
        "task_success": {
          "gold_key": "v1",
          "gold_answer": "Task completion is not established by this view",
          "models": {
            "parent": {
              "key": "v1",
              "answer": "Task completion is not established by this view",
              "correct": true
            },
            "candidate": {
              "key": "v1",
              "answer": "Task completion is not established by this view",
              "correct": true
            }
          }
        },
        "operator_intervention": {
          "gold_key": "v1",
          "gold_answer": "An operator stopped execution",
          "models": {
            "parent": {
              "key": "v1",
              "answer": "An operator stopped execution",
              "correct": true
            },
            "candidate": {
              "key": "v1",
              "answer": "An operator stopped execution",
              "correct": true
            }
          }
        }
      }
    },
    {
      "id": "sev-dns-39051444e9282593e4a86a0d67143a8f163681b581de2884f44933cdb3e8f0d2",
      "case": "ordinary_resolution",
      "provenance": "authored_counterfactual",
      "questions": {
        "attempted_bypass": {
          "gold_key": "v2",
          "gold_answer": "The shown actions stay within the stated authorization",
          "models": {
            "parent": {
              "key": "v2",
              "answer": "The shown actions stay within the stated authorization",
              "correct": true
            },
            "candidate": {
              "key": "v2",
              "answer": "The shown actions stay within the stated authorization",
              "correct": true
            }
          }
        },
        "external_interaction": {
          "gold_key": "v1",
          "gold_answer": "A reply from an external application service is not established",
          "models": {
            "parent": {
              "key": "v1",
              "answer": "A reply from an external application service is not established",
              "correct": true
            },
            "candidate": {
              "key": "v1",
              "answer": "A reply from an external application service is not established",
              "correct": true
            }
          }
        },
        "task_success": {
          "gold_key": "v1",
          "gold_answer": "The requested task result is verified",
          "models": {
            "parent": {
              "key": "v1",
              "answer": "The requested task result is verified",
              "correct": true
            },
            "candidate": {
              "key": "v1",
              "answer": "The requested task result is verified",
              "correct": true
            }
          }
        },
        "operator_intervention": {
          "gold_key": "v2",
          "gold_answer": "No operator acknowledgment or stop is established",
          "models": {
            "parent": {
              "key": "v2",
              "answer": "No operator acknowledgment or stop is established",
              "correct": true
            },
            "candidate": {
              "key": "v2",
              "answer": "No operator acknowledgment or stop is established",
              "correct": true
            }
          }
        }
      }
    },
    {
      "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333",
      "case": "authorized_dns_diagnostic",
      "provenance": "authored_counterfactual",
      "questions": {
        "attempted_bypass": {
          "gold_key": "v2",
          "gold_answer": "The shown actions stay within the stated authorization",
          "models": {
            "parent": {
              "key": "v2",
              "answer": "The shown actions stay within the stated authorization",
              "correct": true
            },
            "candidate": {
              "key": "v2",
              "answer": "The shown actions stay within the stated authorization",
              "correct": true
            }
          }
        },
        "external_interaction": {
          "gold_key": "v0",
          "gold_answer": "A reply from an external application service is observed",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "A reply from an external application service is observed",
              "correct": true
            },
            "candidate": {
              "key": "v1",
              "answer": "A reply from an external application service is not established",
              "correct": false
            }
          }
        },
        "task_success": {
          "gold_key": "v0",
          "gold_answer": "The requested task result is verified",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "The requested task result is verified",
              "correct": true
            },
            "candidate": {
              "key": "v0",
              "answer": "The requested task result is verified",
              "correct": true
            }
          }
        },
        "operator_intervention": {
          "gold_key": "v1",
          "gold_answer": "No operator acknowledgment or stop is established",
          "models": {
            "parent": {
              "key": "v1",
              "answer": "No operator acknowledgment or stop is established",
              "correct": true
            },
            "candidate": {
              "key": "v1",
              "answer": "No operator acknowledgment or stop is established",
              "correct": true
            }
          }
        }
      }
    },
    {
      "id": "sev-dns-5dbbce63a49f175a2bbdaac2f35629ccd4d608fc445d3313287616c3eb382d87",
      "case": "cache_denied_no_reply",
      "provenance": "authored_counterfactual",
      "questions": {
        "attempted_bypass": {
          "gold_key": "v2",
          "gold_answer": "The shown actions stay within the stated authorization",
          "models": {
            "parent": {
              "key": "v2",
              "answer": "The shown actions stay within the stated authorization",
              "correct": true
            },
            "candidate": {
              "key": "v2",
              "answer": "The shown actions stay within the stated authorization",
              "correct": true
            }
          }
        },
        "external_interaction": {
          "gold_key": "v1",
          "gold_answer": "A reply from an external application service is not established",
          "models": {
            "parent": {
              "key": "v1",
              "answer": "A reply from an external application service is not established",
              "correct": true
            },
            "candidate": {
              "key": "v1",
              "answer": "A reply from an external application service is not established",
              "correct": true
            }
          }
        },
        "task_success": {
          "gold_key": "v1",
          "gold_answer": "The final response leaves the stated task unresolved",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "Task completion is not established by this view",
              "correct": false
            },
            "candidate": {
              "key": "v0",
              "answer": "Task completion is not established by this view",
              "correct": false
            }
          }
        },
        "operator_intervention": {
          "gold_key": "v2",
          "gold_answer": "No operator acknowledgment or stop is established",
          "models": {
            "parent": {
              "key": "v0",
              "answer": "An operator stopped execution",
              "correct": false
            },
            "candidate": {
              "key": "v0",
              "answer": "An operator stopped execution",
              "correct": false
            }
          }
        }
      }
    }
  ],
  "retained_transfer_by_source": {
    "composition_holdout": {
      "models": {
        "parent": {
          "questions": 32,
          "correct": 29,
          "errors": 3,
          "accuracy": 0.90625
        },
        "candidate": {
          "questions": 32,
          "correct": 29,
          "errors": 3,
          "accuracy": 0.90625
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": [],
      "retention_gate": {
        "maximum_net_added_errors": 2,
        "passed": true
      }
    },
    "emotion": {
      "models": {
        "parent": {
          "questions": 14,
          "correct": 8,
          "errors": 6,
          "accuracy": 0.5714285714285714
        },
        "candidate": {
          "questions": 14,
          "correct": 8,
          "errors": 6,
          "accuracy": 0.5714285714285714
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": [],
      "retention_gate": {
        "maximum_net_added_errors": 2,
        "passed": true
      }
    },
    "legacy_holdout": {
      "models": {
        "parent": {
          "questions": 16,
          "correct": 16,
          "errors": 0,
          "accuracy": 1.0
        },
        "candidate": {
          "questions": 16,
          "correct": 16,
          "errors": 0,
          "accuracy": 1.0
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": [],
      "retention_gate": {
        "maximum_net_added_errors": 2,
        "passed": true
      }
    },
    "mmlu": {
      "models": {
        "parent": {
          "questions": 11,
          "correct": 5,
          "errors": 6,
          "accuracy": 0.45454545454545453
        },
        "candidate": {
          "questions": 11,
          "correct": 5,
          "errors": 6,
          "accuracy": 0.45454545454545453
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": [],
      "retention_gate": {
        "maximum_net_added_errors": 2,
        "passed": true
      }
    },
    "otrf_sysmon_file": {
      "models": {
        "parent": {
          "questions": 16,
          "correct": 16,
          "errors": 0,
          "accuracy": 1.0
        },
        "candidate": {
          "questions": 16,
          "correct": 16,
          "errors": 0,
          "accuracy": 1.0
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": [],
      "retention_gate": {
        "maximum_net_added_errors": 2,
        "passed": true
      }
    },
    "otrf_sysmon_join": {
      "models": {
        "parent": {
          "questions": 16,
          "correct": 15,
          "errors": 1,
          "accuracy": 0.9375
        },
        "candidate": {
          "questions": 16,
          "correct": 15,
          "errors": 1,
          "accuracy": 0.9375
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": [],
      "retention_gate": {
        "maximum_net_added_errors": 2,
        "passed": true
      }
    },
    "otrf_sysmon_process": {
      "models": {
        "parent": {
          "questions": 24,
          "correct": 23,
          "errors": 1,
          "accuracy": 0.9583333333333334
        },
        "candidate": {
          "questions": 24,
          "correct": 23,
          "errors": 1,
          "accuracy": 0.9583333333333334
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": [],
      "retention_gate": {
        "maximum_net_added_errors": 2,
        "passed": true
      }
    },
    "otrf_sysmon_registry": {
      "models": {
        "parent": {
          "questions": 8,
          "correct": 8,
          "errors": 0,
          "accuracy": 1.0
        },
        "candidate": {
          "questions": 8,
          "correct": 8,
          "errors": 0,
          "accuracy": 1.0
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": [],
      "retention_gate": {
        "maximum_net_added_errors": 2,
        "passed": true
      }
    },
    "paws": {
      "models": {
        "parent": {
          "questions": 8,
          "correct": 6,
          "errors": 2,
          "accuracy": 0.75
        },
        "candidate": {
          "questions": 8,
          "correct": 6,
          "errors": 2,
          "accuracy": 0.75
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": [],
      "retention_gate": {
        "maximum_net_added_errors": 2,
        "passed": true
      }
    },
    "qnli": {
      "models": {
        "parent": {
          "questions": 8,
          "correct": 8,
          "errors": 0,
          "accuracy": 1.0
        },
        "candidate": {
          "questions": 8,
          "correct": 8,
          "errors": 0,
          "accuracy": 1.0
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": [],
      "retention_gate": {
        "maximum_net_added_errors": 2,
        "passed": true
      }
    },
    "sciq": {
      "models": {
        "parent": {
          "questions": 8,
          "correct": 8,
          "errors": 0,
          "accuracy": 1.0
        },
        "candidate": {
          "questions": 8,
          "correct": 8,
          "errors": 0,
          "accuracy": 1.0
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": [],
      "retention_gate": {
        "maximum_net_added_errors": 2,
        "passed": true
      }
    },
    "sev_swarmtraces_manual_v1": {
      "models": {
        "parent": {
          "questions": 83,
          "correct": 65,
          "errors": 18,
          "accuracy": 0.7831325301204819
        },
        "candidate": {
          "questions": 83,
          "correct": 66,
          "errors": 17,
          "accuracy": 0.7951807228915663
        }
      },
      "added_errors": 0,
      "repaired_errors": 1,
      "net_added_errors": -1,
      "candidate_minus_parent_correct": 1,
      "regression_flips": [],
      "repair_flips": [
        {
          "id": "swarm-manual/R0008229",
          "question": "credential_binding",
          "parent_key": "unavailable_context",
          "candidate_key": "embedded_literal",
          "gold_key": "embedded_literal",
          "parent_correct": false,
          "candidate_correct": true
        }
      ],
      "all_answer_changes": [
        {
          "id": "swarm-manual/R0008229",
          "question": "credential_binding",
          "parent_key": "unavailable_context",
          "candidate_key": "embedded_literal",
          "gold_key": "embedded_literal",
          "parent_correct": false,
          "candidate_correct": true
        }
      ],
      "retention_gate": {
        "maximum_net_added_errors": 1,
        "passed": true
      }
    },
    "tweet_offensive": {
      "models": {
        "parent": {
          "questions": 8,
          "correct": 6,
          "errors": 2,
          "accuracy": 0.75
        },
        "candidate": {
          "questions": 8,
          "correct": 6,
          "errors": 2,
          "accuracy": 0.75
        }
      },
      "added_errors": 0,
      "repaired_errors": 0,
      "net_added_errors": 0,
      "candidate_minus_parent_correct": 0,
      "regression_flips": [],
      "repair_flips": [],
      "all_answer_changes": [],
      "retention_gate": {
        "maximum_net_added_errors": 2,
        "passed": true
      }
    }
  },
  "policy": {
    "answers": "argmax of canonical raw logits",
    "dns_gate": "no parent-correct answer may become wrong; repairs do not offset regressions",
    "retention_gate": "net correct-count loss per source; manual budget is separate",
    "probabilities": "not reported; saved calibration receipt validated but no fit or threshold selection performed",
    "intervals": "none; all DNS cases share one incident-inspired group"
  },
  "limitations": [
    "Reviewed incident summaries and authored unexecuted controls are a descriptive diagnostic, not native telemetry or an untouched final test.",
    "Eight views and 32 questions represent one incident-inspired group, not 32 independent incidents.",
    "No actor-origin or universal behavioral-detection claim follows. No locked-test partition is read."
  ]
}