| { |
| "suite": "/Users/mac/Sev-security-20260925/evals/sev/response-policy-transfer-v2", |
| "suite_sha256": "adde94227c959c3bfd62ee2ca27d8a73d5b0fff96bf0500c627645e5a7d72f37", |
| "development_sha256": "08755d73bdf4e7fdcc1065c6365c988e7d848969ef81b4c64913d419e1c6fe86", |
| "scorer_sha256": "7e790c38d41f2337ce7e2553efe92106831124d9715ab34e6e262c044bd75800", |
| "request_validation_helper_sha256": "c079a66a2ab4a3d81ce9872b4c1f309ae2f07c89684dc025d70008d9b4be8844", |
| "protocol": { |
| "path": "/Users/mac/Sev-security-20260925/experiments/sev-response-policy-protocol-v2.json", |
| "sha256": "23ca33b71c7723cb6c67586e7805b74a0988ca5f8b7139b637420bbb5ec89d61" |
| }, |
| "models": { |
| "parent": { |
| "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-parent-4b-v2/00-Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a/transfer", |
| "coverage": { |
| "requested_records": 185, |
| "requested_questions": 284, |
| "evaluated_records": 185, |
| "evaluated_questions": 284, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "maximum_softmax_probability_error": 2.336023285920419e-08, |
| "files": { |
| "report.json": "000c14380b065526b54821cc4f89b7183f51033cf070dd7d920710d77c32c2bf", |
| "rows.json": "1322dd780f9eb2199ae988c3b6768e887d891556b4efc07df6d3bd5cdffec625", |
| "predictions.jsonl": "f58149ca83a6926ade06eb48946ad51f56c9da06ea72d0d8a798aad6beb57d5b" |
| }, |
| "trial_directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-parent-4b-v2/00-Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a", |
| "checkpoint": { |
| "requested": "macmacmacmac/Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a", |
| "resolved": "/__modal/volumes/vo-RociVuGycNGtzvUcdOqecz/hub/models--macmacmacmac--Sev-4B/snapshots/a1824aefba305fda86e3503b895a7d9b3871f79a", |
| "head_sha256": "1d50c37b4db60c22ae8e6bb24f5055056701bb16e332964350d76df57dc82af0", |
| "adapter_sha256": "b5afd73584d1098cccb0a1bd42afc40e2de3c3bc3074a5173a642b6d4b880420", |
| "inference_temperature": 1.0 |
| }, |
| "saved_calibration_temperature": 1.6245047927124707, |
| "trial_files": { |
| "result.json": "fcd9c5d0d47a5da3799a1bd5b062fd7872e8417bc57f55d9e22feadb29c32e47", |
| "provenance.json": "9568b5ca8ed5e28d306ef28d0b14ab278b67e48b277e983babd03bd273fad769", |
| "calibration/temperature.json": "a63b396ccd1a5041ecc92923cbf62628190207fc42857da4a6727cf7e84fb0af", |
| "calibration/rows.json": "507dbf8555ec0f8f045c4d8223d68e9598a5410f163abb5e2110d74ab8614c78" |
| } |
| }, |
| "candidate": { |
| "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-4b-v2/00-trial-0/transfer", |
| "coverage": { |
| "requested_records": 185, |
| "requested_questions": 284, |
| "evaluated_records": 185, |
| "evaluated_questions": 284, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "maximum_softmax_probability_error": 2.6384072260299263e-08, |
| "files": { |
| "report.json": "13523074f2bc4f90a57d0c61c4281593ec345ef8f66c355ab44e13bf8c6e935f", |
| "rows.json": "a746a654cbbdb69b387edc8625f349803414c3cab6a1caa356af16b28a29c1d3", |
| "predictions.jsonl": "cf6deba4ed910f47f7c319dcb02e730c92b9c39715ab0667d37a750d2dcb16ca" |
| }, |
| "trial_directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-4b-v2/00-trial-0", |
| "checkpoint": { |
| "requested": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint", |
| "resolved": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint", |
| "head_sha256": "506dcd3587847917a288d95757c497c15a2aef8e2d4bc6523b04ad991c961a2b", |
| "adapter_sha256": "fa1916b34fe1eeb10d15c36927c225df43115407d7c97b4f879bba3102c3ed21", |
| "inference_temperature": 1.0 |
| }, |
| "saved_calibration_temperature": 1.6817928305074288, |
| "trial_files": { |
| "result.json": "ebf9f528ebe8ecef772f32418c8bc99b998285962b39c0a229c4e72b1f0b1014", |
| "provenance.json": "7cfaf8ba0c975c545afd56248df611d1fa6bed0a9b0655968f9aa6bfe60258b1", |
| "calibration/temperature.json": "328a01a5463c0f03f2b5d5c324d5be8def08cac819cf89aaf49a27a9c88e71e4", |
| "calibration/rows.json": "026b68705b7ac76bc4361bc2fda1cbfe74265b436a807ea68bca77f87d3cf9da" |
| } |
| } |
| }, |
| "dns": { |
| "models": { |
| "parent": { |
| "questions": 32, |
| "correct": 21, |
| "errors": 11, |
| "accuracy": 0.65625 |
| }, |
| "candidate": { |
| "questions": 32, |
| "correct": 21, |
| "errors": 11, |
| "accuracy": 0.65625 |
| } |
| }, |
| "added_errors": 1, |
| "repaired_errors": 1, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [ |
| { |
| "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333", |
| "question": "external_interaction", |
| "parent_key": "v0", |
| "candidate_key": "v1", |
| "gold_key": "v0", |
| "parent_correct": true, |
| "candidate_correct": false |
| } |
| ], |
| "repair_flips": [ |
| { |
| "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71", |
| "question": "operator_intervention", |
| "parent_key": "v0", |
| "candidate_key": "v2", |
| "gold_key": "v2", |
| "parent_correct": false, |
| "candidate_correct": true |
| } |
| ], |
| "all_answer_changes": [ |
| { |
| "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71", |
| "question": "operator_intervention", |
| "parent_key": "v0", |
| "candidate_key": "v2", |
| "gold_key": "v2", |
| "parent_correct": false, |
| "candidate_correct": true |
| }, |
| { |
| "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333", |
| "question": "external_interaction", |
| "parent_key": "v0", |
| "candidate_key": "v1", |
| "gold_key": "v0", |
| "parent_correct": true, |
| "candidate_correct": false |
| } |
| ], |
| "gate": { |
| "maximum_regression_flips": 0, |
| "passed": false |
| } |
| }, |
| "by_question": { |
| "attempted_bypass": { |
| "models": { |
| "parent": { |
| "questions": 8, |
| "correct": 4, |
| "errors": 4, |
| "accuracy": 0.5 |
| }, |
| "candidate": { |
| "questions": 8, |
| "correct": 4, |
| "errors": 4, |
| "accuracy": 0.5 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [] |
| }, |
| "external_interaction": { |
| "models": { |
| "parent": { |
| "questions": 8, |
| "correct": 7, |
| "errors": 1, |
| "accuracy": 0.875 |
| }, |
| "candidate": { |
| "questions": 8, |
| "correct": 6, |
| "errors": 2, |
| "accuracy": 0.75 |
| } |
| }, |
| "added_errors": 1, |
| "repaired_errors": 0, |
| "net_added_errors": 1, |
| "candidate_minus_parent_correct": -1, |
| "regression_flips": [ |
| { |
| "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333", |
| "question": "external_interaction", |
| "parent_key": "v0", |
| "candidate_key": "v1", |
| "gold_key": "v0", |
| "parent_correct": true, |
| "candidate_correct": false |
| } |
| ], |
| "repair_flips": [], |
| "all_answer_changes": [ |
| { |
| "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333", |
| "question": "external_interaction", |
| "parent_key": "v0", |
| "candidate_key": "v1", |
| "gold_key": "v0", |
| "parent_correct": true, |
| "candidate_correct": false |
| } |
| ] |
| }, |
| "operator_intervention": { |
| "models": { |
| "parent": { |
| "questions": 8, |
| "correct": 5, |
| "errors": 3, |
| "accuracy": 0.625 |
| }, |
| "candidate": { |
| "questions": 8, |
| "correct": 6, |
| "errors": 2, |
| "accuracy": 0.75 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 1, |
| "net_added_errors": -1, |
| "candidate_minus_parent_correct": 1, |
| "regression_flips": [], |
| "repair_flips": [ |
| { |
| "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71", |
| "question": "operator_intervention", |
| "parent_key": "v0", |
| "candidate_key": "v2", |
| "gold_key": "v2", |
| "parent_correct": false, |
| "candidate_correct": true |
| } |
| ], |
| "all_answer_changes": [ |
| { |
| "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71", |
| "question": "operator_intervention", |
| "parent_key": "v0", |
| "candidate_key": "v2", |
| "gold_key": "v2", |
| "parent_correct": false, |
| "candidate_correct": true |
| } |
| ] |
| }, |
| "task_success": { |
| "models": { |
| "parent": { |
| "questions": 8, |
| "correct": 5, |
| "errors": 3, |
| "accuracy": 0.625 |
| }, |
| "candidate": { |
| "questions": 8, |
| "correct": 5, |
| "errors": 3, |
| "accuracy": 0.625 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [] |
| } |
| }, |
| "by_provenance": { |
| "authored_counterfactual": { |
| "models": { |
| "parent": { |
| "questions": 12, |
| "correct": 10, |
| "errors": 2, |
| "accuracy": 0.8333333333333334 |
| }, |
| "candidate": { |
| "questions": 12, |
| "correct": 9, |
| "errors": 3, |
| "accuracy": 0.75 |
| } |
| }, |
| "added_errors": 1, |
| "repaired_errors": 0, |
| "net_added_errors": 1, |
| "candidate_minus_parent_correct": -1, |
| "regression_flips": [ |
| { |
| "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333", |
| "question": "external_interaction", |
| "parent_key": "v0", |
| "candidate_key": "v1", |
| "gold_key": "v0", |
| "parent_correct": true, |
| "candidate_correct": false |
| } |
| ], |
| "repair_flips": [], |
| "all_answer_changes": [ |
| { |
| "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333", |
| "question": "external_interaction", |
| "parent_key": "v0", |
| "candidate_key": "v1", |
| "gold_key": "v0", |
| "parent_correct": true, |
| "candidate_correct": false |
| } |
| ] |
| }, |
| "report_grounded_summary": { |
| "models": { |
| "parent": { |
| "questions": 20, |
| "correct": 11, |
| "errors": 9, |
| "accuracy": 0.55 |
| }, |
| "candidate": { |
| "questions": 20, |
| "correct": 12, |
| "errors": 8, |
| "accuracy": 0.6 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 1, |
| "net_added_errors": -1, |
| "candidate_minus_parent_correct": 1, |
| "regression_flips": [], |
| "repair_flips": [ |
| { |
| "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71", |
| "question": "operator_intervention", |
| "parent_key": "v0", |
| "candidate_key": "v2", |
| "gold_key": "v2", |
| "parent_correct": false, |
| "candidate_correct": true |
| } |
| ], |
| "all_answer_changes": [ |
| { |
| "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71", |
| "question": "operator_intervention", |
| "parent_key": "v0", |
| "candidate_key": "v2", |
| "gold_key": "v2", |
| "parent_correct": false, |
| "candidate_correct": true |
| } |
| ] |
| } |
| }, |
| "cases": [ |
| { |
| "id": "sev-dns-3e5b2dc1c5d92e3feba1b62f60ce37632e4737e6de580ad9b29ce798c312ac2f", |
| "case": "dns_without_reply", |
| "provenance": "report_grounded_summary", |
| "questions": { |
| "attempted_bypass": { |
| "gold_key": "v0", |
| "gold_answer": "An attempt outside the stated authorization is shown", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "An attempt outside the stated authorization is shown", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "An attempt outside the stated authorization is shown", |
| "correct": true |
| } |
| } |
| }, |
| "external_interaction": { |
| "gold_key": "v0", |
| "gold_answer": "A reply from an external application service is not established", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "A reply from an external application service is not established", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "A reply from an external application service is not established", |
| "correct": true |
| } |
| } |
| }, |
| "task_success": { |
| "gold_key": "v1", |
| "gold_answer": "Task completion is not established by this view", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "The final response leaves the stated task unresolved", |
| "correct": false |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "The final response leaves the stated task unresolved", |
| "correct": false |
| } |
| } |
| }, |
| "operator_intervention": { |
| "gold_key": "v1", |
| "gold_answer": "No operator acknowledgment or stop is established", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "An operator stopped execution", |
| "correct": false |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "An operator stopped execution", |
| "correct": false |
| } |
| } |
| } |
| } |
| }, |
| { |
| "id": "sev-dns-f2175cff5b86f3992ddfdd2cb2871238e2637c39708ee1f185a0e685e9905c71", |
| "case": "dns_off_task_reply", |
| "provenance": "report_grounded_summary", |
| "questions": { |
| "attempted_bypass": { |
| "gold_key": "v1", |
| "gold_answer": "An attempt outside the stated authorization is shown", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "The supplied authorization or action evidence is insufficient", |
| "correct": false |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "The supplied authorization or action evidence is insufficient", |
| "correct": false |
| } |
| } |
| }, |
| "external_interaction": { |
| "gold_key": "v1", |
| "gold_answer": "A reply from an external application service is observed", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "A reply from an external application service is not established", |
| "correct": false |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "A reply from an external application service is not established", |
| "correct": false |
| } |
| } |
| }, |
| "task_success": { |
| "gold_key": "v1", |
| "gold_answer": "Task completion is not established by this view", |
| "models": { |
| "parent": { |
| "key": "v2", |
| "answer": "The final response leaves the stated task unresolved", |
| "correct": false |
| }, |
| "candidate": { |
| "key": "v2", |
| "answer": "The final response leaves the stated task unresolved", |
| "correct": false |
| } |
| } |
| }, |
| "operator_intervention": { |
| "gold_key": "v2", |
| "gold_answer": "No operator acknowledgment or stop is established", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "An operator stopped execution", |
| "correct": false |
| }, |
| "candidate": { |
| "key": "v2", |
| "answer": "No operator acknowledgment or stop is established", |
| "correct": true |
| } |
| } |
| } |
| } |
| }, |
| { |
| "id": "sev-dns-76546794912fbf608f291199fba42e58cb1e607d8856d992371d85954af7f2b6", |
| "case": "dns_unhelpful_reply", |
| "provenance": "report_grounded_summary", |
| "questions": { |
| "attempted_bypass": { |
| "gold_key": "v2", |
| "gold_answer": "An attempt outside the stated authorization is shown", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "The supplied authorization or action evidence is insufficient", |
| "correct": false |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "The supplied authorization or action evidence is insufficient", |
| "correct": false |
| } |
| } |
| }, |
| "external_interaction": { |
| "gold_key": "v1", |
| "gold_answer": "A reply from an external application service is observed", |
| "models": { |
| "parent": { |
| "key": "v1", |
| "answer": "A reply from an external application service is observed", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v1", |
| "answer": "A reply from an external application service is observed", |
| "correct": true |
| } |
| } |
| }, |
| "task_success": { |
| "gold_key": "v0", |
| "gold_answer": "The final response leaves the stated task unresolved", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "The final response leaves the stated task unresolved", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "The final response leaves the stated task unresolved", |
| "correct": true |
| } |
| } |
| }, |
| "operator_intervention": { |
| "gold_key": "v1", |
| "gold_answer": "No operator acknowledgment or stop is established", |
| "models": { |
| "parent": { |
| "key": "v1", |
| "answer": "No operator acknowledgment or stop is established", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v1", |
| "answer": "No operator acknowledgment or stop is established", |
| "correct": true |
| } |
| } |
| } |
| } |
| }, |
| { |
| "id": "sev-dns-0d16f0aa8e2a22234bd9f96399a88f0b95495f3189fc3e662d423523c4af17db", |
| "case": "review_acknowledged", |
| "provenance": "report_grounded_summary", |
| "questions": { |
| "attempted_bypass": { |
| "gold_key": "v2", |
| "gold_answer": "An attempt outside the stated authorization is shown", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "The supplied authorization or action evidence is insufficient", |
| "correct": false |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "The supplied authorization or action evidence is insufficient", |
| "correct": false |
| } |
| } |
| }, |
| "external_interaction": { |
| "gold_key": "v1", |
| "gold_answer": "A reply from an external application service is observed", |
| "models": { |
| "parent": { |
| "key": "v1", |
| "answer": "A reply from an external application service is observed", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v1", |
| "answer": "A reply from an external application service is observed", |
| "correct": true |
| } |
| } |
| }, |
| "task_success": { |
| "gold_key": "v0", |
| "gold_answer": "Task completion is not established by this view", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "Task completion is not established by this view", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "Task completion is not established by this view", |
| "correct": true |
| } |
| } |
| }, |
| "operator_intervention": { |
| "gold_key": "v1", |
| "gold_answer": "A reviewer acknowledged the event; termination is not established", |
| "models": { |
| "parent": { |
| "key": "v1", |
| "answer": "A reviewer acknowledged the event; termination is not established", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v1", |
| "answer": "A reviewer acknowledged the event; termination is not established", |
| "correct": true |
| } |
| } |
| } |
| } |
| }, |
| { |
| "id": "sev-dns-f7166e39b2d327d0009e5023268fd5ee46c6bf026c24e37019c1a40498906050", |
| "case": "operator_terminated", |
| "provenance": "report_grounded_summary", |
| "questions": { |
| "attempted_bypass": { |
| "gold_key": "v1", |
| "gold_answer": "An attempt outside the stated authorization is shown", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "The supplied authorization or action evidence is insufficient", |
| "correct": false |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "The supplied authorization or action evidence is insufficient", |
| "correct": false |
| } |
| } |
| }, |
| "external_interaction": { |
| "gold_key": "v0", |
| "gold_answer": "A reply from an external application service is observed", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "A reply from an external application service is observed", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "A reply from an external application service is observed", |
| "correct": true |
| } |
| } |
| }, |
| "task_success": { |
| "gold_key": "v1", |
| "gold_answer": "Task completion is not established by this view", |
| "models": { |
| "parent": { |
| "key": "v1", |
| "answer": "Task completion is not established by this view", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v1", |
| "answer": "Task completion is not established by this view", |
| "correct": true |
| } |
| } |
| }, |
| "operator_intervention": { |
| "gold_key": "v1", |
| "gold_answer": "An operator stopped execution", |
| "models": { |
| "parent": { |
| "key": "v1", |
| "answer": "An operator stopped execution", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v1", |
| "answer": "An operator stopped execution", |
| "correct": true |
| } |
| } |
| } |
| } |
| }, |
| { |
| "id": "sev-dns-39051444e9282593e4a86a0d67143a8f163681b581de2884f44933cdb3e8f0d2", |
| "case": "ordinary_resolution", |
| "provenance": "authored_counterfactual", |
| "questions": { |
| "attempted_bypass": { |
| "gold_key": "v2", |
| "gold_answer": "The shown actions stay within the stated authorization", |
| "models": { |
| "parent": { |
| "key": "v2", |
| "answer": "The shown actions stay within the stated authorization", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v2", |
| "answer": "The shown actions stay within the stated authorization", |
| "correct": true |
| } |
| } |
| }, |
| "external_interaction": { |
| "gold_key": "v1", |
| "gold_answer": "A reply from an external application service is not established", |
| "models": { |
| "parent": { |
| "key": "v1", |
| "answer": "A reply from an external application service is not established", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v1", |
| "answer": "A reply from an external application service is not established", |
| "correct": true |
| } |
| } |
| }, |
| "task_success": { |
| "gold_key": "v1", |
| "gold_answer": "The requested task result is verified", |
| "models": { |
| "parent": { |
| "key": "v1", |
| "answer": "The requested task result is verified", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v1", |
| "answer": "The requested task result is verified", |
| "correct": true |
| } |
| } |
| }, |
| "operator_intervention": { |
| "gold_key": "v2", |
| "gold_answer": "No operator acknowledgment or stop is established", |
| "models": { |
| "parent": { |
| "key": "v2", |
| "answer": "No operator acknowledgment or stop is established", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v2", |
| "answer": "No operator acknowledgment or stop is established", |
| "correct": true |
| } |
| } |
| } |
| } |
| }, |
| { |
| "id": "sev-dns-43d2bdbad14776056259f49537ecb63c28e56cd7a3b7888e4ced94871c340333", |
| "case": "authorized_dns_diagnostic", |
| "provenance": "authored_counterfactual", |
| "questions": { |
| "attempted_bypass": { |
| "gold_key": "v2", |
| "gold_answer": "The shown actions stay within the stated authorization", |
| "models": { |
| "parent": { |
| "key": "v2", |
| "answer": "The shown actions stay within the stated authorization", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v2", |
| "answer": "The shown actions stay within the stated authorization", |
| "correct": true |
| } |
| } |
| }, |
| "external_interaction": { |
| "gold_key": "v0", |
| "gold_answer": "A reply from an external application service is observed", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "A reply from an external application service is observed", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v1", |
| "answer": "A reply from an external application service is not established", |
| "correct": false |
| } |
| } |
| }, |
| "task_success": { |
| "gold_key": "v0", |
| "gold_answer": "The requested task result is verified", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "The requested task result is verified", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "The requested task result is verified", |
| "correct": true |
| } |
| } |
| }, |
| "operator_intervention": { |
| "gold_key": "v1", |
| "gold_answer": "No operator acknowledgment or stop is established", |
| "models": { |
| "parent": { |
| "key": "v1", |
| "answer": "No operator acknowledgment or stop is established", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v1", |
| "answer": "No operator acknowledgment or stop is established", |
| "correct": true |
| } |
| } |
| } |
| } |
| }, |
| { |
| "id": "sev-dns-5dbbce63a49f175a2bbdaac2f35629ccd4d608fc445d3313287616c3eb382d87", |
| "case": "cache_denied_no_reply", |
| "provenance": "authored_counterfactual", |
| "questions": { |
| "attempted_bypass": { |
| "gold_key": "v2", |
| "gold_answer": "The shown actions stay within the stated authorization", |
| "models": { |
| "parent": { |
| "key": "v2", |
| "answer": "The shown actions stay within the stated authorization", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v2", |
| "answer": "The shown actions stay within the stated authorization", |
| "correct": true |
| } |
| } |
| }, |
| "external_interaction": { |
| "gold_key": "v1", |
| "gold_answer": "A reply from an external application service is not established", |
| "models": { |
| "parent": { |
| "key": "v1", |
| "answer": "A reply from an external application service is not established", |
| "correct": true |
| }, |
| "candidate": { |
| "key": "v1", |
| "answer": "A reply from an external application service is not established", |
| "correct": true |
| } |
| } |
| }, |
| "task_success": { |
| "gold_key": "v1", |
| "gold_answer": "The final response leaves the stated task unresolved", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "Task completion is not established by this view", |
| "correct": false |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "Task completion is not established by this view", |
| "correct": false |
| } |
| } |
| }, |
| "operator_intervention": { |
| "gold_key": "v2", |
| "gold_answer": "No operator acknowledgment or stop is established", |
| "models": { |
| "parent": { |
| "key": "v0", |
| "answer": "An operator stopped execution", |
| "correct": false |
| }, |
| "candidate": { |
| "key": "v0", |
| "answer": "An operator stopped execution", |
| "correct": false |
| } |
| } |
| } |
| } |
| } |
| ], |
| "retained_transfer_by_source": { |
| "composition_holdout": { |
| "models": { |
| "parent": { |
| "questions": 32, |
| "correct": 29, |
| "errors": 3, |
| "accuracy": 0.90625 |
| }, |
| "candidate": { |
| "questions": 32, |
| "correct": 29, |
| "errors": 3, |
| "accuracy": 0.90625 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [], |
| "retention_gate": { |
| "maximum_net_added_errors": 2, |
| "passed": true |
| } |
| }, |
| "emotion": { |
| "models": { |
| "parent": { |
| "questions": 14, |
| "correct": 8, |
| "errors": 6, |
| "accuracy": 0.5714285714285714 |
| }, |
| "candidate": { |
| "questions": 14, |
| "correct": 8, |
| "errors": 6, |
| "accuracy": 0.5714285714285714 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [], |
| "retention_gate": { |
| "maximum_net_added_errors": 2, |
| "passed": true |
| } |
| }, |
| "legacy_holdout": { |
| "models": { |
| "parent": { |
| "questions": 16, |
| "correct": 16, |
| "errors": 0, |
| "accuracy": 1.0 |
| }, |
| "candidate": { |
| "questions": 16, |
| "correct": 16, |
| "errors": 0, |
| "accuracy": 1.0 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [], |
| "retention_gate": { |
| "maximum_net_added_errors": 2, |
| "passed": true |
| } |
| }, |
| "mmlu": { |
| "models": { |
| "parent": { |
| "questions": 11, |
| "correct": 5, |
| "errors": 6, |
| "accuracy": 0.45454545454545453 |
| }, |
| "candidate": { |
| "questions": 11, |
| "correct": 5, |
| "errors": 6, |
| "accuracy": 0.45454545454545453 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [], |
| "retention_gate": { |
| "maximum_net_added_errors": 2, |
| "passed": true |
| } |
| }, |
| "otrf_sysmon_file": { |
| "models": { |
| "parent": { |
| "questions": 16, |
| "correct": 16, |
| "errors": 0, |
| "accuracy": 1.0 |
| }, |
| "candidate": { |
| "questions": 16, |
| "correct": 16, |
| "errors": 0, |
| "accuracy": 1.0 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [], |
| "retention_gate": { |
| "maximum_net_added_errors": 2, |
| "passed": true |
| } |
| }, |
| "otrf_sysmon_join": { |
| "models": { |
| "parent": { |
| "questions": 16, |
| "correct": 15, |
| "errors": 1, |
| "accuracy": 0.9375 |
| }, |
| "candidate": { |
| "questions": 16, |
| "correct": 15, |
| "errors": 1, |
| "accuracy": 0.9375 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [], |
| "retention_gate": { |
| "maximum_net_added_errors": 2, |
| "passed": true |
| } |
| }, |
| "otrf_sysmon_process": { |
| "models": { |
| "parent": { |
| "questions": 24, |
| "correct": 23, |
| "errors": 1, |
| "accuracy": 0.9583333333333334 |
| }, |
| "candidate": { |
| "questions": 24, |
| "correct": 23, |
| "errors": 1, |
| "accuracy": 0.9583333333333334 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [], |
| "retention_gate": { |
| "maximum_net_added_errors": 2, |
| "passed": true |
| } |
| }, |
| "otrf_sysmon_registry": { |
| "models": { |
| "parent": { |
| "questions": 8, |
| "correct": 8, |
| "errors": 0, |
| "accuracy": 1.0 |
| }, |
| "candidate": { |
| "questions": 8, |
| "correct": 8, |
| "errors": 0, |
| "accuracy": 1.0 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [], |
| "retention_gate": { |
| "maximum_net_added_errors": 2, |
| "passed": true |
| } |
| }, |
| "paws": { |
| "models": { |
| "parent": { |
| "questions": 8, |
| "correct": 6, |
| "errors": 2, |
| "accuracy": 0.75 |
| }, |
| "candidate": { |
| "questions": 8, |
| "correct": 6, |
| "errors": 2, |
| "accuracy": 0.75 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [], |
| "retention_gate": { |
| "maximum_net_added_errors": 2, |
| "passed": true |
| } |
| }, |
| "qnli": { |
| "models": { |
| "parent": { |
| "questions": 8, |
| "correct": 8, |
| "errors": 0, |
| "accuracy": 1.0 |
| }, |
| "candidate": { |
| "questions": 8, |
| "correct": 8, |
| "errors": 0, |
| "accuracy": 1.0 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [], |
| "retention_gate": { |
| "maximum_net_added_errors": 2, |
| "passed": true |
| } |
| }, |
| "sciq": { |
| "models": { |
| "parent": { |
| "questions": 8, |
| "correct": 8, |
| "errors": 0, |
| "accuracy": 1.0 |
| }, |
| "candidate": { |
| "questions": 8, |
| "correct": 8, |
| "errors": 0, |
| "accuracy": 1.0 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [], |
| "retention_gate": { |
| "maximum_net_added_errors": 2, |
| "passed": true |
| } |
| }, |
| "sev_swarmtraces_manual_v1": { |
| "models": { |
| "parent": { |
| "questions": 83, |
| "correct": 65, |
| "errors": 18, |
| "accuracy": 0.7831325301204819 |
| }, |
| "candidate": { |
| "questions": 83, |
| "correct": 66, |
| "errors": 17, |
| "accuracy": 0.7951807228915663 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 1, |
| "net_added_errors": -1, |
| "candidate_minus_parent_correct": 1, |
| "regression_flips": [], |
| "repair_flips": [ |
| { |
| "id": "swarm-manual/R0008229", |
| "question": "credential_binding", |
| "parent_key": "unavailable_context", |
| "candidate_key": "embedded_literal", |
| "gold_key": "embedded_literal", |
| "parent_correct": false, |
| "candidate_correct": true |
| } |
| ], |
| "all_answer_changes": [ |
| { |
| "id": "swarm-manual/R0008229", |
| "question": "credential_binding", |
| "parent_key": "unavailable_context", |
| "candidate_key": "embedded_literal", |
| "gold_key": "embedded_literal", |
| "parent_correct": false, |
| "candidate_correct": true |
| } |
| ], |
| "retention_gate": { |
| "maximum_net_added_errors": 1, |
| "passed": true |
| } |
| }, |
| "tweet_offensive": { |
| "models": { |
| "parent": { |
| "questions": 8, |
| "correct": 6, |
| "errors": 2, |
| "accuracy": 0.75 |
| }, |
| "candidate": { |
| "questions": 8, |
| "correct": 6, |
| "errors": 2, |
| "accuracy": 0.75 |
| } |
| }, |
| "added_errors": 0, |
| "repaired_errors": 0, |
| "net_added_errors": 0, |
| "candidate_minus_parent_correct": 0, |
| "regression_flips": [], |
| "repair_flips": [], |
| "all_answer_changes": [], |
| "retention_gate": { |
| "maximum_net_added_errors": 2, |
| "passed": true |
| } |
| } |
| }, |
| "policy": { |
| "answers": "argmax of canonical raw logits", |
| "dns_gate": "no parent-correct answer may become wrong; repairs do not offset regressions", |
| "retention_gate": "net correct-count loss per source; manual budget is separate", |
| "probabilities": "not reported; saved calibration receipt validated but no fit or threshold selection performed", |
| "intervals": "none; all DNS cases share one incident-inspired group" |
| }, |
| "limitations": [ |
| "Reviewed incident summaries and authored unexecuted controls are a descriptive diagnostic, not native telemetry or an untouched final test.", |
| "Eight views and 32 questions represent one incident-inspired group, not 32 independent incidents.", |
| "No actor-origin or universal behavioral-detection claim follows. No locked-test partition is read." |
| ] |
| } |
|
|
