| { | |
| "scope": "Synthetic current-prefix acceptable-action accuracy; not end-to-end agent utility", | |
| "human_gold": false, | |
| "validation": { | |
| "lexical128": { | |
| "n": 64, | |
| "correct": 43, | |
| "acceptable_accuracy": 0.671875, | |
| "false_abstain": 19, | |
| "false_abstain_rate": 0.8260869565217391, | |
| "missed_abstain": 1, | |
| "missed_abstain_rate": 0.024390243902439025 | |
| }, | |
| "lexical32": { | |
| "n": 64, | |
| "correct": 41, | |
| "acceptable_accuracy": 0.640625, | |
| "false_abstain": 20, | |
| "false_abstain_rate": 0.8695652173913043, | |
| "missed_abstain": 3, | |
| "missed_abstain_rate": 0.07317073170731707 | |
| }, | |
| "lexical64": { | |
| "n": 64, | |
| "correct": 37, | |
| "acceptable_accuracy": 0.578125, | |
| "false_abstain": 20, | |
| "false_abstain_rate": 0.8695652173913043, | |
| "missed_abstain": 7, | |
| "missed_abstain_rate": 0.17073170731707318 | |
| }, | |
| "semantic128": { | |
| "n": 64, | |
| "correct": 55, | |
| "acceptable_accuracy": 0.859375, | |
| "false_abstain": 6, | |
| "false_abstain_rate": 0.2608695652173913, | |
| "missed_abstain": 1, | |
| "missed_abstain_rate": 0.024390243902439025 | |
| } | |
| }, | |
| "fresh_test_paired_all64": { | |
| "baseline_correct": 54, | |
| "candidate_correct": 44, | |
| "baseline_accuracy": 0.84375, | |
| "candidate_accuracy": 0.6875, | |
| "accuracy_difference": -0.15625, | |
| "paired_bootstrap95": [ | |
| -0.25, | |
| -0.078125 | |
| ], | |
| "bootstrap_replicates": 10000, | |
| "bootstrap_seed": 101, | |
| "gained_correct_worlds": 0, | |
| "lost_correct_worlds": 10, | |
| "select_denominator": 20, | |
| "baseline_false_abstain": 5, | |
| "candidate_false_abstain": 5, | |
| "baseline_false_abstain_rate": 0.25, | |
| "candidate_false_abstain_rate": 0.25, | |
| "abstain_denominator": 44, | |
| "baseline_missed_abstain": 3, | |
| "candidate_missed_abstain": 10, | |
| "baseline_missed_abstain_rate": 0.06818181818181818, | |
| "candidate_missed_abstain_rate": 0.22727272727272727, | |
| "baseline_conditional_tool_errors": 4, | |
| "candidate_conditional_tool_errors": 5, | |
| "baseline_mean_joint_target_set_nll": 0.9775434669867754, | |
| "candidate_mean_joint_target_set_nll": 3.0828047417095714, | |
| "additional_false_abstain_worlds": [ | |
| "gec1_dw010" | |
| ] | |
| }, | |
| "fresh_test_gate_checks": { | |
| "accuracy_at_least_80_percent": false, | |
| "false_abstain_at_most_10_percent": false, | |
| "missed_abstain_at_most_10_percent": false, | |
| "both_classes_nonzero": true, | |
| "extra_correct_at_least_4": false, | |
| "no_added_false_abstain_worlds": false, | |
| "candidate_gate_passed": false, | |
| "baseline_also_fails_false_abstain_gate": true | |
| }, | |
| "test_independence": { | |
| "archive_committed_before_fresh_labels": true, | |
| "labels_committed_before_predictions": true, | |
| "first_use_of_reserved_fresh64": true, | |
| "raw_author": "Existing gpt-6-astra, unchanged role", | |
| "source_adapter": "gpt-6.1-sol high, source only", | |
| "expert": "gpt-6-astra medium, fresh context, individual current-prefix manual labels", | |
| "expert_prediction_access": false, | |
| "human_gold": false, | |
| "PASS": 64, | |
| "HOLD": 0, | |
| "excluded_worlds": 0, | |
| "semantic_relabels": 0, | |
| "threshold_searches": 0, | |
| "replays": 0 | |
| }, | |
| "published_weights": "semantic128 (previous reference; not rejected candidate384)", | |
| "published_weights_sha256": "645a82c68576b207723a21ee413edbc9e69980a41520cdcb475bf28455c94887", | |
| "training": { | |
| "HOLD": 2, | |
| "PASS": 126, | |
| "acceptable_target_coverage": { | |
| "list_directory": 7, | |
| "read_file": 21, | |
| "search_content": 30, | |
| "search_files": 13 | |
| }, | |
| "archive_sha256": "645a82c68576b207723a21ee413edbc9e69980a41520cdcb475bf28455c94887", | |
| "arm": "semantic128", | |
| "converged": false, | |
| "optimizer_receipt_sha256": "9583dd8b0f65c4a388b4e30132d2eaba820e14b2bb9e6ee04e3b02880cb5d5a8", | |
| "optimizer_stop": "evaluation_budget_exhausted", | |
| "optimizer_trace_sha256": "ea2462e18f60755348689bc450f3dba96c5768448f1a7a913cedc31a5f93e37c", | |
| "seconds": 559.4276709999995, | |
| "target_abstain": 91, | |
| "target_select": 37, | |
| "train_count": 128, | |
| "train_predictions_sha256": "08bbcd8fa0728e48bcc50825049f42b1e770716a8850e3b8d0d73290869da518", | |
| "training_fingerprint": "33a7f43795a6ad43bbfdb9cf9a2f6602dcb14be3c21c95fee189514ac292c4c8" | |
| }, | |
| "probabilities": "uncalibrated_softmax; no threshold search", | |
| "test_policy": "Historical test is public and closed. Do not use it for training, selection, threshold tuning or relabeling and then report independent test results on it.", | |
| "new_quality_evaluation_or_training_for_release": false | |
| } | |