| { | |
| "model": "openai/gpt-oss-20b", | |
| "selected_checkpoint": "proof-v5-learned-epoch-4", | |
| "primary_metric": "certificate_pass_rate", | |
| "improved_over_base": true, | |
| "rows_sha256": "3051cd3b9dfdb597a5be54873968908491d056f39ec6dd875f0bae847a20675c", | |
| "results": { | |
| "overall": { | |
| "base": { | |
| "n": 768, | |
| "certificate_pass_rate": 0.049479166666666664, | |
| "claim_accuracy": 0.5690104166666666, | |
| "mean_generated_tokens": 1375.0546875, | |
| "token_caps": 60, | |
| "failure_types": { | |
| "fact_index": 5, | |
| "not_closed": 317, | |
| "redundant_step": 32, | |
| "schema": 78, | |
| "unavailable_key": 238, | |
| "unclean_termination": 60, | |
| "valid": 38 | |
| } | |
| }, | |
| "learned": { | |
| "n": 768, | |
| "certificate_pass_rate": 0.5572916666666666, | |
| "claim_accuracy": 0.9752604166666666, | |
| "mean_generated_tokens": 492.2161458333333, | |
| "token_caps": 4, | |
| "failure_types": { | |
| "fact_index": 6, | |
| "not_closed": 36, | |
| "query_mismatch": 1, | |
| "redundant_step": 38, | |
| "unavailable_key": 255, | |
| "unclean_termination": 4, | |
| "valid": 428 | |
| } | |
| }, | |
| "paired": { | |
| "groups": 384, | |
| "adapted_minus_base": 0.5078125, | |
| "paired_group_bootstrap_95": [ | |
| 0.4609375, | |
| 0.5533854166666666 | |
| ] | |
| } | |
| }, | |
| "test_depth": { | |
| "base": { | |
| "n": 256, | |
| "certificate_pass_rate": 0.03125, | |
| "claim_accuracy": 0.5625, | |
| "mean_generated_tokens": 1391.953125, | |
| "token_caps": 18, | |
| "failure_types": { | |
| "fact_index": 3, | |
| "not_closed": 126, | |
| "redundant_step": 1, | |
| "schema": 21, | |
| "unavailable_key": 79, | |
| "unclean_termination": 18, | |
| "valid": 8 | |
| } | |
| }, | |
| "learned": { | |
| "n": 256, | |
| "certificate_pass_rate": 0.4296875, | |
| "claim_accuracy": 0.9609375, | |
| "mean_generated_tokens": 646.69140625, | |
| "token_caps": 4, | |
| "failure_types": { | |
| "fact_index": 1, | |
| "not_closed": 7, | |
| "redundant_step": 6, | |
| "unavailable_key": 128, | |
| "unclean_termination": 4, | |
| "valid": 110 | |
| } | |
| }, | |
| "paired": { | |
| "groups": 128, | |
| "adapted_minus_base": 0.3984375, | |
| "paired_group_bootstrap_95": [ | |
| 0.3203125, | |
| 0.48046875 | |
| ] | |
| } | |
| }, | |
| "test_iid": { | |
| "base": { | |
| "n": 256, | |
| "certificate_pass_rate": 0.11328125, | |
| "claim_accuracy": 0.62109375, | |
| "mean_generated_tokens": 1256.66015625, | |
| "token_caps": 18, | |
| "failure_types": { | |
| "fact_index": 1, | |
| "not_closed": 94, | |
| "redundant_step": 10, | |
| "schema": 27, | |
| "unavailable_key": 77, | |
| "unclean_termination": 18, | |
| "valid": 29 | |
| } | |
| }, | |
| "learned": { | |
| "n": 256, | |
| "certificate_pass_rate": 0.8046875, | |
| "claim_accuracy": 0.99609375, | |
| "mean_generated_tokens": 389.2890625, | |
| "token_caps": 0, | |
| "failure_types": { | |
| "not_closed": 6, | |
| "query_mismatch": 1, | |
| "redundant_step": 4, | |
| "unavailable_key": 39, | |
| "valid": 206 | |
| } | |
| }, | |
| "paired": { | |
| "groups": 128, | |
| "adapted_minus_base": 0.69140625, | |
| "paired_group_bootstrap_95": [ | |
| 0.6171875, | |
| 0.76171875 | |
| ] | |
| } | |
| }, | |
| "test_topology": { | |
| "base": { | |
| "n": 256, | |
| "certificate_pass_rate": 0.00390625, | |
| "claim_accuracy": 0.5234375, | |
| "mean_generated_tokens": 1476.55078125, | |
| "token_caps": 24, | |
| "failure_types": { | |
| "fact_index": 1, | |
| "not_closed": 97, | |
| "redundant_step": 21, | |
| "schema": 30, | |
| "unavailable_key": 82, | |
| "unclean_termination": 24, | |
| "valid": 1 | |
| } | |
| }, | |
| "learned": { | |
| "n": 256, | |
| "certificate_pass_rate": 0.4375, | |
| "claim_accuracy": 0.96875, | |
| "mean_generated_tokens": 440.66796875, | |
| "token_caps": 0, | |
| "failure_types": { | |
| "fact_index": 5, | |
| "not_closed": 23, | |
| "redundant_step": 28, | |
| "unavailable_key": 88, | |
| "valid": 112 | |
| } | |
| }, | |
| "paired": { | |
| "groups": 128, | |
| "adapted_minus_base": 0.43359375, | |
| "paired_group_bootstrap_95": [ | |
| 0.35546875, | |
| 0.51171875 | |
| ] | |
| } | |
| } | |
| } | |
| } | |