{ "easy": { "accuracy": 1.0, "brier_mean": 0.011964783954900882, "calibration_n": 48, "complete": true, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "dataset_hash": "3cbd0e951fa8059741955ef2bd3273b0bb60aecae8f4a68b89872f55f7376b3d", "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5003303289413452, "n": 1 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8724372982978821, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9944215284453498, "n": 45 } ], "ece": 0.020954714467128106, "n": 48 }, "latency": { "n": 48, "p50_s": 0.06915471702814102, "p95_s": 0.10234953109174966 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "ledger_charged_usd": null, "macro_accuracy": 1.0, "model_identities": [ "pcdm:native:runs/tm1b" ], "n_attempted": 48, "n_correct": 48, "n_planned": 48, "n_renormalized": 0, "n_scorable": 48, "n_valid": 48, "operational_success": 1.0, "ordinal_mae": null, "paraphrase_consistency": { "agree": 0, "agreement": null, "both_correct_rate_all_pairs": null, "both_valid": 0, "pairs": 0 }, "per_family": { "extraction": { "accuracy": 1.0, "brier_mean": 0.003787310148891915, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8558686375617981, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9874209653247487, "n": 11 } ], "ece": 0.023541728655497224, "n": 12 }, "latency": { "n": 12, "p50_s": 0.06931246258318424, "p95_s": 0.08583159018307922 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "fact": { "accuracy": 1.0, "brier_mean": 0.044071809342586504, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5003303289413452, "n": 1 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8890059590339661, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.988783085346222, "n": 10 } ], "ece": 0.06023607154687243, "n": 12 }, "latency": { "n": 12, "p50_s": 0.07128027640283108, "p95_s": 0.1037625202909112 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 1.0, "brier_mean": 1.4921552066029873e-08, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9999667853116989, "n": 12 } ], "ece": 3.3214688301086426e-05, "n": 12 }, "latency": { "n": 12, "p50_s": 0.0686907097697258, "p95_s": 0.1342240288853645 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "tool_selection": { "accuracy": 1.0, "brier_mean": 1.4065730446045023e-09, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9999921570221583, "n": 12 } ], "ece": 7.842977841732157e-06, "n": 12 }, "latency": { "n": 12, "p50_s": 0.06903873011469841, "p95_s": 0.07324958574026823 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "price_per_1000_decisions_usd": null, "probability_sources": [ "native" ], "schema_validity": 1.0, "schema_validity_strict": 1.0, "splits": { "public": { "accuracy": 1.0, "brier_mean": 0.011964783954900882, "calibration_n": 48, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5003303289413452, "n": 1 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8724372982978821, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9944215284453498, "n": 45 } ], "ece": 0.020954714467128106, "n": 48 }, "latency": { "n": 48, "p50_s": 0.06915471702814102, "p95_s": 0.10234953109174966 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 48, "n_correct": 48, "n_planned": 48, "n_renormalized": 0, "n_scorable": 48, "n_valid": 48, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "version": "jevbench-v1" }, "hard": { "accuracy": 0.42342342342342343, "brier_mean": 0.7731682475813886, "calibration_n": 111, "complete": true, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "dataset_hash": "e816e106f562dec75a8da12e69b75e957a4bbd9f6300f1145fc71837327a3c61", "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.3, "lo": 0.2, "mean_confidence": 0.27849190682172775, "n": 4 }, { "accuracy": 0.125, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.37296687439084053, "n": 8 }, { "accuracy": 0.4166666666666667, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4450460250178973, "n": 12 }, { "accuracy": 0.35, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5447314709424973, "n": 20 }, { "accuracy": 0.5, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6454204533781324, "n": 14 }, { "accuracy": 0.3333333333333333, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7522401312987009, "n": 6 }, { "accuracy": 0.25, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8441648408770561, "n": 8 }, { "accuracy": 0.5897435897435898, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.965268883949671, "n": 39 } ], "ece": 0.2818109610596219, "n": 111 }, "latency": { "n": 111, "p50_s": 0.11237071454524994, "p95_s": 0.14899961836636066 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "ledger_charged_usd": null, "macro_accuracy": 0.4814144429701705, "model_identities": [ "pcdm:native:runs/tm1b" ], "n_attempted": 111, "n_correct": 47, "n_planned": 111, "n_renormalized": 0, "n_scorable": 111, "n_valid": 111, "operational_success": 1.0, "ordinal_mae": 0.7110752028723558, "paraphrase_consistency": { "agree": 0, "agreement": null, "both_correct_rate_all_pairs": null, "both_valid": 0, "pairs": 0 }, "per_family": { "adversarial": { "accuracy": 0.6666666666666666, "brier_mean": 0.5843859976304459, "calibration_n": 6, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6800556182861328, "n": 1 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8503317534923553, "n": 2 }, { "accuracy": 0.6666666666666666, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9747096498807272, "n": 3 } ], "ece": 0.3241228063901266, "n": 6 }, "latency": { "n": 6, "p50_s": 0.10217797197401524, "p95_s": 0.10633919388055801 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 6, "n_correct": 4, "n_planned": 6, "n_renormalized": 0, "n_scorable": 6, "n_valid": 6, "operational_success": 1.0, "ordinal_mae": 0.0494021475315094, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ambiguous": { "accuracy": 0.5714285714285714, "brier_mean": 0.6891428318952786, "calibration_n": 7, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5438312590122223, "n": 2 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7460008561611176, "n": 2 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8554508686065674, "n": 1 }, { "accuracy": 0.5, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9503115713596344, "n": 2 } ], "ece": 0.23211950063705444, "n": 7 }, "latency": { "n": 7, "p50_s": 0.108931515365839, "p95_s": 0.11300073452293874 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 7, "n_correct": 4, "n_planned": 7, "n_renormalized": 0, "n_scorable": 7, "n_valid": 7, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "judge_hard": { "accuracy": 0.5294117647058824, "brier_mean": 0.8882525997132734, "calibration_n": 17, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7447604537010193, "n": 1 }, { "accuracy": 0.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8594815731048584, "n": 1 }, { "accuracy": 0.5333333333333333, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9754487832387289, "n": 15 } ], "ece": 0.4556736981167513, "n": 17 }, "latency": { "n": 17, "p50_s": 0.1022290475666523, "p95_s": 0.11644351556897163 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 17, "n_correct": 9, "n_planned": 17, "n_renormalized": 0, "n_scorable": 17, "n_valid": 17, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "long_policy": { "accuracy": 0.21052631578947367, "brier_mean": 0.956868090190627, "calibration_n": 19, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.3, "lo": 0.2, "mean_confidence": 0.2781612128019333, "n": 2 }, { "accuracy": 0.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3873806794484456, "n": 3 }, { "accuracy": 0.5, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.43897129595279694, "n": 4 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5172954201698303, "n": 1 }, { "accuracy": 0.6666666666666666, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6477566162745158, "n": 3 }, { "accuracy": 0.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7533348798751831, "n": 2 }, { "accuracy": 0.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8167205850283304, "n": 3 }, { "accuracy": 0.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9532382488250732, "n": 1 } ], "ece": 0.3919302429023541, "n": 19 }, "latency": { "n": 19, "p50_s": 0.14529041200876236, "p95_s": 0.16292529068887227 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 19, "n_correct": 4, "n_planned": 19, "n_renormalized": 0, "n_scorable": 19, "n_valid": 19, "operational_success": 1.0, "ordinal_mae": 0.5590988993644714, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "multi_hop": { "accuracy": 0.4444444444444444, "brier_mean": 0.6872533730651412, "calibration_n": 18, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.33783966302871704, "n": 1 }, { "accuracy": 0.4, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.44226173162460325, "n": 5 }, { "accuracy": 0.5, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5598814710974693, "n": 8 }, { "accuracy": 0.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6466758847236633, "n": 2 }, { "accuracy": 0.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7700088620185852, "n": 1 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9851491451263428, "n": 1 } ], "ece": 0.19059623612297905, "n": 18 }, "latency": { "n": 18, "p50_s": 0.128933347761631, "p95_s": 0.13938846364617347 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 18, "n_correct": 8, "n_planned": 18, "n_renormalized": 0, "n_scorable": 18, "n_valid": 18, "operational_success": 1.0, "ordinal_mae": 0.4032743573188782, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "probability": { "accuracy": 0.4, "brier_mean": 0.7593166581141717, "calibration_n": 10, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4478049576282501, "n": 1 }, { "accuracy": 0.5, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5372944176197052, "n": 4 }, { "accuracy": 0.5, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6290208697319031, "n": 2 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.3333333333333333, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9370233019193014, "n": 3 } ], "ece": 0.2666094273328781, "n": 10 }, "latency": { "n": 10, "p50_s": 0.11380846984684467, "p95_s": 0.12198328394442795 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 10, "n_correct": 4, "n_planned": 10, "n_renormalized": 0, "n_scorable": 10, "n_valid": 10, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing_hard": { "accuracy": 1.0, "brier_mean": 0.0021777005726523737, "calibration_n": 5, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9799626469612122, "n": 5 } ], "ece": 0.020037353038787842, "n": 5 }, "latency": { "n": 5, "p50_s": 0.10272485390305519, "p95_s": 0.10452966764569283 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 5, "n_correct": 5, "n_planned": 5, "n_renormalized": 0, "n_scorable": 5, "n_valid": 5, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "temporal_numeric": { "accuracy": 0.2, "brier_mean": 0.9778733222516895, "calibration_n": 15, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.3, "lo": 0.2, "mean_confidence": 0.2788226008415222, "n": 2 }, { "accuracy": 0.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3616597553094228, "n": 3 }, { "accuracy": 0.5, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4627767503261566, "n": 2 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5329880118370056, "n": 3 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6510051488876343, "n": 2 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.887561023235321, "n": 1 }, { "accuracy": 0.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.963701069355011, "n": 2 } ], "ece": 0.45526585777600603, "n": 15 }, "latency": { "n": 15, "p50_s": 0.11358731612563133, "p95_s": 0.1201222252100706 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 15, "n_correct": 3, "n_planned": 15, "n_renormalized": 0, "n_scorable": 15, "n_valid": 15, "operational_success": 1.0, "ordinal_mae": 1.3456641659140587, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "tradeoff": { "accuracy": 0.16666666666666666, "brier_mean": 1.1503974298690245, "calibration_n": 6, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5204590559005737, "n": 1 }, { "accuracy": 0.3333333333333333, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6232326428095499, "n": 3 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9465669393539429, "n": 2 } ], "ece": 0.5472151438395182, "n": 6 }, "latency": { "n": 6, "p50_s": 0.10983631946146488, "p95_s": 0.11511344648897648 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 6, "n_correct": 1, "n_planned": 6, "n_renormalized": 0, "n_scorable": 6, "n_valid": 6, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "trap": { "accuracy": 0.625, "brier_mean": 0.3331839439462419, "calibration_n": 8, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.39877402782440186, "n": 1 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5420189499855042, "n": 1 }, { "accuracy": 0.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6894591450691223, "n": 1 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.9, "lo": 0.8, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9438392162322998, "n": 5 } ], "ece": 0.23888200521469116, "n": 8 }, "latency": { "n": 8, "p50_s": 0.10395996645092964, "p95_s": 0.10869606416672467 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 8, "n_correct": 5, "n_planned": 8, "n_renormalized": 0, "n_scorable": 8, "n_valid": 8, "operational_success": 1.0, "ordinal_mae": 1.3499127477407455, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "price_per_1000_decisions_usd": null, "probability_sources": [ "native" ], "schema_validity": 1.0, "schema_validity_strict": 1.0, "splits": { "public": { "accuracy": 0.42342342342342343, "brier_mean": 0.7731682475813886, "calibration_n": 111, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.3, "lo": 0.2, "mean_confidence": 0.27849190682172775, "n": 4 }, { "accuracy": 0.125, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.37296687439084053, "n": 8 }, { "accuracy": 0.4166666666666667, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4450460250178973, "n": 12 }, { "accuracy": 0.35, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5447314709424973, "n": 20 }, { "accuracy": 0.5, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6454204533781324, "n": 14 }, { "accuracy": 0.3333333333333333, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7522401312987009, "n": 6 }, { "accuracy": 0.25, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8441648408770561, "n": 8 }, { "accuracy": 0.5897435897435898, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.965268883949671, "n": 39 } ], "ece": 0.2818109610596219, "n": 111 }, "latency": { "n": 111, "p50_s": 0.11237071454524994, "p95_s": 0.14899961836636066 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 111, "n_correct": 47, "n_planned": 111, "n_renormalized": 0, "n_scorable": 111, "n_valid": 111, "operational_success": 1.0, "ordinal_mae": 0.7110752028723558, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "version": "jevbench-v1" }, "original": { "accuracy": 0.8055555555555556, "brier_mean": 0.29652247410236326, "calibration_n": 72, "complete": true, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "dataset_hash": "914276022fa5f0ea70e4231f2af98e68c67b274b61bea8c440b33890401cd2f9", "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3444499373435974, "n": 2 }, { "accuracy": 1.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4325794279575348, "n": 1 }, { "accuracy": 0.42857142857142855, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5371253064700535, "n": 7 }, { "accuracy": 0.6666666666666666, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6374844577577379, "n": 9 }, { "accuracy": 0.8571428571428571, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7471651860645839, "n": 7 }, { "accuracy": 0.9, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8518721401691437, "n": 10 }, { "accuracy": 0.8888888888888888, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9730205883582433, "n": 36 } ], "ece": 0.08584584999415612, "n": 72 }, "latency": { "n": 72, "p50_s": 0.07467062026262283, "p95_s": 0.1292133398354054 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "ledger_charged_usd": null, "macro_accuracy": 0.8055555555555555, "model_identities": [ "pcdm:native:runs/tm1b" ], "n_attempted": 72, "n_correct": 58, "n_planned": 72, "n_renormalized": 0, "n_scorable": 72, "n_valid": 72, "operational_success": 1.0, "ordinal_mae": 0.39357847470091656, "paraphrase_consistency": { "agree": 30, "agreement": 0.8333333333333334, "both_correct_rate_all_pairs": 0.7222222222222222, "both_valid": 36, "pairs": 36 }, "per_family": { "adequacy": { "accuracy": 0.6666666666666666, "brier_mean": 0.40836962254148307, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5817238092422485, "n": 1 }, { "accuracy": 0.5, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6328441649675369, "n": 4 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8540955781936646, "n": 1 }, { "accuracy": 0.8333333333333334, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.972666323184967, "n": 6 } ], "ece": 0.1745835691690445, "n": 12 }, "latency": { "n": 12, "p50_s": 0.07050643116235733, "p95_s": 0.08810084797441958 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 8, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "extraction": { "accuracy": 1.0, "brier_mean": 0.02007323946682944, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7408376932144165, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8441984951496124, "n": 2 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.968041201432546, "n": 9 } ], "ece": 0.07153287529945371, "n": 12 }, "latency": { "n": 12, "p50_s": 0.05768968164920807, "p95_s": 0.08274002466350794 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 12, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "intent": { "accuracy": 0.9166666666666666, "brier_mean": 0.22698352684655446, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5113146007061005, "n": 2 }, { "accuracy": null, "hi": 0.7, "lo": 0.6, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7275521159172058, "n": 1 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8921765089035034, "n": 1 }, { "accuracy": 0.875, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9776975587010384, "n": 8 } ], "ece": 0.1816018869479497, "n": 12 }, "latency": { "n": 12, "p50_s": 0.11113478429615498, "p95_s": 0.19621948227286334 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 11, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "ordinal": { "accuracy": 0.6666666666666666, "brier_mean": 0.387267867931643, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3444499373435974, "n": 2 }, { "accuracy": 1.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4325794279575348, "n": 1 }, { "accuracy": 0.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5386167963345846, "n": 3 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6215285360813141, "n": 2 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.839689165353775, "n": 4 }, { "accuracy": null, "hi": 1.0, "lo": 0.9, "mean_confidence": null, "n": 0 } ], "ece": 0.32437977939844137, "n": 12 }, "latency": { "n": 12, "p50_s": 0.07719611376523972, "p95_s": 0.11410039383918047 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 8, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": 0.39357847470091656, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "policy": { "accuracy": 0.75, "brier_mean": 0.3966635065443795, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.6, "lo": 0.5, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6735906898975372, "n": 2 }, { "accuracy": null, "hi": 0.8, "lo": 0.7, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.859863817691803, "n": 1 }, { "accuracy": 0.7777777777777778, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9743029342757331, "n": 9 } ], "ece": 0.18800366421540576, "n": 12 }, "latency": { "n": 12, "p50_s": 0.10365468636155128, "p95_s": 0.15637512318789956 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 9, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 }, "routing": { "accuracy": 0.8333333333333334, "brier_mean": 0.3397770812832901, "calibration_n": 12, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.4, "lo": 0.3, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.5, "lo": 0.4, "mean_confidence": null, "n": 0 }, { "accuracy": 1.0, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5396737456321716, "n": 1 }, { "accuracy": 1.0, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6157450079917908, "n": 1 }, { "accuracy": 0.8, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7523532986640931, "n": 5 }, { "accuracy": 0.0, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8654318451881409, "n": 1 }, { "accuracy": 1.0, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9725163877010345, "n": 4 } ], "ece": 0.17151508728663126, "n": 12 }, "latency": { "n": 12, "p50_s": 0.053897660225629807, "p95_s": 0.08854644279927014 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 12, "n_correct": 10, "n_planned": 12, "n_renormalized": 0, "n_scorable": 12, "n_valid": 12, "operational_success": 1.0, "ordinal_mae": null, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "price_per_1000_decisions_usd": null, "probability_sources": [ "native" ], "schema_validity": 1.0, "schema_validity_strict": 1.0, "splits": { "public": { "accuracy": 0.8055555555555556, "brier_mean": 0.29652247410236326, "calibration_n": 72, "cost_basis": [ "local_gpu_no_provider_tariff" ], "coverage": 1.0, "ece": { "bins": [ { "accuracy": null, "hi": 0.1, "lo": 0.0, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.2, "lo": 0.1, "mean_confidence": null, "n": 0 }, { "accuracy": null, "hi": 0.3, "lo": 0.2, "mean_confidence": null, "n": 0 }, { "accuracy": 0.5, "hi": 0.4, "lo": 0.3, "mean_confidence": 0.3444499373435974, "n": 2 }, { "accuracy": 1.0, "hi": 0.5, "lo": 0.4, "mean_confidence": 0.4325794279575348, "n": 1 }, { "accuracy": 0.42857142857142855, "hi": 0.6, "lo": 0.5, "mean_confidence": 0.5371253064700535, "n": 7 }, { "accuracy": 0.6666666666666666, "hi": 0.7, "lo": 0.6, "mean_confidence": 0.6374844577577379, "n": 9 }, { "accuracy": 0.8571428571428571, "hi": 0.8, "lo": 0.7, "mean_confidence": 0.7471651860645839, "n": 7 }, { "accuracy": 0.9, "hi": 0.9, "lo": 0.8, "mean_confidence": 0.8518721401691437, "n": 10 }, { "accuracy": 0.8888888888888888, "hi": 1.0, "lo": 0.9, "mean_confidence": 0.9730205883582433, "n": 36 } ], "ece": 0.08584584999415612, "n": 72 }, "latency": { "n": 72, "p50_s": 0.07467062026262283, "p95_s": 0.1292133398354054 }, "latency_failures": { "n": 0, "p50_s": null, "p95_s": null }, "n_attempted": 72, "n_correct": 58, "n_planned": 72, "n_renormalized": 0, "n_scorable": 72, "n_valid": 72, "operational_success": 1.0, "ordinal_mae": 0.39357847470091656, "price_per_1000_decisions_usd": null, "schema_validity": 1.0, "schema_validity_strict": 1.0 } }, "version": "jevbench-v1" } }