| { |
| "easy": { |
| "accuracy": 1.0, |
| "brier_mean": 0.011964783954900882, |
| "calibration_n": 48, |
| "complete": true, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "dataset_hash": "3cbd0e951fa8059741955ef2bd3273b0bb60aecae8f4a68b89872f55f7376b3d", |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5003303289413452, |
| "n": 1 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8724372982978821, |
| "n": 2 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9944215284453498, |
| "n": 45 |
| } |
| ], |
| "ece": 0.020954714467128106, |
| "n": 48 |
| }, |
| "latency": { |
| "n": 48, |
| "p50_s": 0.06915471702814102, |
| "p95_s": 0.10234953109174966 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "ledger_charged_usd": null, |
| "macro_accuracy": 1.0, |
| "model_identities": [ |
| "pcdm:native:runs/tm1b" |
| ], |
| "n_attempted": 48, |
| "n_correct": 48, |
| "n_planned": 48, |
| "n_renormalized": 0, |
| "n_scorable": 48, |
| "n_valid": 48, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "paraphrase_consistency": { |
| "agree": 0, |
| "agreement": null, |
| "both_correct_rate_all_pairs": null, |
| "both_valid": 0, |
| "pairs": 0 |
| }, |
| "per_family": { |
| "extraction": { |
| "accuracy": 1.0, |
| "brier_mean": 0.003787310148891915, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8558686375617981, |
| "n": 1 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9874209653247487, |
| "n": 11 |
| } |
| ], |
| "ece": 0.023541728655497224, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.06931246258318424, |
| "p95_s": 0.08583159018307922 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 12, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "fact": { |
| "accuracy": 1.0, |
| "brier_mean": 0.044071809342586504, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5003303289413452, |
| "n": 1 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8890059590339661, |
| "n": 1 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.988783085346222, |
| "n": 10 |
| } |
| ], |
| "ece": 0.06023607154687243, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.07128027640283108, |
| "p95_s": 0.1037625202909112 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 12, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "intent": { |
| "accuracy": 1.0, |
| "brier_mean": 1.4921552066029873e-08, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9999667853116989, |
| "n": 12 |
| } |
| ], |
| "ece": 3.3214688301086426e-05, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.0686907097697258, |
| "p95_s": 0.1342240288853645 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 12, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "tool_selection": { |
| "accuracy": 1.0, |
| "brier_mean": 1.4065730446045023e-09, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9999921570221583, |
| "n": 12 |
| } |
| ], |
| "ece": 7.842977841732157e-06, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.06903873011469841, |
| "p95_s": 0.07324958574026823 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 12, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| } |
| }, |
| "price_per_1000_decisions_usd": null, |
| "probability_sources": [ |
| "native" |
| ], |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0, |
| "splits": { |
| "public": { |
| "accuracy": 1.0, |
| "brier_mean": 0.011964783954900882, |
| "calibration_n": 48, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5003303289413452, |
| "n": 1 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8724372982978821, |
| "n": 2 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9944215284453498, |
| "n": 45 |
| } |
| ], |
| "ece": 0.020954714467128106, |
| "n": 48 |
| }, |
| "latency": { |
| "n": 48, |
| "p50_s": 0.06915471702814102, |
| "p95_s": 0.10234953109174966 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 48, |
| "n_correct": 48, |
| "n_planned": 48, |
| "n_renormalized": 0, |
| "n_scorable": 48, |
| "n_valid": 48, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| } |
| }, |
| "version": "jevbench-v1" |
| }, |
| "hard": { |
| "accuracy": 0.42342342342342343, |
| "brier_mean": 0.7731682475813886, |
| "calibration_n": 111, |
| "complete": true, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "dataset_hash": "e816e106f562dec75a8da12e69b75e957a4bbd9f6300f1145fc71837327a3c61", |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": 0.27849190682172775, |
| "n": 4 |
| }, |
| { |
| "accuracy": 0.125, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.37296687439084053, |
| "n": 8 |
| }, |
| { |
| "accuracy": 0.4166666666666667, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.4450460250178973, |
| "n": 12 |
| }, |
| { |
| "accuracy": 0.35, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5447314709424973, |
| "n": 20 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6454204533781324, |
| "n": 14 |
| }, |
| { |
| "accuracy": 0.3333333333333333, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7522401312987009, |
| "n": 6 |
| }, |
| { |
| "accuracy": 0.25, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8441648408770561, |
| "n": 8 |
| }, |
| { |
| "accuracy": 0.5897435897435898, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.965268883949671, |
| "n": 39 |
| } |
| ], |
| "ece": 0.2818109610596219, |
| "n": 111 |
| }, |
| "latency": { |
| "n": 111, |
| "p50_s": 0.11237071454524994, |
| "p95_s": 0.14899961836636066 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "ledger_charged_usd": null, |
| "macro_accuracy": 0.4814144429701705, |
| "model_identities": [ |
| "pcdm:native:runs/tm1b" |
| ], |
| "n_attempted": 111, |
| "n_correct": 47, |
| "n_planned": 111, |
| "n_renormalized": 0, |
| "n_scorable": 111, |
| "n_valid": 111, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.7110752028723558, |
| "paraphrase_consistency": { |
| "agree": 0, |
| "agreement": null, |
| "both_correct_rate_all_pairs": null, |
| "both_valid": 0, |
| "pairs": 0 |
| }, |
| "per_family": { |
| "adversarial": { |
| "accuracy": 0.6666666666666666, |
| "brier_mean": 0.5843859976304459, |
| "calibration_n": 6, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6800556182861328, |
| "n": 1 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8503317534923553, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.6666666666666666, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9747096498807272, |
| "n": 3 |
| } |
| ], |
| "ece": 0.3241228063901266, |
| "n": 6 |
| }, |
| "latency": { |
| "n": 6, |
| "p50_s": 0.10217797197401524, |
| "p95_s": 0.10633919388055801 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 6, |
| "n_correct": 4, |
| "n_planned": 6, |
| "n_renormalized": 0, |
| "n_scorable": 6, |
| "n_valid": 6, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.0494021475315094, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "ambiguous": { |
| "accuracy": 0.5714285714285714, |
| "brier_mean": 0.6891428318952786, |
| "calibration_n": 7, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5438312590122223, |
| "n": 2 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7460008561611176, |
| "n": 2 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8554508686065674, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9503115713596344, |
| "n": 2 |
| } |
| ], |
| "ece": 0.23211950063705444, |
| "n": 7 |
| }, |
| "latency": { |
| "n": 7, |
| "p50_s": 0.108931515365839, |
| "p95_s": 0.11300073452293874 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 7, |
| "n_correct": 4, |
| "n_planned": 7, |
| "n_renormalized": 0, |
| "n_scorable": 7, |
| "n_valid": 7, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "judge_hard": { |
| "accuracy": 0.5294117647058824, |
| "brier_mean": 0.8882525997132734, |
| "calibration_n": 17, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7447604537010193, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8594815731048584, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.5333333333333333, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9754487832387289, |
| "n": 15 |
| } |
| ], |
| "ece": 0.4556736981167513, |
| "n": 17 |
| }, |
| "latency": { |
| "n": 17, |
| "p50_s": 0.1022290475666523, |
| "p95_s": 0.11644351556897163 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 17, |
| "n_correct": 9, |
| "n_planned": 17, |
| "n_renormalized": 0, |
| "n_scorable": 17, |
| "n_valid": 17, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "long_policy": { |
| "accuracy": 0.21052631578947367, |
| "brier_mean": 0.956868090190627, |
| "calibration_n": 19, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": 0.2781612128019333, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.3873806794484456, |
| "n": 3 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.43897129595279694, |
| "n": 4 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5172954201698303, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.6666666666666666, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6477566162745158, |
| "n": 3 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7533348798751831, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8167205850283304, |
| "n": 3 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9532382488250732, |
| "n": 1 |
| } |
| ], |
| "ece": 0.3919302429023541, |
| "n": 19 |
| }, |
| "latency": { |
| "n": 19, |
| "p50_s": 0.14529041200876236, |
| "p95_s": 0.16292529068887227 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 19, |
| "n_correct": 4, |
| "n_planned": 19, |
| "n_renormalized": 0, |
| "n_scorable": 19, |
| "n_valid": 19, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.5590988993644714, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "multi_hop": { |
| "accuracy": 0.4444444444444444, |
| "brier_mean": 0.6872533730651412, |
| "calibration_n": 18, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.33783966302871704, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.4, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.44226173162460325, |
| "n": 5 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5598814710974693, |
| "n": 8 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6466758847236633, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7700088620185852, |
| "n": 1 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9851491451263428, |
| "n": 1 |
| } |
| ], |
| "ece": 0.19059623612297905, |
| "n": 18 |
| }, |
| "latency": { |
| "n": 18, |
| "p50_s": 0.128933347761631, |
| "p95_s": 0.13938846364617347 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 18, |
| "n_correct": 8, |
| "n_planned": 18, |
| "n_renormalized": 0, |
| "n_scorable": 18, |
| "n_valid": 18, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.4032743573188782, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "probability": { |
| "accuracy": 0.4, |
| "brier_mean": 0.7593166581141717, |
| "calibration_n": 10, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.4478049576282501, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5372944176197052, |
| "n": 4 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6290208697319031, |
| "n": 2 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.3333333333333333, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9370233019193014, |
| "n": 3 |
| } |
| ], |
| "ece": 0.2666094273328781, |
| "n": 10 |
| }, |
| "latency": { |
| "n": 10, |
| "p50_s": 0.11380846984684467, |
| "p95_s": 0.12198328394442795 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 10, |
| "n_correct": 4, |
| "n_planned": 10, |
| "n_renormalized": 0, |
| "n_scorable": 10, |
| "n_valid": 10, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "routing_hard": { |
| "accuracy": 1.0, |
| "brier_mean": 0.0021777005726523737, |
| "calibration_n": 5, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9799626469612122, |
| "n": 5 |
| } |
| ], |
| "ece": 0.020037353038787842, |
| "n": 5 |
| }, |
| "latency": { |
| "n": 5, |
| "p50_s": 0.10272485390305519, |
| "p95_s": 0.10452966764569283 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 5, |
| "n_correct": 5, |
| "n_planned": 5, |
| "n_renormalized": 0, |
| "n_scorable": 5, |
| "n_valid": 5, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "temporal_numeric": { |
| "accuracy": 0.2, |
| "brier_mean": 0.9778733222516895, |
| "calibration_n": 15, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": 0.2788226008415222, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.3616597553094228, |
| "n": 3 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.4627767503261566, |
| "n": 2 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5329880118370056, |
| "n": 3 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6510051488876343, |
| "n": 2 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.887561023235321, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.963701069355011, |
| "n": 2 |
| } |
| ], |
| "ece": 0.45526585777600603, |
| "n": 15 |
| }, |
| "latency": { |
| "n": 15, |
| "p50_s": 0.11358731612563133, |
| "p95_s": 0.1201222252100706 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 15, |
| "n_correct": 3, |
| "n_planned": 15, |
| "n_renormalized": 0, |
| "n_scorable": 15, |
| "n_valid": 15, |
| "operational_success": 1.0, |
| "ordinal_mae": 1.3456641659140587, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "tradeoff": { |
| "accuracy": 0.16666666666666666, |
| "brier_mean": 1.1503974298690245, |
| "calibration_n": 6, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5204590559005737, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.3333333333333333, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6232326428095499, |
| "n": 3 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9465669393539429, |
| "n": 2 |
| } |
| ], |
| "ece": 0.5472151438395182, |
| "n": 6 |
| }, |
| "latency": { |
| "n": 6, |
| "p50_s": 0.10983631946146488, |
| "p95_s": 0.11511344648897648 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 6, |
| "n_correct": 1, |
| "n_planned": 6, |
| "n_renormalized": 0, |
| "n_scorable": 6, |
| "n_valid": 6, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "trap": { |
| "accuracy": 0.625, |
| "brier_mean": 0.3331839439462419, |
| "calibration_n": 8, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.39877402782440186, |
| "n": 1 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5420189499855042, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6894591450691223, |
| "n": 1 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9438392162322998, |
| "n": 5 |
| } |
| ], |
| "ece": 0.23888200521469116, |
| "n": 8 |
| }, |
| "latency": { |
| "n": 8, |
| "p50_s": 0.10395996645092964, |
| "p95_s": 0.10869606416672467 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 8, |
| "n_correct": 5, |
| "n_planned": 8, |
| "n_renormalized": 0, |
| "n_scorable": 8, |
| "n_valid": 8, |
| "operational_success": 1.0, |
| "ordinal_mae": 1.3499127477407455, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| } |
| }, |
| "price_per_1000_decisions_usd": null, |
| "probability_sources": [ |
| "native" |
| ], |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0, |
| "splits": { |
| "public": { |
| "accuracy": 0.42342342342342343, |
| "brier_mean": 0.7731682475813886, |
| "calibration_n": 111, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": 0.27849190682172775, |
| "n": 4 |
| }, |
| { |
| "accuracy": 0.125, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.37296687439084053, |
| "n": 8 |
| }, |
| { |
| "accuracy": 0.4166666666666667, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.4450460250178973, |
| "n": 12 |
| }, |
| { |
| "accuracy": 0.35, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5447314709424973, |
| "n": 20 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6454204533781324, |
| "n": 14 |
| }, |
| { |
| "accuracy": 0.3333333333333333, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7522401312987009, |
| "n": 6 |
| }, |
| { |
| "accuracy": 0.25, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8441648408770561, |
| "n": 8 |
| }, |
| { |
| "accuracy": 0.5897435897435898, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.965268883949671, |
| "n": 39 |
| } |
| ], |
| "ece": 0.2818109610596219, |
| "n": 111 |
| }, |
| "latency": { |
| "n": 111, |
| "p50_s": 0.11237071454524994, |
| "p95_s": 0.14899961836636066 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 111, |
| "n_correct": 47, |
| "n_planned": 111, |
| "n_renormalized": 0, |
| "n_scorable": 111, |
| "n_valid": 111, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.7110752028723558, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| } |
| }, |
| "version": "jevbench-v1" |
| }, |
| "original": { |
| "accuracy": 0.8055555555555556, |
| "brier_mean": 0.29652247410236326, |
| "calibration_n": 72, |
| "complete": true, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "dataset_hash": "914276022fa5f0ea70e4231f2af98e68c67b274b61bea8c440b33890401cd2f9", |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.3444499373435974, |
| "n": 2 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.4325794279575348, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.42857142857142855, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5371253064700535, |
| "n": 7 |
| }, |
| { |
| "accuracy": 0.6666666666666666, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6374844577577379, |
| "n": 9 |
| }, |
| { |
| "accuracy": 0.8571428571428571, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7471651860645839, |
| "n": 7 |
| }, |
| { |
| "accuracy": 0.9, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8518721401691437, |
| "n": 10 |
| }, |
| { |
| "accuracy": 0.8888888888888888, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9730205883582433, |
| "n": 36 |
| } |
| ], |
| "ece": 0.08584584999415612, |
| "n": 72 |
| }, |
| "latency": { |
| "n": 72, |
| "p50_s": 0.07467062026262283, |
| "p95_s": 0.1292133398354054 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "ledger_charged_usd": null, |
| "macro_accuracy": 0.8055555555555555, |
| "model_identities": [ |
| "pcdm:native:runs/tm1b" |
| ], |
| "n_attempted": 72, |
| "n_correct": 58, |
| "n_planned": 72, |
| "n_renormalized": 0, |
| "n_scorable": 72, |
| "n_valid": 72, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.39357847470091656, |
| "paraphrase_consistency": { |
| "agree": 30, |
| "agreement": 0.8333333333333334, |
| "both_correct_rate_all_pairs": 0.7222222222222222, |
| "both_valid": 36, |
| "pairs": 36 |
| }, |
| "per_family": { |
| "adequacy": { |
| "accuracy": 0.6666666666666666, |
| "brier_mean": 0.40836962254148307, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5817238092422485, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6328441649675369, |
| "n": 4 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8540955781936646, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.8333333333333334, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.972666323184967, |
| "n": 6 |
| } |
| ], |
| "ece": 0.1745835691690445, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.07050643116235733, |
| "p95_s": 0.08810084797441958 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 8, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "extraction": { |
| "accuracy": 1.0, |
| "brier_mean": 0.02007323946682944, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7408376932144165, |
| "n": 1 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8441984951496124, |
| "n": 2 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.968041201432546, |
| "n": 9 |
| } |
| ], |
| "ece": 0.07153287529945371, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.05768968164920807, |
| "p95_s": 0.08274002466350794 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 12, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "intent": { |
| "accuracy": 0.9166666666666666, |
| "brier_mean": 0.22698352684655446, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5113146007061005, |
| "n": 2 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7275521159172058, |
| "n": 1 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8921765089035034, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.875, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9776975587010384, |
| "n": 8 |
| } |
| ], |
| "ece": 0.1816018869479497, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.11113478429615498, |
| "p95_s": 0.19621948227286334 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 11, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "ordinal": { |
| "accuracy": 0.6666666666666666, |
| "brier_mean": 0.387267867931643, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.3444499373435974, |
| "n": 2 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.4325794279575348, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5386167963345846, |
| "n": 3 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6215285360813141, |
| "n": 2 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.839689165353775, |
| "n": 4 |
| }, |
| { |
| "accuracy": null, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": null, |
| "n": 0 |
| } |
| ], |
| "ece": 0.32437977939844137, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.07719611376523972, |
| "p95_s": 0.11410039383918047 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 8, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.39357847470091656, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "policy": { |
| "accuracy": 0.75, |
| "brier_mean": 0.3966635065443795, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6735906898975372, |
| "n": 2 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.859863817691803, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.7777777777777778, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9743029342757331, |
| "n": 9 |
| } |
| ], |
| "ece": 0.18800366421540576, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.10365468636155128, |
| "p95_s": 0.15637512318789956 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 9, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| }, |
| "routing": { |
| "accuracy": 0.8333333333333334, |
| "brier_mean": 0.3397770812832901, |
| "calibration_n": 12, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5396737456321716, |
| "n": 1 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6157450079917908, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.8, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7523532986640931, |
| "n": 5 |
| }, |
| { |
| "accuracy": 0.0, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8654318451881409, |
| "n": 1 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9725163877010345, |
| "n": 4 |
| } |
| ], |
| "ece": 0.17151508728663126, |
| "n": 12 |
| }, |
| "latency": { |
| "n": 12, |
| "p50_s": 0.053897660225629807, |
| "p95_s": 0.08854644279927014 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 12, |
| "n_correct": 10, |
| "n_planned": 12, |
| "n_renormalized": 0, |
| "n_scorable": 12, |
| "n_valid": 12, |
| "operational_success": 1.0, |
| "ordinal_mae": null, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| } |
| }, |
| "price_per_1000_decisions_usd": null, |
| "probability_sources": [ |
| "native" |
| ], |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0, |
| "splits": { |
| "public": { |
| "accuracy": 0.8055555555555556, |
| "brier_mean": 0.29652247410236326, |
| "calibration_n": 72, |
| "cost_basis": [ |
| "local_gpu_no_provider_tariff" |
| ], |
| "coverage": 1.0, |
| "ece": { |
| "bins": [ |
| { |
| "accuracy": null, |
| "hi": 0.1, |
| "lo": 0.0, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.2, |
| "lo": 0.1, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": null, |
| "hi": 0.3, |
| "lo": 0.2, |
| "mean_confidence": null, |
| "n": 0 |
| }, |
| { |
| "accuracy": 0.5, |
| "hi": 0.4, |
| "lo": 0.3, |
| "mean_confidence": 0.3444499373435974, |
| "n": 2 |
| }, |
| { |
| "accuracy": 1.0, |
| "hi": 0.5, |
| "lo": 0.4, |
| "mean_confidence": 0.4325794279575348, |
| "n": 1 |
| }, |
| { |
| "accuracy": 0.42857142857142855, |
| "hi": 0.6, |
| "lo": 0.5, |
| "mean_confidence": 0.5371253064700535, |
| "n": 7 |
| }, |
| { |
| "accuracy": 0.6666666666666666, |
| "hi": 0.7, |
| "lo": 0.6, |
| "mean_confidence": 0.6374844577577379, |
| "n": 9 |
| }, |
| { |
| "accuracy": 0.8571428571428571, |
| "hi": 0.8, |
| "lo": 0.7, |
| "mean_confidence": 0.7471651860645839, |
| "n": 7 |
| }, |
| { |
| "accuracy": 0.9, |
| "hi": 0.9, |
| "lo": 0.8, |
| "mean_confidence": 0.8518721401691437, |
| "n": 10 |
| }, |
| { |
| "accuracy": 0.8888888888888888, |
| "hi": 1.0, |
| "lo": 0.9, |
| "mean_confidence": 0.9730205883582433, |
| "n": 36 |
| } |
| ], |
| "ece": 0.08584584999415612, |
| "n": 72 |
| }, |
| "latency": { |
| "n": 72, |
| "p50_s": 0.07467062026262283, |
| "p95_s": 0.1292133398354054 |
| }, |
| "latency_failures": { |
| "n": 0, |
| "p50_s": null, |
| "p95_s": null |
| }, |
| "n_attempted": 72, |
| "n_correct": 58, |
| "n_planned": 72, |
| "n_renormalized": 0, |
| "n_scorable": 72, |
| "n_valid": 72, |
| "operational_success": 1.0, |
| "ordinal_mae": 0.39357847470091656, |
| "price_per_1000_decisions_usd": null, |
| "schema_validity": 1.0, |
| "schema_validity_strict": 1.0 |
| } |
| }, |
| "version": "jevbench-v1" |
| } |
| } |
