PACE / performance /metrics.json
rocky250's picture
Upload PACE-SME proposed inference model and evaluation assets
6cbbcc4 verified
Raw History Blame Contribute Delete
18.6 kB
{
"model": "PACE_Proposed",
"decision_threshold": 0.22682374024878768,
"calibration_method": "isotonic",
"validation_metrics": {
"n": 192,
"evaluation_weight_sum": 192.0,
"effective_sample_size": 192.0,
"positive_rate": 0.21875,
"threshold": 0.5,
"accuracy": 0.8020833333333334,
"balanced_accuracy": 0.6590476190476191,
"precision": 0.5666666666666667,
"recall": 0.40476190476190477,
"specificity": 0.9133333333333333,
"negative_predictive_value": 0.845679012345679,
"mcc": 0.36216736040534453,
"f1_macro": 0.6752136752136753,
"alert_count": 30,
"alert_share": 0.15625,
"true_negative": 137.0,
"false_positive": 13.0,
"false_negative": 25.0,
"true_positive": 17.0,
"brier": 0.1505598980894374,
"ece": 0.1073413013806345,
"roc_auc": 0.753968253968254,
"pr_auc": 0.4763557579842879,
"model": "PACE_Proposed",
"model_family": "PACE_Proposed",
"training_balance_mode": "ensemble_of_development_selected_variants",
"selected_calibration_method": "isotonic",
"calibration_selection_brier": 0.1505598980894374,
"calibration_selection_log_loss": 0.5206843925204533,
"calibration_method_candidates_completed": 2,
"calibration_method_failures": "[]",
"validation_threshold": 0.22682374024878768,
"validation_weighted_error_cost": 95.0,
"is_proposed_pace_architecture": true,
"validation_selection_rank": 2,
"validation_cost_tie_group_size": 1,
"selection_tie_break_rule": "lowest validation cost; threshold ties choose the highest threshold (lowest alert share); model ties then use highest AP, lowest Brier, and alphabetical name",
"selected_on_primary_validation_objective": false
},
"locked_test_metrics": {
"n": 240,
"evaluation_weight_sum": 240.0,
"effective_sample_size": 240.0,
"positive_rate": 0.25416666666666665,
"threshold": 0.22682374024878768,
"accuracy": 0.6791666666666667,
"balanced_accuracy": 0.7200750984522393,
"precision": 0.4298245614035088,
"recall": 0.8032786885245902,
"specificity": 0.6368715083798883,
"negative_predictive_value": 0.9047619047619048,
"mcc": 0.3837555197834689,
"f1_macro": 0.6537704918032787,
"alert_count": 114,
"alert_share": 0.475,
"true_negative": 114.0,
"false_positive": 65.0,
"false_negative": 12.0,
"true_positive": 49.0,
"brier": 0.14867123005093147,
"ece": 0.019425255616746365,
"roc_auc": 0.7910522941661324,
"pr_auc": 0.5844628769710181
},
"locked_test_comparison": [
{
"n": 240,
"evaluation_weight_sum": 240.0,
"effective_sample_size": 240.0,
"positive_rate": 0.25416666666666665,
"threshold": 0.18413703819805452,
"accuracy": 0.6416666666666667,
"balanced_accuracy": 0.69493543364777,
"precision": 0.3983739837398374,
"recall": 0.8032786885245902,
"specificity": 0.5865921787709497,
"negative_predictive_value": 0.8974358974358975,
"mcc": 0.33959925637223404,
"f1_macro": 0.6210340775558167,
"alert_count": 123,
"alert_share": 0.5125,
"true_negative": 105.0,
"false_positive": 74.0,
"false_negative": 12.0,
"true_positive": 49.0,
"brier": 0.15007812956253685,
"ece": 0.028939999890003995,
"roc_auc": 0.7905485850352596,
"pr_auc": 0.57955576009116,
"model": "LightGBM_Unweighted",
"validation_pr_auc": 0.4377390719231052,
"validation_threshold": 0.18413703819805452,
"selected_calibration_method": "sigmoid",
"validation_weighted_error_cost": 94.0,
"test_weighted_error_cost": 134.0,
"test_weighted_error_cost_per_case": 0.5583333333333333,
"test_cost_is_reporting_only": true,
"selected_on_validation": true,
"is_proposed_pace_architecture": false
},
{
"n": 240,
"evaluation_weight_sum": 240.0,
"effective_sample_size": 240.0,
"positive_rate": 0.25416666666666665,
"threshold": 0.22682374024878768,
"accuracy": 0.6791666666666667,
"balanced_accuracy": 0.7200750984522393,
"precision": 0.4298245614035088,
"recall": 0.8032786885245902,
"specificity": 0.6368715083798883,
"negative_predictive_value": 0.9047619047619048,
"mcc": 0.3837555197834689,
"f1_macro": 0.6537704918032787,
"alert_count": 114,
"alert_share": 0.475,
"true_negative": 114.0,
"false_positive": 65.0,
"false_negative": 12.0,
"true_positive": 49.0,
"brier": 0.14867123005093147,
"ece": 0.019425255616746365,
"roc_auc": 0.7910522941661324,
"pr_auc": 0.5844628769710181,
"model": "PACE_Proposed",
"validation_pr_auc": 0.4763557579842879,
"validation_threshold": 0.22682374024878768,
"selected_calibration_method": "isotonic",
"validation_weighted_error_cost": 95.0,
"test_weighted_error_cost": 125.0,
"test_weighted_error_cost_per_case": 0.5208333333333334,
"test_cost_is_reporting_only": true,
"selected_on_validation": false,
"is_proposed_pace_architecture": true
},
{
"n": 240,
"evaluation_weight_sum": 240.0,
"effective_sample_size": 240.0,
"positive_rate": 0.25416666666666665,
"threshold": 0.18357069160652448,
"accuracy": 0.6666666666666666,
"balanced_accuracy": 0.7062917849619929,
"precision": 0.41739130434782606,
"recall": 0.7868852459016393,
"specificity": 0.6256983240223464,
"negative_predictive_value": 0.896,
"mcc": 0.3595832353307929,
"f1_macro": 0.6411483253588517,
"alert_count": 115,
"alert_share": 0.4791666666666667,
"true_negative": 112.0,
"false_positive": 67.0,
"false_negative": 13.0,
"true_positive": 48.0,
"brier": 0.14999969329298418,
"ece": 0.02559250633385094,
"roc_auc": 0.7905485850352597,
"pr_auc": 0.566264650742579,
"model": "CatBoost_Unweighted",
"validation_pr_auc": 0.5068125800878094,
"validation_threshold": 0.18357069160652448,
"selected_calibration_method": "sigmoid",
"validation_weighted_error_cost": 98.0,
"test_weighted_error_cost": 132.0,
"test_weighted_error_cost_per_case": 0.55,
"test_cost_is_reporting_only": true,
"selected_on_validation": false,
"is_proposed_pace_architecture": false
},
{
"n": 240,
"evaluation_weight_sum": 240.0,
"effective_sample_size": 240.0,
"positive_rate": 0.25416666666666665,
"threshold": 0.27742241938337947,
"accuracy": 0.7,
"balanced_accuracy": 0.6746039014561773,
"precision": 0.4367816091954023,
"recall": 0.6229508196721312,
"specificity": 0.7262569832402235,
"negative_predictive_value": 0.8496732026143791,
"mcc": 0.31627876227430407,
"f1_macro": 0.6483230218169977,
"alert_count": 87,
"alert_share": 0.3625,
"true_negative": 130.0,
"false_positive": 49.0,
"false_negative": 23.0,
"true_positive": 38.0,
"brier": 0.1648318646505202,
"ece": 0.056896358772106276,
"roc_auc": 0.752907775437311,
"pr_auc": 0.4665372151676643,
"model": "Random_Forest_Unweighted",
"validation_pr_auc": 0.4584422034509623,
"validation_threshold": 0.27742241938337947,
"selected_calibration_method": "sigmoid",
"validation_weighted_error_cost": 106.0,
"test_weighted_error_cost": 164.0,
"test_weighted_error_cost_per_case": 0.6833333333333333,
"test_cost_is_reporting_only": true,
"selected_on_validation": false,
"is_proposed_pace_architecture": false
},
{
"n": 240,
"evaluation_weight_sum": 240.0,
"effective_sample_size": 240.0,
"positive_rate": 0.25416666666666665,
"threshold": 0.2911463280719015,
"accuracy": 0.7083333333333334,
"balanced_accuracy": 0.6315596666361388,
"precision": 0.43283582089552236,
"recall": 0.47540983606557374,
"specificity": 0.7877094972067039,
"negative_predictive_value": 0.815028901734104,
"mcc": 0.2553781520804498,
"f1_macro": 0.6271306818181819,
"alert_count": 67,
"alert_share": 0.2791666666666667,
"true_negative": 141.0,
"false_positive": 38.0,
"false_negative": 32.0,
"true_positive": 29.0,
"brier": 0.17419403294978,
"ece": 0.06063549040957894,
"roc_auc": 0.7074823701804194,
"pr_auc": 0.44296505336737324,
"model": "Extra_Trees_Unweighted",
"validation_pr_auc": 0.450221132106621,
"validation_threshold": 0.2911463280719015,
"selected_calibration_method": "sigmoid",
"validation_weighted_error_cost": 110.0,
"test_weighted_error_cost": 198.0,
"test_weighted_error_cost_per_case": 0.825,
"test_cost_is_reporting_only": true,
"selected_on_validation": false,
"is_proposed_pace_architecture": false
}
],
"validation_model_comparison": [
{
"n": 192,
"evaluation_weight_sum": 192.0,
"effective_sample_size": 192.0,
"positive_rate": 0.21875,
"threshold": 0.5,
"accuracy": 0.7864583333333334,
"balanced_accuracy": 0.6319047619047619,
"precision": 0.5172413793103449,
"recall": 0.35714285714285715,
"specificity": 0.9066666666666666,
"negative_predictive_value": 0.8343558282208589,
"mcc": 0.304556549579163,
"f1_macro": 0.6457723979660712,
"alert_count": 29,
"alert_share": 0.15104166666666666,
"true_negative": 136.0,
"false_positive": 14.0,
"false_negative": 27.0,
"true_positive": 15.0,
"brier": 0.15617480591586372,
"ece": 0.07823760055852294,
"roc_auc": 0.744920634920635,
"pr_auc": 0.4377390719231052,
"model": "LightGBM_Unweighted",
"model_family": "LightGBM",
"training_balance_mode": "unweighted_natural_prevalence",
"selected_calibration_method": "sigmoid",
"calibration_selection_brier": 0.15617480591586372,
"calibration_selection_log_loss": 0.4798808065590463,
"calibration_method_candidates_completed": 2,
"calibration_method_failures": "[]",
"validation_threshold": 0.18413703819805452,
"validation_weighted_error_cost": 94.0,
"is_proposed_pace_architecture": false,
"validation_selection_rank": 1,
"validation_cost_tie_group_size": 1,
"selection_tie_break_rule": "lowest validation cost; threshold ties choose the highest threshold (lowest alert share); model ties then use highest AP, lowest Brier, and alphabetical name",
"selected_on_primary_validation_objective": true
},
{
"n": 192,
"evaluation_weight_sum": 192.0,
"effective_sample_size": 192.0,
"positive_rate": 0.21875,
"threshold": 0.5,
"accuracy": 0.8020833333333334,
"balanced_accuracy": 0.6590476190476191,
"precision": 0.5666666666666667,
"recall": 0.40476190476190477,
"specificity": 0.9133333333333333,
"negative_predictive_value": 0.845679012345679,
"mcc": 0.36216736040534453,
"f1_macro": 0.6752136752136753,
"alert_count": 30,
"alert_share": 0.15625,
"true_negative": 137.0,
"false_positive": 13.0,
"false_negative": 25.0,
"true_positive": 17.0,
"brier": 0.1505598980894374,
"ece": 0.1073413013806345,
"roc_auc": 0.753968253968254,
"pr_auc": 0.4763557579842879,
"model": "PACE_Proposed",
"model_family": "PACE_Proposed",
"training_balance_mode": "ensemble_of_development_selected_variants",
"selected_calibration_method": "isotonic",
"calibration_selection_brier": 0.1505598980894374,
"calibration_selection_log_loss": 0.5206843925204533,
"calibration_method_candidates_completed": 2,
"calibration_method_failures": "[]",
"validation_threshold": 0.22682374024878768,
"validation_weighted_error_cost": 95.0,
"is_proposed_pace_architecture": true,
"validation_selection_rank": 2,
"validation_cost_tie_group_size": 1,
"selection_tie_break_rule": "lowest validation cost; threshold ties choose the highest threshold (lowest alert share); model ties then use highest AP, lowest Brier, and alphabetical name",
"selected_on_primary_validation_objective": false
},
{
"n": 192,
"evaluation_weight_sum": 192.0,
"effective_sample_size": 192.0,
"positive_rate": 0.21875,
"threshold": 0.5,
"accuracy": 0.796875,
"balanced_accuracy": 0.6471428571428571,
"precision": 0.5517241379310345,
"recall": 0.38095238095238093,
"specificity": 0.9133333333333333,
"negative_predictive_value": 0.8404907975460123,
"mcc": 0.33973997768939124,
"f1_macro": 0.6630517931872385,
"alert_count": 29,
"alert_share": 0.15104166666666666,
"true_negative": 137.0,
"false_positive": 13.0,
"false_negative": 26.0,
"true_positive": 16.0,
"brier": 0.14544193657274437,
"ece": 0.04818281599257537,
"roc_auc": 0.7561904761904761,
"pr_auc": 0.5068125800878094,
"model": "CatBoost_Unweighted",
"model_family": "CatBoost",
"training_balance_mode": "unweighted_natural_prevalence",
"selected_calibration_method": "sigmoid",
"calibration_selection_brier": 0.14544193657274437,
"calibration_selection_log_loss": 0.4563620390839209,
"calibration_method_candidates_completed": 2,
"calibration_method_failures": "[]",
"validation_threshold": 0.18357069160652448,
"validation_weighted_error_cost": 98.0,
"is_proposed_pace_architecture": false,
"validation_selection_rank": 3,
"validation_cost_tie_group_size": 1,
"selection_tie_break_rule": "lowest validation cost; threshold ties choose the highest threshold (lowest alert share); model ties then use highest AP, lowest Brier, and alphabetical name",
"selected_on_primary_validation_objective": false
},
{
"n": 192,
"evaluation_weight_sum": 192.0,
"effective_sample_size": 192.0,
"positive_rate": 0.21875,
"threshold": 0.5,
"accuracy": 0.7708333333333334,
"balanced_accuracy": 0.5961904761904762,
"precision": 0.46153846153846156,
"recall": 0.2857142857142857,
"specificity": 0.9066666666666666,
"negative_predictive_value": 0.8192771084337349,
"mcc": 0.23242970290961337,
"f1_macro": 0.6068503350707372,
"alert_count": 26,
"alert_share": 0.13541666666666666,
"true_negative": 136.0,
"false_positive": 14.0,
"false_negative": 30.0,
"true_positive": 12.0,
"brier": 0.1505577797810214,
"ece": 0.062201546107280084,
"roc_auc": 0.7373015873015873,
"pr_auc": 0.4584422034509623,
"model": "Random_Forest_Unweighted",
"model_family": "Random_Forest",
"training_balance_mode": "unweighted_natural_prevalence",
"selected_calibration_method": "sigmoid",
"calibration_selection_brier": 0.1505577797810214,
"calibration_selection_log_loss": 0.46614095848259546,
"calibration_method_candidates_completed": 2,
"calibration_method_failures": "[]",
"validation_threshold": 0.27742241938337947,
"validation_weighted_error_cost": 106.0,
"is_proposed_pace_architecture": false,
"validation_selection_rank": 4,
"validation_cost_tie_group_size": 1,
"selection_tie_break_rule": "lowest validation cost; threshold ties choose the highest threshold (lowest alert share); model ties then use highest AP, lowest Brier, and alphabetical name",
"selected_on_primary_validation_objective": false
},
{
"n": 192,
"evaluation_weight_sum": 192.0,
"effective_sample_size": 192.0,
"positive_rate": 0.21875,
"threshold": 0.5,
"accuracy": 0.7760416666666666,
"balanced_accuracy": 0.5480952380952381,
"precision": 0.46153846153846156,
"recall": 0.14285714285714285,
"specificity": 0.9533333333333334,
"negative_predictive_value": 0.7988826815642458,
"mcc": 0.15827202458147038,
"f1_macro": 0.5437413650179608,
"alert_count": 13,
"alert_share": 0.06770833333333333,
"true_negative": 143.0,
"false_positive": 7.0,
"false_negative": 36.0,
"true_positive": 6.0,
"brier": 0.15353397855036136,
"ece": 0.05887421621997732,
"roc_auc": 0.7211111111111111,
"pr_auc": 0.450221132106621,
"model": "Extra_Trees_Unweighted",
"model_family": "Extra_Trees",
"training_balance_mode": "unweighted_natural_prevalence",
"selected_calibration_method": "sigmoid",
"calibration_selection_brier": 0.15353397855036136,
"calibration_selection_log_loss": 0.4766880125804945,
"calibration_method_candidates_completed": 2,
"calibration_method_failures": "[]",
"validation_threshold": 0.2911463280719015,
"validation_weighted_error_cost": 110.0,
"is_proposed_pace_architecture": false,
"validation_selection_rank": 5,
"validation_cost_tie_group_size": 1,
"selection_tie_break_rule": "lowest validation cost; threshold ties choose the highest threshold (lowest alert share); model ties then use highest AP, lowest Brier, and alphabetical name",
"selected_on_primary_validation_objective": false
}
],
"nested_cv_proposed_model_summary": [
{
"model": "PACE_nested_simplex_proposed",
"model_type": "pace_simplex_proposed",
"model_family": "PACE",
"outer_folds": 3,
"mean_average_precision": 0.5925735951377111,
"sd_average_precision": 0.04039160600707779,
"mean_roc_auc": 0.7874683918404233,
"sd_roc_auc": 0.03521949945958485,
"mean_brier": 0.1530745166497782,
"sd_brier": 0.01693497587921028,
"mean_balanced_accuracy": 0.6850699702682824,
"mean_weighted_error_cost": 180.33333333333334,
"sd_weighted_error_cost": 16.258331197676267,
"mean_cost_per_case": 0.5635416666666667,
"mean_selected_threshold": 0.18364686054410828,
"sd_selected_threshold": 0.03541113054853995,
"all_expected_outer_folds_present": true
}
],
"nested_cv_status": "COMPLETED",
"selected_validation_model": "LightGBM_Unweighted",
"data_and_claim_boundary": "Only the primary Bangladesh retrospective target is directly relevant to this model. Other dataset results are separate benchmarks and must not be pooled with these metrics."
}