Download performance/metrics.json from rocky250/PACE: direct link, hf CLI and curl.
- Browser
- Download file 18.6 kB
-
https://huggingface.co/rocky250/PACE/resolve/main/performance/metrics.json
- Command line
-
hf download hf://rocky250/PACE/performance/metrics.json
-
curl -L -o metrics.json https://huggingface.co/rocky250/PACE/resolve/main/performance/metrics.json
18.6 kB
| { | |
| "model": "PACE_Proposed", | |
| "decision_threshold": 0.22682374024878768, | |
| "calibration_method": "isotonic", | |
| "validation_metrics": { | |
| "n": 192, | |
| "evaluation_weight_sum": 192.0, | |
| "effective_sample_size": 192.0, | |
| "positive_rate": 0.21875, | |
| "threshold": 0.5, | |
| "accuracy": 0.8020833333333334, | |
| "balanced_accuracy": 0.6590476190476191, | |
| "precision": 0.5666666666666667, | |
| "recall": 0.40476190476190477, | |
| "specificity": 0.9133333333333333, | |
| "negative_predictive_value": 0.845679012345679, | |
| "mcc": 0.36216736040534453, | |
| "f1_macro": 0.6752136752136753, | |
| "alert_count": 30, | |
| "alert_share": 0.15625, | |
| "true_negative": 137.0, | |
| "false_positive": 13.0, | |
| "false_negative": 25.0, | |
| "true_positive": 17.0, | |
| "brier": 0.1505598980894374, | |
| "ece": 0.1073413013806345, | |
| "roc_auc": 0.753968253968254, | |
| "pr_auc": 0.4763557579842879, | |
| "model": "PACE_Proposed", | |
| "model_family": "PACE_Proposed", | |
| "training_balance_mode": "ensemble_of_development_selected_variants", | |
| "selected_calibration_method": "isotonic", | |
| "calibration_selection_brier": 0.1505598980894374, | |
| "calibration_selection_log_loss": 0.5206843925204533, | |
| "calibration_method_candidates_completed": 2, | |
| "calibration_method_failures": "[]", | |
| "validation_threshold": 0.22682374024878768, | |
| "validation_weighted_error_cost": 95.0, | |
| "is_proposed_pace_architecture": true, | |
| "validation_selection_rank": 2, | |
| "validation_cost_tie_group_size": 1, | |
| "selection_tie_break_rule": "lowest validation cost; threshold ties choose the highest threshold (lowest alert share); model ties then use highest AP, lowest Brier, and alphabetical name", | |
| "selected_on_primary_validation_objective": false | |
| }, | |
| "locked_test_metrics": { | |
| "n": 240, | |
| "evaluation_weight_sum": 240.0, | |
| "effective_sample_size": 240.0, | |
| "positive_rate": 0.25416666666666665, | |
| "threshold": 0.22682374024878768, | |
| "accuracy": 0.6791666666666667, | |
| "balanced_accuracy": 0.7200750984522393, | |
| "precision": 0.4298245614035088, | |
| "recall": 0.8032786885245902, | |
| "specificity": 0.6368715083798883, | |
| "negative_predictive_value": 0.9047619047619048, | |
| "mcc": 0.3837555197834689, | |
| "f1_macro": 0.6537704918032787, | |
| "alert_count": 114, | |
| "alert_share": 0.475, | |
| "true_negative": 114.0, | |
| "false_positive": 65.0, | |
| "false_negative": 12.0, | |
| "true_positive": 49.0, | |
| "brier": 0.14867123005093147, | |
| "ece": 0.019425255616746365, | |
| "roc_auc": 0.7910522941661324, | |
| "pr_auc": 0.5844628769710181 | |
| }, | |
| "locked_test_comparison": [ | |
| { | |
| "n": 240, | |
| "evaluation_weight_sum": 240.0, | |
| "effective_sample_size": 240.0, | |
| "positive_rate": 0.25416666666666665, | |
| "threshold": 0.18413703819805452, | |
| "accuracy": 0.6416666666666667, | |
| "balanced_accuracy": 0.69493543364777, | |
| "precision": 0.3983739837398374, | |
| "recall": 0.8032786885245902, | |
| "specificity": 0.5865921787709497, | |
| "negative_predictive_value": 0.8974358974358975, | |
| "mcc": 0.33959925637223404, | |
| "f1_macro": 0.6210340775558167, | |
| "alert_count": 123, | |
| "alert_share": 0.5125, | |
| "true_negative": 105.0, | |
| "false_positive": 74.0, | |
| "false_negative": 12.0, | |
| "true_positive": 49.0, | |
| "brier": 0.15007812956253685, | |
| "ece": 0.028939999890003995, | |
| "roc_auc": 0.7905485850352596, | |
| "pr_auc": 0.57955576009116, | |
| "model": "LightGBM_Unweighted", | |
| "validation_pr_auc": 0.4377390719231052, | |
| "validation_threshold": 0.18413703819805452, | |
| "selected_calibration_method": "sigmoid", | |
| "validation_weighted_error_cost": 94.0, | |
| "test_weighted_error_cost": 134.0, | |
| "test_weighted_error_cost_per_case": 0.5583333333333333, | |
| "test_cost_is_reporting_only": true, | |
| "selected_on_validation": true, | |
| "is_proposed_pace_architecture": false | |
| }, | |
| { | |
| "n": 240, | |
| "evaluation_weight_sum": 240.0, | |
| "effective_sample_size": 240.0, | |
| "positive_rate": 0.25416666666666665, | |
| "threshold": 0.22682374024878768, | |
| "accuracy": 0.6791666666666667, | |
| "balanced_accuracy": 0.7200750984522393, | |
| "precision": 0.4298245614035088, | |
| "recall": 0.8032786885245902, | |
| "specificity": 0.6368715083798883, | |
| "negative_predictive_value": 0.9047619047619048, | |
| "mcc": 0.3837555197834689, | |
| "f1_macro": 0.6537704918032787, | |
| "alert_count": 114, | |
| "alert_share": 0.475, | |
| "true_negative": 114.0, | |
| "false_positive": 65.0, | |
| "false_negative": 12.0, | |
| "true_positive": 49.0, | |
| "brier": 0.14867123005093147, | |
| "ece": 0.019425255616746365, | |
| "roc_auc": 0.7910522941661324, | |
| "pr_auc": 0.5844628769710181, | |
| "model": "PACE_Proposed", | |
| "validation_pr_auc": 0.4763557579842879, | |
| "validation_threshold": 0.22682374024878768, | |
| "selected_calibration_method": "isotonic", | |
| "validation_weighted_error_cost": 95.0, | |
| "test_weighted_error_cost": 125.0, | |
| "test_weighted_error_cost_per_case": 0.5208333333333334, | |
| "test_cost_is_reporting_only": true, | |
| "selected_on_validation": false, | |
| "is_proposed_pace_architecture": true | |
| }, | |
| { | |
| "n": 240, | |
| "evaluation_weight_sum": 240.0, | |
| "effective_sample_size": 240.0, | |
| "positive_rate": 0.25416666666666665, | |
| "threshold": 0.18357069160652448, | |
| "accuracy": 0.6666666666666666, | |
| "balanced_accuracy": 0.7062917849619929, | |
| "precision": 0.41739130434782606, | |
| "recall": 0.7868852459016393, | |
| "specificity": 0.6256983240223464, | |
| "negative_predictive_value": 0.896, | |
| "mcc": 0.3595832353307929, | |
| "f1_macro": 0.6411483253588517, | |
| "alert_count": 115, | |
| "alert_share": 0.4791666666666667, | |
| "true_negative": 112.0, | |
| "false_positive": 67.0, | |
| "false_negative": 13.0, | |
| "true_positive": 48.0, | |
| "brier": 0.14999969329298418, | |
| "ece": 0.02559250633385094, | |
| "roc_auc": 0.7905485850352597, | |
| "pr_auc": 0.566264650742579, | |
| "model": "CatBoost_Unweighted", | |
| "validation_pr_auc": 0.5068125800878094, | |
| "validation_threshold": 0.18357069160652448, | |
| "selected_calibration_method": "sigmoid", | |
| "validation_weighted_error_cost": 98.0, | |
| "test_weighted_error_cost": 132.0, | |
| "test_weighted_error_cost_per_case": 0.55, | |
| "test_cost_is_reporting_only": true, | |
| "selected_on_validation": false, | |
| "is_proposed_pace_architecture": false | |
| }, | |
| { | |
| "n": 240, | |
| "evaluation_weight_sum": 240.0, | |
| "effective_sample_size": 240.0, | |
| "positive_rate": 0.25416666666666665, | |
| "threshold": 0.27742241938337947, | |
| "accuracy": 0.7, | |
| "balanced_accuracy": 0.6746039014561773, | |
| "precision": 0.4367816091954023, | |
| "recall": 0.6229508196721312, | |
| "specificity": 0.7262569832402235, | |
| "negative_predictive_value": 0.8496732026143791, | |
| "mcc": 0.31627876227430407, | |
| "f1_macro": 0.6483230218169977, | |
| "alert_count": 87, | |
| "alert_share": 0.3625, | |
| "true_negative": 130.0, | |
| "false_positive": 49.0, | |
| "false_negative": 23.0, | |
| "true_positive": 38.0, | |
| "brier": 0.1648318646505202, | |
| "ece": 0.056896358772106276, | |
| "roc_auc": 0.752907775437311, | |
| "pr_auc": 0.4665372151676643, | |
| "model": "Random_Forest_Unweighted", | |
| "validation_pr_auc": 0.4584422034509623, | |
| "validation_threshold": 0.27742241938337947, | |
| "selected_calibration_method": "sigmoid", | |
| "validation_weighted_error_cost": 106.0, | |
| "test_weighted_error_cost": 164.0, | |
| "test_weighted_error_cost_per_case": 0.6833333333333333, | |
| "test_cost_is_reporting_only": true, | |
| "selected_on_validation": false, | |
| "is_proposed_pace_architecture": false | |
| }, | |
| { | |
| "n": 240, | |
| "evaluation_weight_sum": 240.0, | |
| "effective_sample_size": 240.0, | |
| "positive_rate": 0.25416666666666665, | |
| "threshold": 0.2911463280719015, | |
| "accuracy": 0.7083333333333334, | |
| "balanced_accuracy": 0.6315596666361388, | |
| "precision": 0.43283582089552236, | |
| "recall": 0.47540983606557374, | |
| "specificity": 0.7877094972067039, | |
| "negative_predictive_value": 0.815028901734104, | |
| "mcc": 0.2553781520804498, | |
| "f1_macro": 0.6271306818181819, | |
| "alert_count": 67, | |
| "alert_share": 0.2791666666666667, | |
| "true_negative": 141.0, | |
| "false_positive": 38.0, | |
| "false_negative": 32.0, | |
| "true_positive": 29.0, | |
| "brier": 0.17419403294978, | |
| "ece": 0.06063549040957894, | |
| "roc_auc": 0.7074823701804194, | |
| "pr_auc": 0.44296505336737324, | |
| "model": "Extra_Trees_Unweighted", | |
| "validation_pr_auc": 0.450221132106621, | |
| "validation_threshold": 0.2911463280719015, | |
| "selected_calibration_method": "sigmoid", | |
| "validation_weighted_error_cost": 110.0, | |
| "test_weighted_error_cost": 198.0, | |
| "test_weighted_error_cost_per_case": 0.825, | |
| "test_cost_is_reporting_only": true, | |
| "selected_on_validation": false, | |
| "is_proposed_pace_architecture": false | |
| } | |
| ], | |
| "validation_model_comparison": [ | |
| { | |
| "n": 192, | |
| "evaluation_weight_sum": 192.0, | |
| "effective_sample_size": 192.0, | |
| "positive_rate": 0.21875, | |
| "threshold": 0.5, | |
| "accuracy": 0.7864583333333334, | |
| "balanced_accuracy": 0.6319047619047619, | |
| "precision": 0.5172413793103449, | |
| "recall": 0.35714285714285715, | |
| "specificity": 0.9066666666666666, | |
| "negative_predictive_value": 0.8343558282208589, | |
| "mcc": 0.304556549579163, | |
| "f1_macro": 0.6457723979660712, | |
| "alert_count": 29, | |
| "alert_share": 0.15104166666666666, | |
| "true_negative": 136.0, | |
| "false_positive": 14.0, | |
| "false_negative": 27.0, | |
| "true_positive": 15.0, | |
| "brier": 0.15617480591586372, | |
| "ece": 0.07823760055852294, | |
| "roc_auc": 0.744920634920635, | |
| "pr_auc": 0.4377390719231052, | |
| "model": "LightGBM_Unweighted", | |
| "model_family": "LightGBM", | |
| "training_balance_mode": "unweighted_natural_prevalence", | |
| "selected_calibration_method": "sigmoid", | |
| "calibration_selection_brier": 0.15617480591586372, | |
| "calibration_selection_log_loss": 0.4798808065590463, | |
| "calibration_method_candidates_completed": 2, | |
| "calibration_method_failures": "[]", | |
| "validation_threshold": 0.18413703819805452, | |
| "validation_weighted_error_cost": 94.0, | |
| "is_proposed_pace_architecture": false, | |
| "validation_selection_rank": 1, | |
| "validation_cost_tie_group_size": 1, | |
| "selection_tie_break_rule": "lowest validation cost; threshold ties choose the highest threshold (lowest alert share); model ties then use highest AP, lowest Brier, and alphabetical name", | |
| "selected_on_primary_validation_objective": true | |
| }, | |
| { | |
| "n": 192, | |
| "evaluation_weight_sum": 192.0, | |
| "effective_sample_size": 192.0, | |
| "positive_rate": 0.21875, | |
| "threshold": 0.5, | |
| "accuracy": 0.8020833333333334, | |
| "balanced_accuracy": 0.6590476190476191, | |
| "precision": 0.5666666666666667, | |
| "recall": 0.40476190476190477, | |
| "specificity": 0.9133333333333333, | |
| "negative_predictive_value": 0.845679012345679, | |
| "mcc": 0.36216736040534453, | |
| "f1_macro": 0.6752136752136753, | |
| "alert_count": 30, | |
| "alert_share": 0.15625, | |
| "true_negative": 137.0, | |
| "false_positive": 13.0, | |
| "false_negative": 25.0, | |
| "true_positive": 17.0, | |
| "brier": 0.1505598980894374, | |
| "ece": 0.1073413013806345, | |
| "roc_auc": 0.753968253968254, | |
| "pr_auc": 0.4763557579842879, | |
| "model": "PACE_Proposed", | |
| "model_family": "PACE_Proposed", | |
| "training_balance_mode": "ensemble_of_development_selected_variants", | |
| "selected_calibration_method": "isotonic", | |
| "calibration_selection_brier": 0.1505598980894374, | |
| "calibration_selection_log_loss": 0.5206843925204533, | |
| "calibration_method_candidates_completed": 2, | |
| "calibration_method_failures": "[]", | |
| "validation_threshold": 0.22682374024878768, | |
| "validation_weighted_error_cost": 95.0, | |
| "is_proposed_pace_architecture": true, | |
| "validation_selection_rank": 2, | |
| "validation_cost_tie_group_size": 1, | |
| "selection_tie_break_rule": "lowest validation cost; threshold ties choose the highest threshold (lowest alert share); model ties then use highest AP, lowest Brier, and alphabetical name", | |
| "selected_on_primary_validation_objective": false | |
| }, | |
| { | |
| "n": 192, | |
| "evaluation_weight_sum": 192.0, | |
| "effective_sample_size": 192.0, | |
| "positive_rate": 0.21875, | |
| "threshold": 0.5, | |
| "accuracy": 0.796875, | |
| "balanced_accuracy": 0.6471428571428571, | |
| "precision": 0.5517241379310345, | |
| "recall": 0.38095238095238093, | |
| "specificity": 0.9133333333333333, | |
| "negative_predictive_value": 0.8404907975460123, | |
| "mcc": 0.33973997768939124, | |
| "f1_macro": 0.6630517931872385, | |
| "alert_count": 29, | |
| "alert_share": 0.15104166666666666, | |
| "true_negative": 137.0, | |
| "false_positive": 13.0, | |
| "false_negative": 26.0, | |
| "true_positive": 16.0, | |
| "brier": 0.14544193657274437, | |
| "ece": 0.04818281599257537, | |
| "roc_auc": 0.7561904761904761, | |
| "pr_auc": 0.5068125800878094, | |
| "model": "CatBoost_Unweighted", | |
| "model_family": "CatBoost", | |
| "training_balance_mode": "unweighted_natural_prevalence", | |
| "selected_calibration_method": "sigmoid", | |
| "calibration_selection_brier": 0.14544193657274437, | |
| "calibration_selection_log_loss": 0.4563620390839209, | |
| "calibration_method_candidates_completed": 2, | |
| "calibration_method_failures": "[]", | |
| "validation_threshold": 0.18357069160652448, | |
| "validation_weighted_error_cost": 98.0, | |
| "is_proposed_pace_architecture": false, | |
| "validation_selection_rank": 3, | |
| "validation_cost_tie_group_size": 1, | |
| "selection_tie_break_rule": "lowest validation cost; threshold ties choose the highest threshold (lowest alert share); model ties then use highest AP, lowest Brier, and alphabetical name", | |
| "selected_on_primary_validation_objective": false | |
| }, | |
| { | |
| "n": 192, | |
| "evaluation_weight_sum": 192.0, | |
| "effective_sample_size": 192.0, | |
| "positive_rate": 0.21875, | |
| "threshold": 0.5, | |
| "accuracy": 0.7708333333333334, | |
| "balanced_accuracy": 0.5961904761904762, | |
| "precision": 0.46153846153846156, | |
| "recall": 0.2857142857142857, | |
| "specificity": 0.9066666666666666, | |
| "negative_predictive_value": 0.8192771084337349, | |
| "mcc": 0.23242970290961337, | |
| "f1_macro": 0.6068503350707372, | |
| "alert_count": 26, | |
| "alert_share": 0.13541666666666666, | |
| "true_negative": 136.0, | |
| "false_positive": 14.0, | |
| "false_negative": 30.0, | |
| "true_positive": 12.0, | |
| "brier": 0.1505577797810214, | |
| "ece": 0.062201546107280084, | |
| "roc_auc": 0.7373015873015873, | |
| "pr_auc": 0.4584422034509623, | |
| "model": "Random_Forest_Unweighted", | |
| "model_family": "Random_Forest", | |
| "training_balance_mode": "unweighted_natural_prevalence", | |
| "selected_calibration_method": "sigmoid", | |
| "calibration_selection_brier": 0.1505577797810214, | |
| "calibration_selection_log_loss": 0.46614095848259546, | |
| "calibration_method_candidates_completed": 2, | |
| "calibration_method_failures": "[]", | |
| "validation_threshold": 0.27742241938337947, | |
| "validation_weighted_error_cost": 106.0, | |
| "is_proposed_pace_architecture": false, | |
| "validation_selection_rank": 4, | |
| "validation_cost_tie_group_size": 1, | |
| "selection_tie_break_rule": "lowest validation cost; threshold ties choose the highest threshold (lowest alert share); model ties then use highest AP, lowest Brier, and alphabetical name", | |
| "selected_on_primary_validation_objective": false | |
| }, | |
| { | |
| "n": 192, | |
| "evaluation_weight_sum": 192.0, | |
| "effective_sample_size": 192.0, | |
| "positive_rate": 0.21875, | |
| "threshold": 0.5, | |
| "accuracy": 0.7760416666666666, | |
| "balanced_accuracy": 0.5480952380952381, | |
| "precision": 0.46153846153846156, | |
| "recall": 0.14285714285714285, | |
| "specificity": 0.9533333333333334, | |
| "negative_predictive_value": 0.7988826815642458, | |
| "mcc": 0.15827202458147038, | |
| "f1_macro": 0.5437413650179608, | |
| "alert_count": 13, | |
| "alert_share": 0.06770833333333333, | |
| "true_negative": 143.0, | |
| "false_positive": 7.0, | |
| "false_negative": 36.0, | |
| "true_positive": 6.0, | |
| "brier": 0.15353397855036136, | |
| "ece": 0.05887421621997732, | |
| "roc_auc": 0.7211111111111111, | |
| "pr_auc": 0.450221132106621, | |
| "model": "Extra_Trees_Unweighted", | |
| "model_family": "Extra_Trees", | |
| "training_balance_mode": "unweighted_natural_prevalence", | |
| "selected_calibration_method": "sigmoid", | |
| "calibration_selection_brier": 0.15353397855036136, | |
| "calibration_selection_log_loss": 0.4766880125804945, | |
| "calibration_method_candidates_completed": 2, | |
| "calibration_method_failures": "[]", | |
| "validation_threshold": 0.2911463280719015, | |
| "validation_weighted_error_cost": 110.0, | |
| "is_proposed_pace_architecture": false, | |
| "validation_selection_rank": 5, | |
| "validation_cost_tie_group_size": 1, | |
| "selection_tie_break_rule": "lowest validation cost; threshold ties choose the highest threshold (lowest alert share); model ties then use highest AP, lowest Brier, and alphabetical name", | |
| "selected_on_primary_validation_objective": false | |
| } | |
| ], | |
| "nested_cv_proposed_model_summary": [ | |
| { | |
| "model": "PACE_nested_simplex_proposed", | |
| "model_type": "pace_simplex_proposed", | |
| "model_family": "PACE", | |
| "outer_folds": 3, | |
| "mean_average_precision": 0.5925735951377111, | |
| "sd_average_precision": 0.04039160600707779, | |
| "mean_roc_auc": 0.7874683918404233, | |
| "sd_roc_auc": 0.03521949945958485, | |
| "mean_brier": 0.1530745166497782, | |
| "sd_brier": 0.01693497587921028, | |
| "mean_balanced_accuracy": 0.6850699702682824, | |
| "mean_weighted_error_cost": 180.33333333333334, | |
| "sd_weighted_error_cost": 16.258331197676267, | |
| "mean_cost_per_case": 0.5635416666666667, | |
| "mean_selected_threshold": 0.18364686054410828, | |
| "sd_selected_threshold": 0.03541113054853995, | |
| "all_expected_outer_folds_present": true | |
| } | |
| ], | |
| "nested_cv_status": "COMPLETED", | |
| "selected_validation_model": "LightGBM_Unweighted", | |
| "data_and_claim_boundary": "Only the primary Bangladesh retrospective target is directly relevant to this model. Other dataset results are separate benchmarks and must not be pooled with these metrics." | |
| } |