| { |
| "validation": { |
| "count": 143, |
| "accuracy": 0.7412587412587412, |
| "macro_f1": 0.7247123725991762, |
| "classification_report": { |
| "EDUCATION": { |
| "precision": 1.0, |
| "recall": 0.42424242424242425, |
| "f1-score": 0.5957446808510638, |
| "support": 33.0 |
| }, |
| "FINANCE": { |
| "precision": 0.75, |
| "recall": 0.8333333333333334, |
| "f1-score": 0.7894736842105263, |
| "support": 36.0 |
| }, |
| "HEALTHCARE": { |
| "precision": 0.5609756097560976, |
| "recall": 0.8518518518518519, |
| "f1-score": 0.6764705882352942, |
| "support": 27.0 |
| }, |
| "SOFTWARE": { |
| "precision": 0.75, |
| "recall": 0.6428571428571429, |
| "f1-score": 0.6923076923076923, |
| "support": 14.0 |
| }, |
| "TRADES": { |
| "precision": 0.8333333333333334, |
| "recall": 0.9090909090909091, |
| "f1-score": 0.8695652173913043, |
| "support": 33.0 |
| }, |
| "accuracy": 0.7412587412587412, |
| "macro avg": { |
| "precision": 0.7788617886178862, |
| "recall": 0.7322751322751323, |
| "f1-score": 0.7247123725991762, |
| "support": 143.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7912331570868157, |
| "recall": 0.7412587412587412, |
| "f1-score": 0.7324006492883758, |
| "support": 143.0 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 14, |
| 5, |
| 11, |
| 0, |
| 3 |
| ], |
| [ |
| 0, |
| 30, |
| 4, |
| 1, |
| 1 |
| ], |
| [ |
| 0, |
| 1, |
| 23, |
| 1, |
| 2 |
| ], |
| [ |
| 0, |
| 3, |
| 2, |
| 9, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 1, |
| 1, |
| 30 |
| ] |
| ], |
| "log_loss": 0.8953816887395478, |
| "routing": { |
| "auto_accept_count": 66, |
| "auto_accept_coverage": 0.46153846153846156, |
| "auto_accept_accuracy": 0.9545454545454546, |
| "escalate_count": 71 |
| } |
| }, |
| "test": { |
| "count": 173, |
| "accuracy": 0.8034682080924855, |
| "macro_f1": 0.7854200542005421, |
| "classification_report": { |
| "EDUCATION": { |
| "precision": 0.9523809523809523, |
| "recall": 0.7407407407407407, |
| "f1-score": 0.8333333333333334, |
| "support": 27.0 |
| }, |
| "FINANCE": { |
| "precision": 0.8222222222222222, |
| "recall": 0.8222222222222222, |
| "f1-score": 0.8222222222222222, |
| "support": 45.0 |
| }, |
| "HEALTHCARE": { |
| "precision": 0.7636363636363637, |
| "recall": 0.84, |
| "f1-score": 0.8, |
| "support": 50.0 |
| }, |
| "SOFTWARE": { |
| "precision": 0.875, |
| "recall": 0.5384615384615384, |
| "f1-score": 0.6666666666666666, |
| "support": 13.0 |
| }, |
| "TRADES": { |
| "precision": 0.75, |
| "recall": 0.868421052631579, |
| "f1-score": 0.8048780487804879, |
| "support": 38.0 |
| }, |
| "accuracy": 0.8034682080924855, |
| "macro avg": { |
| "precision": 0.8326479076479076, |
| "recall": 0.7619691108112161, |
| "f1-score": 0.7854200542005421, |
| "support": 173.0 |
| }, |
| "weighted avg": { |
| "precision": 0.8137058028676527, |
| "recall": 0.8034682080924855, |
| "f1-score": 0.8020348700596832, |
| "support": 173.0 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 20, |
| 1, |
| 3, |
| 0, |
| 3 |
| ], |
| [ |
| 0, |
| 37, |
| 4, |
| 1, |
| 3 |
| ], |
| [ |
| 1, |
| 4, |
| 42, |
| 0, |
| 3 |
| ], |
| [ |
| 0, |
| 2, |
| 2, |
| 7, |
| 2 |
| ], |
| [ |
| 0, |
| 1, |
| 4, |
| 0, |
| 33 |
| ] |
| ], |
| "log_loss": 0.789789166612511, |
| "routing": { |
| "auto_accept_count": 77, |
| "auto_accept_coverage": 0.44508670520231214, |
| "auto_accept_accuracy": 0.987012987012987, |
| "escalate_count": 89 |
| } |
| }, |
| "export_parity": { |
| "samples": 319, |
| "maximum_probability_error": 2.7755575615628914e-16, |
| "tolerance": 1e-10 |
| }, |
| "classes": [ |
| "EDUCATION", |
| "FINANCE", |
| "HEALTHCARE", |
| "SOFTWARE", |
| "TRADES" |
| ], |
| "threshold_selection": { |
| "split": "validation", |
| "minimum_accepted_count": 20, |
| "target_empirical_accuracy": 0.95, |
| "selected": { |
| "auto_accept": 0.52, |
| "escalate": 0.5 |
| }, |
| "sweep": [ |
| { |
| "threshold": 0.5, |
| "accepted_count": 72, |
| "coverage": 0.5034965034965035, |
| "accepted_accuracy": 0.9444444444444444 |
| }, |
| { |
| "threshold": 0.51, |
| "accepted_count": 67, |
| "coverage": 0.46853146853146854, |
| "accepted_accuracy": 0.9402985074626866 |
| }, |
| { |
| "threshold": 0.52, |
| "accepted_count": 66, |
| "coverage": 0.46153846153846156, |
| "accepted_accuracy": 0.9545454545454546 |
| }, |
| { |
| "threshold": 0.53, |
| "accepted_count": 62, |
| "coverage": 0.43356643356643354, |
| "accepted_accuracy": 0.967741935483871 |
| }, |
| { |
| "threshold": 0.54, |
| "accepted_count": 60, |
| "coverage": 0.4195804195804196, |
| "accepted_accuracy": 0.9666666666666667 |
| }, |
| { |
| "threshold": 0.55, |
| "accepted_count": 60, |
| "coverage": 0.4195804195804196, |
| "accepted_accuracy": 0.9666666666666667 |
| }, |
| { |
| "threshold": 0.56, |
| "accepted_count": 53, |
| "coverage": 0.3706293706293706, |
| "accepted_accuracy": 0.9622641509433962 |
| }, |
| { |
| "threshold": 0.57, |
| "accepted_count": 51, |
| "coverage": 0.35664335664335667, |
| "accepted_accuracy": 0.9607843137254902 |
| }, |
| { |
| "threshold": 0.58, |
| "accepted_count": 49, |
| "coverage": 0.34265734265734266, |
| "accepted_accuracy": 0.9795918367346939 |
| }, |
| { |
| "threshold": 0.59, |
| "accepted_count": 47, |
| "coverage": 0.32867132867132864, |
| "accepted_accuracy": 0.9787234042553191 |
| }, |
| { |
| "threshold": 0.6, |
| "accepted_count": 44, |
| "coverage": 0.3076923076923077, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.61, |
| "accepted_count": 43, |
| "coverage": 0.3006993006993007, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.62, |
| "accepted_count": 40, |
| "coverage": 0.27972027972027974, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.63, |
| "accepted_count": 39, |
| "coverage": 0.2727272727272727, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.64, |
| "accepted_count": 39, |
| "coverage": 0.2727272727272727, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.65, |
| "accepted_count": 37, |
| "coverage": 0.25874125874125875, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.66, |
| "accepted_count": 36, |
| "coverage": 0.2517482517482518, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.67, |
| "accepted_count": 34, |
| "coverage": 0.23776223776223776, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.68, |
| "accepted_count": 33, |
| "coverage": 0.23076923076923078, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.69, |
| "accepted_count": 31, |
| "coverage": 0.21678321678321677, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "accepted_count": 29, |
| "coverage": 0.20279720279720279, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.71, |
| "accepted_count": 28, |
| "coverage": 0.1958041958041958, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.72, |
| "accepted_count": 25, |
| "coverage": 0.17482517482517482, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.73, |
| "accepted_count": 23, |
| "coverage": 0.16083916083916083, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.74, |
| "accepted_count": 22, |
| "coverage": 0.15384615384615385, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.75, |
| "accepted_count": 19, |
| "coverage": 0.13286713286713286, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.76, |
| "accepted_count": 18, |
| "coverage": 0.1258741258741259, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.77, |
| "accepted_count": 12, |
| "coverage": 0.08391608391608392, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.78, |
| "accepted_count": 11, |
| "coverage": 0.07692307692307693, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.79, |
| "accepted_count": 11, |
| "coverage": 0.07692307692307693, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.8, |
| "accepted_count": 10, |
| "coverage": 0.06993006993006994, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.81, |
| "accepted_count": 9, |
| "coverage": 0.06293706293706294, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.82, |
| "accepted_count": 9, |
| "coverage": 0.06293706293706294, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.83, |
| "accepted_count": 9, |
| "coverage": 0.06293706293706294, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.84, |
| "accepted_count": 7, |
| "coverage": 0.04895104895104895, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.85, |
| "accepted_count": 7, |
| "coverage": 0.04895104895104895, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.86, |
| "accepted_count": 6, |
| "coverage": 0.04195804195804196, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.87, |
| "accepted_count": 3, |
| "coverage": 0.02097902097902098, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.88, |
| "accepted_count": 3, |
| "coverage": 0.02097902097902098, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.89, |
| "accepted_count": 2, |
| "coverage": 0.013986013986013986, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.9, |
| "accepted_count": 1, |
| "coverage": 0.006993006993006993, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.91, |
| "accepted_count": 1, |
| "coverage": 0.006993006993006993, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.92, |
| "accepted_count": 1, |
| "coverage": 0.006993006993006993, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.93, |
| "accepted_count": 1, |
| "coverage": 0.006993006993006993, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.94, |
| "accepted_count": 1, |
| "coverage": 0.006993006993006993, |
| "accepted_accuracy": 1.0 |
| }, |
| { |
| "threshold": 0.95, |
| "accepted_count": 0, |
| "coverage": 0.0, |
| "accepted_accuracy": null |
| }, |
| { |
| "threshold": 0.96, |
| "accepted_count": 0, |
| "coverage": 0.0, |
| "accepted_accuracy": null |
| }, |
| { |
| "threshold": 0.97, |
| "accepted_count": 0, |
| "coverage": 0.0, |
| "accepted_accuracy": null |
| }, |
| { |
| "threshold": 0.98, |
| "accepted_count": 0, |
| "coverage": 0.0, |
| "accepted_accuracy": null |
| }, |
| { |
| "threshold": 0.99, |
| "accepted_count": 0, |
| "coverage": 0.0, |
| "accepted_accuracy": null |
| } |
| ], |
| "limitation": "Empirical validation accuracy is not a future accuracy guarantee; probabilities are not calibrated." |
| } |
| } |
