soft-decider-421m: RLCD fine-tune of laya on typed-decisions with holdout calibration
ad67f6b verified | { | |
| "benchmark": "LocalLLaMA/typed-decisions test split", | |
| "published": { | |
| "TypeSafe Jev 1.13.0 (published)": { | |
| "acc": 0.727, | |
| "soft": 0.58, | |
| "brier": 0.148, | |
| "ece": 0.144, | |
| "mae": 0.391, | |
| "ms": 710 | |
| }, | |
| "laya-typed-decisions (published)": { | |
| "acc": 0.766, | |
| "soft": 0.471, | |
| "brier": 0.062, | |
| "ece": 0.213, | |
| "mae": 0.242, | |
| "ms": null | |
| }, | |
| "teacher self-agreement ceiling": { | |
| "acc": 0.735, | |
| "soft": null, | |
| "brier": null, | |
| "ece": null, | |
| "mae": null, | |
| "ms": null | |
| }, | |
| "majority class": { | |
| "acc": 0.461, | |
| "soft": null, | |
| "brier": null, | |
| "ece": null, | |
| "mae": null, | |
| "ms": null | |
| } | |
| }, | |
| "measured": { | |
| "ours": { | |
| "n_decisions": 1995, | |
| "accuracy": 0.7609, | |
| "soft_accuracy": 0.5404, | |
| "brier": 0.2014, | |
| "kl": 0.3524, | |
| "tv": 0.2798, | |
| "ece": 0.1382, | |
| "score_mae": 0.2368, | |
| "within_1_level": 0.9925, | |
| "auroc_conf": 0.7862, | |
| "aurc": 0.0931, | |
| "latency_p50_ms": 49.2, | |
| "latency_p95_ms": 61.1, | |
| "per_workflow": { | |
| "agent_trace_observability": 0.7434, | |
| "customer_service": 0.762, | |
| "invoice_processing": 0.786, | |
| "security_incidents": 0.752 | |
| }, | |
| "per_type": { | |
| "choice": 0.7441, | |
| "noul": 0.8581, | |
| "score": 0.7005 | |
| } | |
| }, | |
| "laya-td": { | |
| "n_decisions": 1995, | |
| "accuracy": 0.7664, | |
| "soft_accuracy": 0.4999, | |
| "brier": 0.2134, | |
| "kl": 0.3839, | |
| "tv": 0.3126, | |
| "ece": 0.2135, | |
| "score_mae": 0.2429, | |
| "within_1_level": 0.995, | |
| "auroc_conf": 0.7569, | |
| "aurc": 0.099, | |
| "auto_decidable_5pct": 0.3779, | |
| "choice_flip_rate": 0.0652, | |
| "latency_p50_ms": 52.6, | |
| "latency_p95_ms": 66.5, | |
| "per_workflow": { | |
| "agent_trace_observability": 0.7293, | |
| "customer_service": 0.762, | |
| "invoice_processing": 0.802, | |
| "security_incidents": 0.772 | |
| }, | |
| "per_type": { | |
| "choice": 0.7291, | |
| "noul": 0.8564, | |
| "score": 0.7268 | |
| } | |
| }, | |
| "dir:d:\\QoderCN\\jev_finetune\\output\\model_soft": { | |
| "n_decisions": 1995, | |
| "accuracy": 0.7739, | |
| "soft_accuracy": 0.5511, | |
| "brier": 0.1922, | |
| "kl": 0.3395, | |
| "tv": 0.2696, | |
| "ece": 0.1407, | |
| "score_mae": 0.2212, | |
| "within_1_level": 0.99, | |
| "auroc_conf": 0.7825, | |
| "aurc": 0.0872, | |
| "auto_decidable_5pct": 0.3739, | |
| "choice_flip_rate": 0.0769, | |
| "latency_p50_ms": 53.6, | |
| "latency_p95_ms": 68.2, | |
| "per_workflow": { | |
| "agent_trace_observability": 0.7293, | |
| "customer_service": 0.788, | |
| "invoice_processing": 0.814, | |
| "security_incidents": 0.764 | |
| }, | |
| "per_type": { | |
| "choice": 0.7475, | |
| "noul": 0.8514, | |
| "score": 0.7356 | |
| } | |
| }, | |
| "dir:d:\\QoderCN\\jev_finetune\\output\\model_aug": { | |
| "n_decisions": 1995, | |
| "accuracy": 0.7444, | |
| "soft_accuracy": 0.5225, | |
| "brier": 0.2179, | |
| "kl": 0.3868, | |
| "tv": 0.299, | |
| "ece": 0.1404, | |
| "score_mae": 0.2857, | |
| "within_1_level": 0.9862, | |
| "auroc_conf": 0.7683, | |
| "aurc": 0.1079, | |
| "auto_decidable_5pct": 0.3208, | |
| "choice_flip_rate": 0.0585, | |
| "latency_p50_ms": 52.8, | |
| "latency_p95_ms": 64.8, | |
| "per_workflow": { | |
| "agent_trace_observability": 0.7374, | |
| "customer_service": 0.738, | |
| "invoice_processing": 0.752, | |
| "security_incidents": 0.75 | |
| }, | |
| "per_type": { | |
| "choice": 0.7207, | |
| "noul": 0.8381, | |
| "score": 0.6917 | |
| } | |
| }, | |
| "open-jev-deberta": { | |
| "n_decisions": 2000, | |
| "accuracy": 0.435, | |
| "soft_accuracy": 0.3539, | |
| "brier": 0.4513, | |
| "kl": 0.8118, | |
| "tv": 0.505, | |
| "ece": 0.0775, | |
| "score_mae": 0.7213, | |
| "within_1_level": 0.7388, | |
| "auroc_conf": 0.6413, | |
| "aurc": 0.4539, | |
| "auto_decidable_5pct": 0.002, | |
| "choice_flip_rate": 0.1917, | |
| "latency_p50_ms": 163.8, | |
| "latency_p95_ms": 178.6, | |
| "per_workflow": { | |
| "agent_trace_observability": 0.394, | |
| "customer_service": 0.512, | |
| "invoice_processing": 0.394, | |
| "security_incidents": 0.44 | |
| }, | |
| "per_type": { | |
| "choice": 0.4483, | |
| "noul": 0.6133, | |
| "score": 0.2913 | |
| } | |
| } | |
| } | |
| } |