{
  "benchmark": "LocalLLaMA/typed-decisions test split",
  "published": {
    "TypeSafe Jev 1.13.0 (published)": {
      "acc": 0.727,
      "soft": 0.58,
      "brier": 0.148,
      "ece": 0.144,
      "mae": 0.391,
      "ms": 710
    },
    "laya-typed-decisions (published)": {
      "acc": 0.766,
      "soft": 0.471,
      "brier": 0.062,
      "ece": 0.213,
      "mae": 0.242,
      "ms": null
    },
    "teacher self-agreement ceiling": {
      "acc": 0.735,
      "soft": null,
      "brier": null,
      "ece": null,
      "mae": null,
      "ms": null
    },
    "majority class": {
      "acc": 0.461,
      "soft": null,
      "brier": null,
      "ece": null,
      "mae": null,
      "ms": null
    }
  },
  "measured": {
    "ours": {
      "n_decisions": 1995,
      "accuracy": 0.7609,
      "soft_accuracy": 0.5404,
      "brier": 0.2014,
      "kl": 0.3524,
      "tv": 0.2798,
      "ece": 0.1382,
      "score_mae": 0.2368,
      "within_1_level": 0.9925,
      "auroc_conf": 0.7862,
      "aurc": 0.0931,
      "latency_p50_ms": 49.2,
      "latency_p95_ms": 61.1,
      "per_workflow": {
        "agent_trace_observability": 0.7434,
        "customer_service": 0.762,
        "invoice_processing": 0.786,
        "security_incidents": 0.752
      },
      "per_type": {
        "choice": 0.7441,
        "noul": 0.8581,
        "score": 0.7005
      }
    },
    "laya-td": {
      "n_decisions": 1995,
      "accuracy": 0.7664,
      "soft_accuracy": 0.4999,
      "brier": 0.2134,
      "kl": 0.3839,
      "tv": 0.3126,
      "ece": 0.2135,
      "score_mae": 0.2429,
      "within_1_level": 0.995,
      "auroc_conf": 0.7569,
      "aurc": 0.099,
      "auto_decidable_5pct": 0.3779,
      "choice_flip_rate": 0.0652,
      "latency_p50_ms": 52.6,
      "latency_p95_ms": 66.5,
      "per_workflow": {
        "agent_trace_observability": 0.7293,
        "customer_service": 0.762,
        "invoice_processing": 0.802,
        "security_incidents": 0.772
      },
      "per_type": {
        "choice": 0.7291,
        "noul": 0.8564,
        "score": 0.7268
      }
    },
    "dir:d:\\QoderCN\\jev_finetune\\output\\model_soft": {
      "n_decisions": 1995,
      "accuracy": 0.7739,
      "soft_accuracy": 0.5511,
      "brier": 0.1922,
      "kl": 0.3395,
      "tv": 0.2696,
      "ece": 0.1407,
      "score_mae": 0.2212,
      "within_1_level": 0.99,
      "auroc_conf": 0.7825,
      "aurc": 0.0872,
      "auto_decidable_5pct": 0.3739,
      "choice_flip_rate": 0.0769,
      "latency_p50_ms": 53.6,
      "latency_p95_ms": 68.2,
      "per_workflow": {
        "agent_trace_observability": 0.7293,
        "customer_service": 0.788,
        "invoice_processing": 0.814,
        "security_incidents": 0.764
      },
      "per_type": {
        "choice": 0.7475,
        "noul": 0.8514,
        "score": 0.7356
      }
    },
    "dir:d:\\QoderCN\\jev_finetune\\output\\model_aug": {
      "n_decisions": 1995,
      "accuracy": 0.7444,
      "soft_accuracy": 0.5225,
      "brier": 0.2179,
      "kl": 0.3868,
      "tv": 0.299,
      "ece": 0.1404,
      "score_mae": 0.2857,
      "within_1_level": 0.9862,
      "auroc_conf": 0.7683,
      "aurc": 0.1079,
      "auto_decidable_5pct": 0.3208,
      "choice_flip_rate": 0.0585,
      "latency_p50_ms": 52.8,
      "latency_p95_ms": 64.8,
      "per_workflow": {
        "agent_trace_observability": 0.7374,
        "customer_service": 0.738,
        "invoice_processing": 0.752,
        "security_incidents": 0.75
      },
      "per_type": {
        "choice": 0.7207,
        "noul": 0.8381,
        "score": 0.6917
      }
    },
    "open-jev-deberta": {
      "n_decisions": 2000,
      "accuracy": 0.435,
      "soft_accuracy": 0.3539,
      "brier": 0.4513,
      "kl": 0.8118,
      "tv": 0.505,
      "ece": 0.0775,
      "score_mae": 0.7213,
      "within_1_level": 0.7388,
      "auroc_conf": 0.6413,
      "aurc": 0.4539,
      "auto_decidable_5pct": 0.002,
      "choice_flip_rate": 0.1917,
      "latency_p50_ms": 163.8,
      "latency_p95_ms": 178.6,
      "per_workflow": {
        "agent_trace_observability": 0.394,
        "customer_service": 0.512,
        "invoice_processing": 0.394,
        "security_incidents": 0.44
      },
      "per_type": {
        "choice": 0.4483,
        "noul": 0.6133,
        "score": 0.2913
      }
    }
  }
}