eval_wf.json

{
 "run": "runs/nc_v3_tap20_wf",
 "mode": "native",
 "max_state": 4096,
 "limit": 500,
 "eval": {
  "wf_heldout_choice": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 0.019871275152449026,
    "brier": 0.010466870652060482,
    "jsd": 0.005765777761771096,
    "acc": 0.994,
    "acc_k": 0.9937238493723849,
    "ece": 0.0063112200498580615,
    "sel_acc80": 1.0,
    "conf_wrong": 0.0,
    "coverage": 0.956,
    "false_abstain": 0.0,
    "null_recall": 1.0,
    "auroc_null": 1.0,
    "auroc_conf": 1.0
   }
  },
  "wf_heldout_noul": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 0.853929875950949,
    "brier": 0.4878031580677193,
    "jsd": 0.17909886291906896,
    "acc": 0.682,
    "acc_k": 0.682,
    "ece": 0.1986952285766601,
    "sel_acc80": 0.715,
    "coverage": 1.0,
    "false_abstain": 0.0,
    "null_recall": "n/a"
   }
  },
  "wf_heldout_score": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 2.068675406193429,
    "brier": 0.7677929317097167,
    "jsd": 0.2869982566550366,
    "acc": 0.514,
    "acc_k": 0.514,
    "ece": 0.35427253764867783,
    "sel_acc80": 0.5725,
    "coverage": 1.0,
    "false_abstain": 0.0,
    "null_recall": "n/a"
   }
  },
  "wf_heldout_style": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 0.2222467604358654,
    "brier": 0.1323241858578231,
    "jsd": 0.06029804682074418,
    "acc": 0.904,
    "acc_k": 0.901840490797546,
    "ece": 0.019670174241066027,
    "sel_acc80": 0.975,
    "conf_wrong": 0.0,
    "coverage": 0.978,
    "false_abstain": 0.0,
    "null_recall": 1.0,
    "auroc_null": 1.0,
    "auroc_conf": 1.0
   }
  },
  "wf_rubric_flip": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 0.7009927043024266,
    "brier": 0.40828858133650053,
    "jsd": 0.15320314054370474,
    "acc": 0.736,
    "acc_k": 0.736,
    "ece": 0.1480658851861954,
    "sel_acc80": 0.7825,
    "coverage": 1.0,
    "false_abstain": 0.0,
    "null_recall": "n/a"
   },
   "flip": {
    "n_pairs": 250,
    "flip_rate": 0.504,
    "both_correct": 0.488,
    "acc": 0.736
   }
  },
  "wf_rubric_shuffled": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 4.60017100450565,
    "brier": 1.0415848067241245,
    "jsd": 0.3751870274955275,
    "acc": 0.386,
    "acc_k": 0.386,
    "ece": 0.5058854785561562,
    "sel_acc80": 0.395,
    "coverage": 1.0,
    "false_abstain": 0.0,
    "null_recall": "n/a"
   }
  },
  "cua_s1_forms_demo": {
   "n": 196,
   "suffix_overflow": 0,
   "raw": {
    "nll": 0.00011844338656585847,
    "brier": 3.8039107351542767e-06,
    "jsd": 4.095999434756056e-05,
    "acc": 1.0,
    "acc_k": 1.0,
    "ece": 0.00011747923432559038,
    "sel_acc80": 1.0,
    "coverage": 1.0,
    "false_abstain": 0.0,
    "null_recall": "n/a"
   }
  },
  "cua_s1_forms_test": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 0.012289381821878892,
    "brier": 0.004094414644142791,
    "jsd": 0.001583947825507208,
    "acc": 0.998,
    "acc_k": 0.998,
    "ece": 0.002032796382904006,
    "sel_acc80": 1.0,
    "coverage": 1.0,
    "false_abstain": 0.0,
    "null_recall": "n/a"
   }
  },
  "jev4b_adversarial": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 0.3490933191998588,
    "brier": 0.09980970638364663,
    "jsd": 0.040403990439927887,
    "acc": 0.94,
    "acc_k": 0.94,
    "ece": 0.04506945902109154,
    "sel_acc80": 0.965,
    "coverage": 1.0,
    "false_abstain": 0.0,
    "null_recall": "n/a"
   }
  },
  "jevlogs_triage": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 0.3044197695283073,
    "brier": 0.17586170974258888,
    "jsd": 0.0972293910378737,
    "acc": 0.832,
    "acc_k": 0.832,
    "ece": 0.23506193399429315,
    "sel_acc80": 1.0,
    "coverage": 1.0,
    "false_abstain": 0.0,
    "null_recall": "n/a"
   }
  },
  "mind2web_choice": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 1.7049515437075962,
    "brier": 0.7232633764663069,
    "jsd": 0.3240744768308897,
    "acc": 0.438,
    "acc_k": 0.438,
    "ece": 0.21502542391419413,
    "sel_acc80": 0.5025,
    "coverage": 1.0,
    "false_abstain": 0.0,
    "null_recall": "n/a"
   }
  },
  "pagerduty_trigger": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 0.6816042798599857,
    "brier": 0.3284166395830621,
    "jsd": 0.14193714639027197,
    "acc": 0.762,
    "acc_k": 0.762,
    "ece": 0.10664146476984024,
    "sel_acc80": 0.865,
    "coverage": 1.0,
    "false_abstain": 0.0,
    "null_recall": "n/a"
   }
  },
  "systemone_lite_hard": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 0.2668189883849799,
    "brier": 0.15873402095268352,
    "jsd": 0.06530097270690566,
    "acc": 0.888,
    "acc_k": 0.888,
    "ece": 0.03896239256858824,
    "sel_acc80": 0.9625,
    "coverage": 1.0,
    "false_abstain": 0.0,
    "null_recall": "n/a"
   }
  },
  "tree_choice_cap": {
   "n": 500,
   "suffix_overflow": 125,
   "raw": {
    "nll": 2.243535988875024,
    "brier": 0.6146624018620521,
    "jsd": 0.34488693403770676,
    "acc": 0.494,
    "acc_k": 0.538,
    "ece": 0.07969999838478839,
    "sel_acc80": 0.6175,
    "coverage": 0.872,
    "false_abstain": 0.128,
    "null_recall": "n/a"
   }
  },
  "typed_decisions_test": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 2.4008272248719806,
    "brier": 0.5420563405144715,
    "jsd": 0.2209605208544786,
    "acc": 0.276,
    "acc_k": 0.276,
    "ece": 0.5269120357036591,
    "sel_acc80": 0.2925,
    "coverage": 1.0,
    "false_abstain": 0.0,
    "null_recall": "n/a"
   }
  },
  "typed_decisions_train": {
   "n": 500,
   "suffix_overflow": 0,
   "raw": {
    "nll": 2.5297219974097365,
    "brier": 0.5571985794974775,
    "jsd": 0.2336201678966866,
    "acc": 0.288,
    "acc_k": 0.288,
    "ece": 0.511909948706627,
    "sel_acc80": 0.2875,
    "coverage": 1.0,
    "false_abstain": 0.0,
    "null_recall": "n/a"
   }
  }
 }
}