{
  "protocol": {
    "created_utc": "2026-09-30T15:17:02.035817+00:00",
    "accuracy_n": 2496,
    "groups": {
      "heldout_tasks": 300,
      "newsfin_long": 56,
      "newsfin_short": 140,
      "typed_decisions": 2000
    },
    "local_omni_train_or_validation_overlap": 0,
    "accuracy_manifest_sha256": "77c421d0c50c76eb89016a4d77b4fde198d58c0bb0ca0d8413af171fdc183d24",
    "speed_workloads_sha256": "04f2d48d8bd6c6b84563b19a55f69153e8c67c513ceb5a727918f34aad78933d",
    "laya_revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851",
    "laya_runtime": "0.3.22",
    "laya_fast_path": "Not enabled: optional TileLang is not installed. Stock public runtime measured.",
    "timing": "Public Python call, tokenization included, CUDA synchronized before/after. No HTTP/network overhead.",
    "normalization": "Identical normalized question text/options and state bytes; noul criteria omitted equally per the existing project protocol.",
    "calibration": "Existing deployed temperatures retained; no fitting, training, or threshold selection on this test set.",
    "typed_decisions": "All 400 test cases / 2000 decisions; teacher-distribution metrics and hard labels reported separately.",
    "laya_td": "Task-specialized checkpoint trained upstream on the benchmark training split; not a zero-shot baseline.",
    "limitations": "Public pretraining overlap unknown; desktop shares GPU; p95 from 20 calls is exploratory, not an SLA."
  },
  "accuracy": {
    "laya": {
      "newsfin_short": {
        "n": 140,
        "accuracy": 0.6071428571428571,
        "accuracy_ci95": [
          0.5285714285714286,
          0.6857142857142857
        ],
        "nll": 1.4223718041679008,
        "brier_onehot": 0.5707742968520072,
        "ece15": 0.16604753199968444,
        "mean_confidence": 0.7438418601631568,
        "uniform_accuracy": 0.2794763973450739,
        "confidence_thresholds": {
          "0.65": {
            "n": 91,
            "coverage": 0.65,
            "accuracy": 0.7032967032967034
          },
          "0.8": {
            "n": 71,
            "coverage": 0.5071428571428571,
            "accuracy": 0.7183098591549296
          },
          "0.9": {
            "n": 43,
            "coverage": 0.30714285714285716,
            "accuracy": 0.7441860465116279
          }
        },
        "reliability": [
          {
            "n": 6,
            "confidence": 0.2996381148700478,
            "accuracy": 0.3333333333333333
          },
          {
            "n": 4,
            "confidence": 0.37458102430107054,
            "accuracy": 0.5
          },
          {
            "n": 10,
            "confidence": 0.43827573330895736,
            "accuracy": 0.2
          },
          {
            "n": 11,
            "confidence": 0.5135863804562275,
            "accuracy": 0.6363636363636364
          },
          {
            "n": 7,
            "confidence": 0.5655142857142857,
            "accuracy": 0.42857142857142855
          },
          {
            "n": 13,
            "confidence": 0.6327408621914118,
            "accuracy": 0.46153846153846156
          },
          {
            "n": 9,
            "confidence": 0.6960221081986655,
            "accuracy": 0.6666666666666666
          },
          {
            "n": 9,
            "confidence": 0.7785007062507501,
            "accuracy": 0.6666666666666666
          },
          {
            "n": 19,
            "confidence": 0.8315686740694261,
            "accuracy": 0.7368421052631579
          },
          {
            "n": 19,
            "confidence": 0.8988784093221,
            "accuracy": 0.6842105263157895
          },
          {
            "n": 33,
            "confidence": 0.984123302889007,
            "accuracy": 0.7272727272727273
          }
        ]
      },
      "newsfin_long": {
        "n": 56,
        "accuracy": 0.6607142857142857,
        "accuracy_ci95": [
          0.5357142857142857,
          0.7857142857142857
        ],
        "nll": 1.2032614087003177,
        "brier_onehot": 0.5529698397175744,
        "ece15": 0.19426640345296486,
        "mean_confidence": 0.784147753253207,
        "uniform_accuracy": 0.24032936018230133,
        "confidence_thresholds": {
          "0.65": {
            "n": 41,
            "coverage": 0.7321428571428571,
            "accuracy": 0.6829268292682927
          },
          "0.8": {
            "n": 34,
            "coverage": 0.6071428571428571,
            "accuracy": 0.6764705882352942
          },
          "0.9": {
            "n": 21,
            "coverage": 0.375,
            "accuracy": 0.7142857142857143
          }
        },
        "reliability": [
          {
            "n": 3,
            "confidence": 0.36054520785411875,
            "accuracy": 0.6666666666666666
          },
          {
            "n": 2,
            "confidence": 0.42155,
            "accuracy": 0.5
          },
          {
            "n": 3,
            "confidence": 0.496984198359848,
            "accuracy": 0.3333333333333333
          },
          {
            "n": 2,
            "confidence": 0.5759784578457847,
            "accuracy": 0.5
          },
          {
            "n": 6,
            "confidence": 0.6315058727460587,
            "accuracy": 0.6666666666666666
          },
          {
            "n": 4,
            "confidence": 0.690932906709329,
            "accuracy": 0.75
          },
          {
            "n": 2,
            "confidence": 0.7695876537653765,
            "accuracy": 1.0
          },
          {
            "n": 11,
            "confidence": 0.8321180001198024,
            "accuracy": 0.5454545454545454
          },
          {
            "n": 6,
            "confidence": 0.9037815511951489,
            "accuracy": 0.6666666666666666
          },
          {
            "n": 17,
            "confidence": 0.9809823275595868,
            "accuracy": 0.7647058823529411
          }
        ]
      },
      "heldout_tasks": {
        "n": 300,
        "accuracy": 0.66,
        "accuracy_ci95": [
          0.6066666666666667,
          0.7133333333333334
        ],
        "nll": 1.5088246130436413,
        "brier_onehot": 0.49109099242960164,
        "ece15": 0.1363256368468856,
        "mean_confidence": 0.7706782853152053,
        "uniform_accuracy": 0.3491236190667541,
        "confidence_thresholds": {
          "0.65": {
            "n": 195,
            "coverage": 0.65,
            "accuracy": 0.7897435897435897
          },
          "0.8": {
            "n": 160,
            "coverage": 0.5333333333333333,
            "accuracy": 0.81875
          },
          "0.9": {
            "n": 136,
            "coverage": 0.4533333333333333,
            "accuracy": 0.8235294117647058
          }
        },
        "reliability": [
          {
            "n": 1,
            "confidence": 0.3066,
            "accuracy": 0.0
          },
          {
            "n": 15,
            "confidence": 0.381084709529047,
            "accuracy": 0.3333333333333333
          },
          {
            "n": 26,
            "confidence": 0.4301559573447903,
            "accuracy": 0.23076923076923078
          },
          {
            "n": 26,
            "confidence": 0.5058806642403066,
            "accuracy": 0.6538461538461539
          },
          {
            "n": 20,
            "confidence": 0.5698708586837585,
            "accuracy": 0.45
          },
          {
            "n": 20,
            "confidence": 0.6283789555242396,
            "accuracy": 0.4
          },
          {
            "n": 15,
            "confidence": 0.7039680905242808,
            "accuracy": 0.6
          },
          {
            "n": 17,
            "confidence": 0.7685221145643976,
            "accuracy": 0.7647058823529411
          },
          {
            "n": 16,
            "confidence": 0.83745649416744,
            "accuracy": 0.6875
          },
          {
            "n": 30,
            "confidence": 0.9053593478879268,
            "accuracy": 0.8
          },
          {
            "n": 114,
            "confidence": 0.988545481205944,
            "accuracy": 0.8421052631578947
          }
        ]
      },
      "typed_decisions": {
        "n": 2000,
        "accuracy": 0.364,
        "accuracy_ci95": [
          0.339,
          0.3885
        ],
        "nll": 1.340352049822895,
        "brier_onehot": 0.7689268871867527,
        "ece15": 0.21537093573642535,
        "mean_confidence": 0.5793709357364253,
        "uniform_accuracy": 0.3175,
        "confidence_thresholds": {
          "0.65": {
            "n": 608,
            "coverage": 0.304,
            "accuracy": 0.45230263157894735
          },
          "0.8": {
            "n": 314,
            "coverage": 0.157,
            "accuracy": 0.4745222929936306
          },
          "0.9": {
            "n": 102,
            "coverage": 0.051,
            "accuracy": 0.4803921568627451
          }
        },
        "reliability": [
          {
            "n": 13,
            "confidence": 0.25357506365936605,
            "accuracy": 0.23076923076923078
          },
          {
            "n": 122,
            "confidence": 0.30950061571517834,
            "accuracy": 0.23770491803278687
          },
          {
            "n": 228,
            "confidence": 0.3670941177030421,
            "accuracy": 0.2719298245614035
          },
          {
            "n": 233,
            "confidence": 0.4363279138876397,
            "accuracy": 0.2875536480686695
          },
          {
            "n": 256,
            "confidence": 0.5028284181113388,
            "accuracy": 0.30859375
          },
          {
            "n": 337,
            "confidence": 0.5673213889715166,
            "accuracy": 0.3857566765578635
          },
          {
            "n": 256,
            "confidence": 0.6312426961373134,
            "accuracy": 0.3828125
          },
          {
            "n": 131,
            "confidence": 0.6972084159331299,
            "accuracy": 0.42748091603053434
          },
          {
            "n": 110,
            "confidence": 0.765639844893953,
            "accuracy": 0.5
          },
          {
            "n": 131,
            "confidence": 0.831922665422135,
            "accuracy": 0.48091603053435117
          },
          {
            "n": 152,
            "confidence": 0.8988913804538349,
            "accuracy": 0.4605263157894737
          },
          {
            "n": 31,
            "confidence": 0.9563516129032258,
            "accuracy": 0.5161290322580645
          }
        ]
      }
    },
    "laya_td": {
      "newsfin_short": {
        "n": 140,
        "accuracy": 0.6142857142857143,
        "accuracy_ci95": [
          0.5357142857142857,
          0.6928571428571428
        ],
        "nll": 1.2190687244899345,
        "brier_onehot": 0.5204963383039098,
        "ece15": 0.10012283564261254,
        "mean_confidence": 0.6159351724446068,
        "uniform_accuracy": 0.2794763973450739,
        "confidence_thresholds": {
          "0.65": {
            "n": 57,
            "coverage": 0.40714285714285714,
            "accuracy": 0.7719298245614035
          },
          "0.8": {
            "n": 24,
            "coverage": 0.17142857142857143,
            "accuracy": 0.8333333333333334
          },
          "0.9": {
            "n": 15,
            "coverage": 0.10714285714285714,
            "accuracy": 0.8666666666666667
          }
        },
        "reliability": [
          {
            "n": 4,
            "confidence": 0.2333194916155514,
            "accuracy": 0.0
          },
          {
            "n": 6,
            "confidence": 0.302599998681,
            "accuracy": 0.16666666666666666
          },
          {
            "n": 13,
            "confidence": 0.36711329999243336,
            "accuracy": 0.38461538461538464
          },
          {
            "n": 11,
            "confidence": 0.4237506192347789,
            "accuracy": 0.45454545454545453
          },
          {
            "n": 18,
            "confidence": 0.511127863340612,
            "accuracy": 0.6111111111111112
          },
          {
            "n": 18,
            "confidence": 0.5625467793025789,
            "accuracy": 0.6111111111111112
          },
          {
            "n": 14,
            "confidence": 0.6292863952966607,
            "accuracy": 0.7142857142857143
          },
          {
            "n": 16,
            "confidence": 0.7084717264693747,
            "accuracy": 0.8125
          },
          {
            "n": 16,
            "confidence": 0.767652998875487,
            "accuracy": 0.625
          },
          {
            "n": 4,
            "confidence": 0.8280340331933613,
            "accuracy": 0.5
          },
          {
            "n": 7,
            "confidence": 0.8859190199163888,
            "accuracy": 1.0
          },
          {
            "n": 13,
            "confidence": 0.9831229609827629,
            "accuracy": 0.8461538461538461
          }
        ]
      },
      "newsfin_long": {
        "n": 56,
        "accuracy": 0.625,
        "accuracy_ci95": [
          0.5,
          0.75
        ],
        "nll": 0.9473697717806606,
        "brier_onehot": 0.5052564707694435,
        "ece15": 0.1282473784340939,
        "mean_confidence": 0.6203433290814203,
        "uniform_accuracy": 0.24032936018230133,
        "confidence_thresholds": {
          "0.65": {
            "n": 23,
            "coverage": 0.4107142857142857,
            "accuracy": 0.7391304347826086
          },
          "0.8": {
            "n": 9,
            "coverage": 0.16071428571428573,
            "accuracy": 0.8888888888888888
          },
          "0.9": {
            "n": 8,
            "coverage": 0.14285714285714285,
            "accuracy": 1.0
          }
        },
        "reliability": [
          {
            "n": 1,
            "confidence": 0.26117388261173874,
            "accuracy": 0.0
          },
          {
            "n": 3,
            "confidence": 0.30080999099909994,
            "accuracy": 1.0
          },
          {
            "n": 7,
            "confidence": 0.36791477390129057,
            "accuracy": 0.42857142857142855
          },
          {
            "n": 4,
            "confidence": 0.4233358232121082,
            "accuracy": 0.5
          },
          {
            "n": 2,
            "confidence": 0.5034250124987502,
            "accuracy": 0.5
          },
          {
            "n": 7,
            "confidence": 0.5632139915755685,
            "accuracy": 0.42857142857142855
          },
          {
            "n": 10,
            "confidence": 0.6319262866286629,
            "accuracy": 0.6
          },
          {
            "n": 5,
            "confidence": 0.70236,
            "accuracy": 0.8
          },
          {
            "n": 8,
            "confidence": 0.7650717596759675,
            "accuracy": 0.625
          },
          {
            "n": 1,
            "confidence": 0.8101810181018102,
            "accuracy": 0.0
          },
          {
            "n": 4,
            "confidence": 0.9228529563843856,
            "accuracy": 1.0
          },
          {
            "n": 4,
            "confidence": 0.9760745273582075,
            "accuracy": 1.0
          }
        ]
      },
      "heldout_tasks": {
        "n": 300,
        "accuracy": 0.63,
        "accuracy_ci95": [
          0.5766666666666667,
          0.6833333333333333
        ],
        "nll": 0.8284287990601404,
        "brier_onehot": 0.4631184306750694,
        "ece15": 0.06414849953652801,
        "mean_confidence": 0.6360782473450751,
        "uniform_accuracy": 0.3491236190667541,
        "confidence_thresholds": {
          "0.65": {
            "n": 147,
            "coverage": 0.49,
            "accuracy": 0.8367346938775511
          },
          "0.8": {
            "n": 93,
            "coverage": 0.31,
            "accuracy": 0.8494623655913979
          },
          "0.9": {
            "n": 33,
            "coverage": 0.11,
            "accuracy": 0.9393939393939394
          }
        },
        "reliability": [
          {
            "n": 4,
            "confidence": 0.2544439619109396,
            "accuracy": 0.5
          },
          {
            "n": 21,
            "confidence": 0.30978139390048054,
            "accuracy": 0.3333333333333333
          },
          {
            "n": 31,
            "confidence": 0.3617844818117397,
            "accuracy": 0.25806451612903225
          },
          {
            "n": 21,
            "confidence": 0.4331388408231027,
            "accuracy": 0.42857142857142855
          },
          {
            "n": 35,
            "confidence": 0.5037856980053429,
            "accuracy": 0.45714285714285713
          },
          {
            "n": 29,
            "confidence": 0.5615874261913189,
            "accuracy": 0.5862068965517241
          },
          {
            "n": 18,
            "confidence": 0.6337426574404175,
            "accuracy": 0.5555555555555556
          },
          {
            "n": 23,
            "confidence": 0.6971773777962333,
            "accuracy": 0.8695652173913043
          },
          {
            "n": 25,
            "confidence": 0.7676468601541085,
            "accuracy": 0.84
          },
          {
            "n": 42,
            "confidence": 0.8360288136827827,
            "accuracy": 0.7857142857142857
          },
          {
            "n": 27,
            "confidence": 0.8914703703703704,
            "accuracy": 0.8148148148148148
          },
          {
            "n": 24,
            "confidence": 0.9689162707922456,
            "accuracy": 1.0
          }
        ]
      },
      "typed_decisions": {
        "n": 2000,
        "accuracy": 0.7425,
        "accuracy_ci95": [
          0.7225,
          0.762
        ],
        "nll": 0.7436633069434331,
        "brier_onehot": 0.4235755392133792,
        "ece15": 0.2097095930770926,
        "mean_confidence": 0.5356001013082533,
        "uniform_accuracy": 0.3175,
        "confidence_thresholds": {
          "0.65": {
            "n": 392,
            "coverage": 0.196,
            "accuracy": 0.9617346938775511
          },
          "0.8": {
            "n": 129,
            "coverage": 0.0645,
            "accuracy": 0.9844961240310077
          },
          "0.9": {
            "n": 0,
            "coverage": 0.0,
            "accuracy": null
          }
        },
        "reliability": [
          {
            "n": 1,
            "confidence": 0.2657,
            "accuracy": 1.0
          },
          {
            "n": 34,
            "confidence": 0.31793218780429244,
            "accuracy": 0.23529411764705882
          },
          {
            "n": 289,
            "confidence": 0.3751108461445502,
            "accuracy": 0.5155709342560554
          },
          {
            "n": 387,
            "confidence": 0.4333358872661366,
            "accuracy": 0.6382428940568475
          },
          {
            "n": 454,
            "confidence": 0.4990018080263838,
            "accuracy": 0.7400881057268722
          },
          {
            "n": 269,
            "confidence": 0.5663064150458634,
            "accuracy": 0.7843866171003717
          },
          {
            "n": 206,
            "confidence": 0.6269887756403439,
            "accuracy": 0.9077669902912622
          },
          {
            "n": 129,
            "confidence": 0.6995473861386212,
            "accuracy": 0.9534883720930233
          },
          {
            "n": 102,
            "confidence": 0.7637360536255958,
            "accuracy": 0.9411764705882353
          },
          {
            "n": 122,
            "confidence": 0.8339088022933011,
            "accuracy": 0.9836065573770492
          },
          {
            "n": 7,
            "confidence": 0.8706124126698384,
            "accuracy": 1.0
          }
        ]
      }
    },
    "jevlike": {
      "newsfin_short": {
        "n": 140,
        "accuracy": 0.6357142857142857,
        "accuracy_ci95": [
          0.5571428571428572,
          0.7142857142857143
        ],
        "nll": 0.8810642049018346,
        "brier_onehot": 0.4594364556189868,
        "ece15": 0.09626927118430638,
        "mean_confidence": 0.7003433504563511,
        "uniform_accuracy": 0.2794763973450739,
        "confidence_thresholds": {
          "0.65": {
            "n": 74,
            "coverage": 0.5285714285714286,
            "accuracy": 0.8243243243243243
          },
          "0.8": {
            "n": 51,
            "coverage": 0.36428571428571427,
            "accuracy": 0.9019607843137255
          },
          "0.9": {
            "n": 36,
            "coverage": 0.2571428571428571,
            "accuracy": 0.9166666666666666
          }
        },
        "reliability": [
          {
            "n": 4,
            "confidence": 0.31435249081859906,
            "accuracy": 0.0
          },
          {
            "n": 4,
            "confidence": 0.3683047456063695,
            "accuracy": 0.5
          },
          {
            "n": 13,
            "confidence": 0.43412605262834336,
            "accuracy": 0.3076923076923077
          },
          {
            "n": 14,
            "confidence": 0.5018749093199865,
            "accuracy": 0.42857142857142855
          },
          {
            "n": 17,
            "confidence": 0.5625792631911706,
            "accuracy": 0.5294117647058824
          },
          {
            "n": 15,
            "confidence": 0.624756897657645,
            "accuracy": 0.5333333333333333
          },
          {
            "n": 6,
            "confidence": 0.6991859315065998,
            "accuracy": 0.8333333333333334
          },
          {
            "n": 16,
            "confidence": 0.765267463088596,
            "accuracy": 0.5625
          },
          {
            "n": 11,
            "confidence": 0.8288046343252776,
            "accuracy": 0.9090909090909091
          },
          {
            "n": 11,
            "confidence": 0.9037337492339783,
            "accuracy": 0.7272727272727273
          },
          {
            "n": 29,
            "confidence": 0.9729322257523271,
            "accuracy": 0.9655172413793104
          }
        ]
      },
      "newsfin_long": {
        "n": 56,
        "accuracy": 0.7142857142857143,
        "accuracy_ci95": [
          0.5892857142857143,
          0.8214285714285714
        ],
        "nll": 0.7747561322704637,
        "brier_onehot": 0.3982446040756354,
        "ece15": 0.1139029663017493,
        "mean_confidence": 0.7618069910792377,
        "uniform_accuracy": 0.24032936018230133,
        "confidence_thresholds": {
          "0.65": {
            "n": 38,
            "coverage": 0.6785714285714286,
            "accuracy": 0.8157894736842105
          },
          "0.8": {
            "n": 33,
            "coverage": 0.5892857142857143,
            "accuracy": 0.8484848484848485
          },
          "0.9": {
            "n": 26,
            "coverage": 0.4642857142857143,
            "accuracy": 0.9230769230769231
          }
        },
        "reliability": [
          {
            "n": 1,
            "confidence": 0.25787782972278783,
            "accuracy": 0.0
          },
          {
            "n": 2,
            "confidence": 0.30044849334548607,
            "accuracy": 0.0
          },
          {
            "n": 1,
            "confidence": 0.33464981710949976,
            "accuracy": 1.0
          },
          {
            "n": 5,
            "confidence": 0.4328088221147037,
            "accuracy": 0.6
          },
          {
            "n": 2,
            "confidence": 0.5085217603938686,
            "accuracy": 0.5
          },
          {
            "n": 6,
            "confidence": 0.5693763805464213,
            "accuracy": 0.5
          },
          {
            "n": 4,
            "confidence": 0.6606546915216647,
            "accuracy": 0.75
          },
          {
            "n": 2,
            "confidence": 0.689887285232544,
            "accuracy": 0.5
          },
          {
            "n": 4,
            "confidence": 0.8263626768434467,
            "accuracy": 0.5
          },
          {
            "n": 10,
            "confidence": 0.9067653114584996,
            "accuracy": 0.9
          },
          {
            "n": 19,
            "confidence": 0.9723644101980917,
            "accuracy": 0.8947368421052632
          }
        ]
      },
      "heldout_tasks": {
        "n": 300,
        "accuracy": 0.58,
        "accuracy_ci95": [
          0.5266666666666666,
          0.6366666666666667
        ],
        "nll": 0.8911869889628542,
        "brier_onehot": 0.5134231622791497,
        "ece15": 0.08311700140649199,
        "mean_confidence": 0.6243801023474437,
        "uniform_accuracy": 0.3491236190667541,
        "confidence_thresholds": {
          "0.65": {
            "n": 137,
            "coverage": 0.45666666666666667,
            "accuracy": 0.7445255474452555
          },
          "0.8": {
            "n": 64,
            "coverage": 0.21333333333333335,
            "accuracy": 0.8125
          },
          "0.9": {
            "n": 37,
            "coverage": 0.12333333333333334,
            "accuracy": 0.8648648648648649
          }
        },
        "reliability": [
          {
            "n": 1,
            "confidence": 0.23789036628440569,
            "accuracy": 1.0
          },
          {
            "n": 21,
            "confidence": 0.29895297878815114,
            "accuracy": 0.23809523809523808
          },
          {
            "n": 28,
            "confidence": 0.3638488940938339,
            "accuracy": 0.5
          },
          {
            "n": 28,
            "confidence": 0.4302174204297628,
            "accuracy": 0.25
          },
          {
            "n": 26,
            "confidence": 0.5017499667404475,
            "accuracy": 0.4230769230769231
          },
          {
            "n": 33,
            "confidence": 0.5623847003664137,
            "accuracy": 0.5151515151515151
          },
          {
            "n": 36,
            "confidence": 0.6323279372286387,
            "accuracy": 0.6666666666666666
          },
          {
            "n": 30,
            "confidence": 0.700325317035411,
            "accuracy": 0.6666666666666666
          },
          {
            "n": 33,
            "confidence": 0.7652276990151226,
            "accuracy": 0.696969696969697
          },
          {
            "n": 18,
            "confidence": 0.8330853245824539,
            "accuracy": 0.7777777777777778
          },
          {
            "n": 23,
            "confidence": 0.9067225341857484,
            "accuracy": 0.7391304347826086
          },
          {
            "n": 23,
            "confidence": 0.960167099210642,
            "accuracy": 0.9130434782608695
          }
        ]
      },
      "typed_decisions": {
        "n": 2000,
        "accuracy": 0.4595,
        "accuracy_ci95": [
          0.433,
          0.485
        ],
        "nll": 1.3444350869309427,
        "brier_onehot": 0.7480423473427633,
        "ece15": 0.23062323931638637,
        "mean_confidence": 0.683465700725639,
        "uniform_accuracy": 0.3175,
        "confidence_thresholds": {
          "0.65": {
            "n": 1103,
            "coverage": 0.5515,
            "accuracy": 0.5058930190389845
          },
          "0.8": {
            "n": 870,
            "coverage": 0.435,
            "accuracy": 0.5149425287356322
          },
          "0.9": {
            "n": 506,
            "coverage": 0.253,
            "accuracy": 0.5731225296442688
          }
        },
        "reliability": [
          {
            "n": 2,
            "confidence": 0.2636414220883421,
            "accuracy": 0.0
          },
          {
            "n": 112,
            "confidence": 0.31072671180670275,
            "accuracy": 0.3392857142857143
          },
          {
            "n": 220,
            "confidence": 0.36609577130409987,
            "accuracy": 0.38181818181818183
          },
          {
            "n": 183,
            "confidence": 0.43270138063245134,
            "accuracy": 0.366120218579235
          },
          {
            "n": 161,
            "confidence": 0.4997503451031316,
            "accuracy": 0.4472049689440994
          },
          {
            "n": 126,
            "confidence": 0.5668673187578344,
            "accuracy": 0.46825396825396826
          },
          {
            "n": 119,
            "confidence": 0.6321133013126154,
            "accuracy": 0.4369747899159664
          },
          {
            "n": 111,
            "confidence": 0.7002285366333068,
            "accuracy": 0.4864864864864865
          },
          {
            "n": 96,
            "confidence": 0.7659850098182762,
            "accuracy": 0.46875
          },
          {
            "n": 203,
            "confidence": 0.8370369211558824,
            "accuracy": 0.4729064039408867
          },
          {
            "n": 347,
            "confidence": 0.9023447644701712,
            "accuracy": 0.3919308357348703
          },
          {
            "n": 320,
            "confidence": 0.970245866206789,
            "accuracy": 0.675
          }
        ]
      }
    },
    "omni": {
      "newsfin_short": {
        "n": 140,
        "accuracy": 0.6642857142857143,
        "accuracy_ci95": [
          0.5857142857142857,
          0.7428571428571429
        ],
        "nll": 1.0239277480868136,
        "brier_onehot": 0.4833709544993766,
        "ece15": 0.086806222512248,
        "mean_confidence": 0.6900754779093492,
        "uniform_accuracy": 0.2794763973450739,
        "confidence_thresholds": {
          "0.65": {
            "n": 86,
            "coverage": 0.6142857142857143,
            "accuracy": 0.7906976744186046
          },
          "0.8": {
            "n": 56,
            "coverage": 0.4,
            "accuracy": 0.8392857142857143
          },
          "0.9": {
            "n": 36,
            "coverage": 0.2571428571428571,
            "accuracy": 0.8611111111111112
          }
        },
        "reliability": [
          {
            "n": 2,
            "confidence": 0.16850247623873293,
            "accuracy": 0.0
          },
          {
            "n": 4,
            "confidence": 0.25145491106610773,
            "accuracy": 0.5
          },
          {
            "n": 3,
            "confidence": 0.30153553466505506,
            "accuracy": 1.0
          },
          {
            "n": 11,
            "confidence": 0.36396977029923866,
            "accuracy": 0.36363636363636365
          },
          {
            "n": 10,
            "confidence": 0.4277883482274337,
            "accuracy": 0.4
          },
          {
            "n": 7,
            "confidence": 0.5038134981024585,
            "accuracy": 0.42857142857142855
          },
          {
            "n": 10,
            "confidence": 0.5570586830396977,
            "accuracy": 0.6
          },
          {
            "n": 18,
            "confidence": 0.6415175415281524,
            "accuracy": 0.5555555555555556
          },
          {
            "n": 9,
            "confidence": 0.6942038665711675,
            "accuracy": 0.7777777777777778
          },
          {
            "n": 10,
            "confidence": 0.7485815357473065,
            "accuracy": 0.7
          },
          {
            "n": 11,
            "confidence": 0.825007934222517,
            "accuracy": 0.8181818181818182
          },
          {
            "n": 19,
            "confidence": 0.8981023578758742,
            "accuracy": 0.7894736842105263
          },
          {
            "n": 26,
            "confidence": 0.98324251744931,
            "accuracy": 0.8846153846153846
          }
        ]
      },
      "newsfin_long": {
        "n": 56,
        "accuracy": 0.5535714285714286,
        "accuracy_ci95": [
          0.4107142857142857,
          0.6785714285714286
        ],
        "nll": 1.0747286532578382,
        "brier_onehot": 0.5316587234647131,
        "ece15": 0.20183825996408222,
        "mean_confidence": 0.6893829250347027,
        "uniform_accuracy": 0.24032936018230133,
        "confidence_thresholds": {
          "0.65": {
            "n": 32,
            "coverage": 0.5714285714285714,
            "accuracy": 0.78125
          },
          "0.8": {
            "n": 23,
            "coverage": 0.4107142857142857,
            "accuracy": 0.7391304347826086
          },
          "0.9": {
            "n": 13,
            "coverage": 0.23214285714285715,
            "accuracy": 0.9230769230769231
          }
        },
        "reliability": [
          {
            "n": 1,
            "confidence": 0.17508739445476842,
            "accuracy": 0.0
          },
          {
            "n": 3,
            "confidence": 0.30233710435052524,
            "accuracy": 0.3333333333333333
          },
          {
            "n": 6,
            "confidence": 0.3677328588099678,
            "accuracy": 0.16666666666666666
          },
          {
            "n": 2,
            "confidence": 0.44699478884245114,
            "accuracy": 0.0
          },
          {
            "n": 4,
            "confidence": 0.4922646316138709,
            "accuracy": 0.0
          },
          {
            "n": 5,
            "confidence": 0.5582380460237041,
            "accuracy": 0.4
          },
          {
            "n": 4,
            "confidence": 0.622083597856155,
            "accuracy": 0.75
          },
          {
            "n": 4,
            "confidence": 0.6889762293752948,
            "accuracy": 1.0
          },
          {
            "n": 4,
            "confidence": 0.7732806664163974,
            "accuracy": 0.75
          },
          {
            "n": 6,
            "confidence": 0.8408541777219157,
            "accuracy": 0.5
          },
          {
            "n": 4,
            "confidence": 0.8780159919507924,
            "accuracy": 0.5
          },
          {
            "n": 13,
            "confidence": 0.9821660460455572,
            "accuracy": 0.9230769230769231
          }
        ]
      },
      "heldout_tasks": {
        "n": 300,
        "accuracy": 0.6333333333333333,
        "accuracy_ci95": [
          0.58,
          0.69
        ],
        "nll": 0.8258969806486628,
        "brier_onehot": 0.4569257718777433,
        "ece15": 0.09554911465107167,
        "mean_confidence": 0.6315795431338078,
        "uniform_accuracy": 0.3491236190667541,
        "confidence_thresholds": {
          "0.65": {
            "n": 148,
            "coverage": 0.49333333333333335,
            "accuracy": 0.7972972972972973
          },
          "0.8": {
            "n": 61,
            "coverage": 0.20333333333333334,
            "accuracy": 0.8688524590163934
          },
          "0.9": {
            "n": 26,
            "coverage": 0.08666666666666667,
            "accuracy": 0.9615384615384616
          }
        },
        "reliability": [
          {
            "n": 1,
            "confidence": 0.1046529915301365,
            "accuracy": 0.0
          },
          {
            "n": 4,
            "confidence": 0.2577561216819676,
            "accuracy": 0.75
          },
          {
            "n": 11,
            "confidence": 0.302752906130526,
            "accuracy": 0.2727272727272727
          },
          {
            "n": 27,
            "confidence": 0.3719780866552427,
            "accuracy": 0.4444444444444444
          },
          {
            "n": 17,
            "confidence": 0.43146238975131995,
            "accuracy": 0.4117647058823529
          },
          {
            "n": 37,
            "confidence": 0.49454230114799697,
            "accuracy": 0.35135135135135137
          },
          {
            "n": 36,
            "confidence": 0.5579886897891916,
            "accuracy": 0.7222222222222222
          },
          {
            "n": 29,
            "confidence": 0.6272471234022386,
            "accuracy": 0.41379310344827586
          },
          {
            "n": 29,
            "confidence": 0.6959158351866087,
            "accuracy": 0.8275862068965517
          },
          {
            "n": 48,
            "confidence": 0.7644648380498819,
            "accuracy": 0.7708333333333334
          },
          {
            "n": 27,
            "confidence": 0.8256353707236136,
            "accuracy": 0.7777777777777778
          },
          {
            "n": 17,
            "confidence": 0.9039215980455362,
            "accuracy": 0.9411764705882353
          },
          {
            "n": 17,
            "confidence": 0.9717127266067751,
            "accuracy": 0.9411764705882353
          }
        ]
      },
      "typed_decisions": {
        "n": 2000,
        "accuracy": 0.4765,
        "accuracy_ci95": [
          0.4545,
          0.499
        ],
        "nll": 1.0863653374222986,
        "brier_onehot": 0.6166074547470112,
        "ece15": 0.11012610270598122,
        "mean_confidence": 0.5241069191428419,
        "uniform_accuracy": 0.3175,
        "confidence_thresholds": {
          "0.65": {
            "n": 523,
            "coverage": 0.2615,
            "accuracy": 0.7074569789674953
          },
          "0.8": {
            "n": 251,
            "coverage": 0.1255,
            "accuracy": 0.6573705179282868
          },
          "0.9": {
            "n": 100,
            "coverage": 0.05,
            "accuracy": 0.72
          }
        },
        "reliability": [
          {
            "n": 10,
            "confidence": 0.25256444635115816,
            "accuracy": 0.3
          },
          {
            "n": 189,
            "confidence": 0.30877873898245745,
            "accuracy": 0.2698412698412698
          },
          {
            "n": 472,
            "confidence": 0.36585330883687267,
            "accuracy": 0.2817796610169492
          },
          {
            "n": 302,
            "confidence": 0.42887444845445033,
            "accuracy": 0.38741721854304634
          },
          {
            "n": 243,
            "confidence": 0.49993474017946105,
            "accuracy": 0.3868312757201646
          },
          {
            "n": 155,
            "confidence": 0.5580824847934738,
            "accuracy": 0.6645161290322581
          },
          {
            "n": 218,
            "confidence": 0.6316491082827173,
            "accuracy": 0.7889908256880734
          },
          {
            "n": 77,
            "confidence": 0.6980893665549133,
            "accuracy": 0.8441558441558441
          },
          {
            "n": 83,
            "confidence": 0.7678636543006925,
            "accuracy": 0.6024096385542169
          },
          {
            "n": 112,
            "confidence": 0.8273256031580205,
            "accuracy": 0.6160714285714286
          },
          {
            "n": 69,
            "confidence": 0.8956754913922487,
            "accuracy": 0.6086956521739131
          },
          {
            "n": 70,
            "confidence": 0.9641727620937258,
            "accuracy": 0.7714285714285715
          }
        ]
      }
    }
  },
  "speed": {
    "short_1q": {
      "n": 20,
      "p50_ms": 26.517087999991418,
      "p95_ms": 29.029050500071207,
      "mean_ms": 27.113317650002955,
      "min_ms": 24.377534999985073,
      "max_ms": 29.310716000054526,
      "samples_ms": [
        26.25009999997019,
        25.432290000026114,
        26.320786000042062,
        28.980330999957005,
        26.646911999932854,
        28.12136200009263,
        25.81902299993999,
        26.38726400004998,
        25.392123000074207,
        28.523082999981852,
        26.326767000000473,
        28.44737800000985,
        28.463398999974743,
        29.310716000054526,
        28.692237999848658,
        24.377534999985073,
        28.48372800008292,
        25.53521799995906,
        29.014226000072085,
        25.741874000004827
      ],
      "warmup_calls": 3,
      "first_shape_call_ms": 27.415586999950392,
      "questions_per_call": 1,
      "window_counts": [
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1
      ]
    },
    "short_5q": {
      "n": 20,
      "p50_ms": 60.07641500002592,
      "p95_ms": 65.5484159998764,
      "mean_ms": 58.98180429998092,
      "min_ms": 51.847296999994796,
      "max_ms": 66.96372600003997,
      "samples_ms": [
        56.865289999905144,
        53.42559599989727,
        54.647222000085094,
        60.657620000029056,
        61.10556999988148,
        55.25912699999935,
        53.47630400001435,
        52.484391000007236,
        51.847296999994796,
        60.60912699990695,
        59.04960399993797,
        61.89990600000783,
        61.19320799984962,
        65.47392599986779,
        63.0494520000866,
        55.42915399996673,
        66.96372600003997,
        63.058288999854994,
        63.59757400014132,
        59.54370300014489
      ],
      "warmup_calls": 3,
      "first_shape_call_ms": 54.20621600001141,
      "questions_per_call": 5,
      "window_counts": [
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1
      ]
    },
    "short_10q": {
      "n": 20,
      "p50_ms": 104.81810399994629,
      "p95_ms": 119.18314979989191,
      "mean_ms": 105.92105819998778,
      "min_ms": 88.4976249999454,
      "max_ms": 120.43251000000055,
      "samples_ms": [
        92.77988300004836,
        104.6406630000547,
        96.89195900000414,
        119.11739399988619,
        105.96081699986826,
        103.6471459999575,
        98.7587220001842,
        98.4291290001238,
        88.4976249999454,
        114.96733699982542,
        109.29184199994779,
        104.99554499983788,
        110.70990500002154,
        115.22976200012636,
        118.5596349998832,
        95.96929799999998,
        120.43251000000055,
        104.30120299997725,
        115.14932800014321,
        100.09146099991995
      ],
      "warmup_calls": 3,
      "first_shape_call_ms": 93.11735100004626,
      "questions_per_call": 10,
      "window_counts": [
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1
      ]
    },
    "short_50q": {
      "n": 20,
      "p50_ms": 682.1585460000961,
      "p95_ms": 754.3870993999008,
      "mean_ms": 689.1575670499719,
      "min_ms": 618.0243280000468,
      "max_ms": 763.9201289998709,
      "samples_ms": [
        674.0678559999651,
        678.1810669999686,
        618.0243280000468,
        753.8853609999023,
        707.519105000074,
        670.8609360000537,
        680.0984779999908,
        671.1490820000563,
        641.9182600000113,
        688.8830089999374,
        693.8877620000312,
        684.2186140002013,
        674.6635569998034,
        738.8550079999732,
        707.0535409998229,
        627.7758280000398,
        747.2571369999059,
        698.2282959997974,
        763.9201289998709,
        662.7039869999862
      ],
      "warmup_calls": 3,
      "first_shape_call_ms": 596.6050259999065,
      "questions_per_call": 50,
      "window_counts": [
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1
      ]
    },
    "long_1q": {
      "n": 20,
      "p50_ms": 123.90287249991161,
      "p95_ms": 131.76431719999755,
      "mean_ms": 124.6096985999884,
      "min_ms": 119.37196000008043,
      "max_ms": 131.96851399993648,
      "samples_ms": [
        127.72182899993823,
        126.77874800010613,
        123.88399099995695,
        123.92175399986627,
        125.15549199997622,
        131.75357000000076,
        130.19190099998923,
        131.96851399993648,
        127.03383000007307,
        119.37196000008043,
        123.77765799988083,
        120.07203400003164,
        119.416425000054,
        119.82972900000277,
        120.96001899999465,
        120.67090499999722,
        120.97485899994354,
        120.59792499985633,
        128.58475899997757,
        129.5280700001058
      ],
      "warmup_calls": 3,
      "first_shape_call_ms": 122.09491500016156,
      "questions_per_call": 1,
      "window_counts": [
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1
      ]
    },
    "long_5q": {
      "n": 20,
      "p50_ms": 685.7518550001487,
      "p95_ms": 728.8032338999642,
      "mean_ms": 696.9230963500081,
      "min_ms": 674.529241000073,
      "max_ms": 733.069986000146,
      "samples_ms": [
        678.4065869999267,
        675.5267930000173,
        680.8560730000863,
        675.0147320001361,
        674.6946719999869,
        680.0351919998775,
        715.0993089999247,
        724.1639889998623,
        722.2430469998926,
        724.2127150000215,
        733.069986000146,
        728.5786679999546,
        719.8392059999605,
        713.666480000029,
        690.3053800001544,
        681.198330000143,
        674.529241000073,
        675.8001450000393,
        680.2929479999875,
        690.9284339999431
      ],
      "warmup_calls": 3,
      "first_shape_call_ms": 690.0589629999558,
      "questions_per_call": 5,
      "window_counts": [
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1
      ]
    },
    "long_10q": {
      "n": 20,
      "p50_ms": 1321.0661100000607,
      "p95_ms": 1336.965719349837,
      "mean_ms": 1321.2527787499992,
      "min_ms": 1311.405681999986,
      "max_ms": 1341.6646919999948,
      "samples_ms": [
        1333.539275000021,
        1322.840443000132,
        1312.4046030000045,
        1312.6979950000077,
        1326.8468310000117,
        1323.0493790001674,
        1317.6598679999643,
        1312.3101529999985,
        1316.093828000021,
        1311.405681999986,
        1311.9573989999935,
        1324.5337289999952,
        1341.6646919999948,
        1319.2917769999895,
        1324.1005910001604,
        1312.415898999916,
        1336.7184049998286,
        1325.0531029998456,
        1324.322876999986,
        1316.149045999964
      ],
      "warmup_calls": 3,
      "first_shape_call_ms": 1341.1980430000767,
      "questions_per_call": 10,
      "window_counts": [
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1,
        1
      ]
    },
    "throughput": {
      "states": 16,
      "questions": 48,
      "repetitions": 3,
      "batch_size_laya": 8,
      "median_ms": 507.1860810000999,
      "questions_per_second": 94.63982115863813,
      "samples_ms": [
        506.4106110000921,
        507.1860810000999,
        507.4538389999361
      ]
    }
  }
}