| { |
| "protocol": { |
| "created_utc": "2026-09-30T15:17:02.035817+00:00", |
| "accuracy_n": 2496, |
| "groups": { |
| "heldout_tasks": 300, |
| "newsfin_long": 56, |
| "newsfin_short": 140, |
| "typed_decisions": 2000 |
| }, |
| "local_omni_train_or_validation_overlap": 0, |
| "accuracy_manifest_sha256": "77c421d0c50c76eb89016a4d77b4fde198d58c0bb0ca0d8413af171fdc183d24", |
| "speed_workloads_sha256": "04f2d48d8bd6c6b84563b19a55f69153e8c67c513ceb5a727918f34aad78933d", |
| "laya_revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851", |
| "laya_runtime": "0.3.22", |
| "laya_fast_path": "Not enabled: optional TileLang is not installed. Stock public runtime measured.", |
| "timing": "Public Python call, tokenization included, CUDA synchronized before/after. No HTTP/network overhead.", |
| "normalization": "Identical normalized question text/options and state bytes; noul criteria omitted equally per the existing project protocol.", |
| "calibration": "Existing deployed temperatures retained; no fitting, training, or threshold selection on this test set.", |
| "typed_decisions": "All 400 test cases / 2000 decisions; teacher-distribution metrics and hard labels reported separately.", |
| "laya_td": "Task-specialized checkpoint trained upstream on the benchmark training split; not a zero-shot baseline.", |
| "limitations": "Public pretraining overlap unknown; desktop shares GPU; p95 from 20 calls is exploratory, not an SLA." |
| }, |
| "accuracy": { |
| "laya": { |
| "newsfin_short": { |
| "n": 140, |
| "accuracy": 0.6071428571428571, |
| "accuracy_ci95": [ |
| 0.5285714285714286, |
| 0.6857142857142857 |
| ], |
| "nll": 1.4223718041679008, |
| "brier_onehot": 0.5707742968520072, |
| "ece15": 0.16604753199968444, |
| "mean_confidence": 0.7438418601631568, |
| "uniform_accuracy": 0.2794763973450739, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 91, |
| "coverage": 0.65, |
| "accuracy": 0.7032967032967034 |
| }, |
| "0.8": { |
| "n": 71, |
| "coverage": 0.5071428571428571, |
| "accuracy": 0.7183098591549296 |
| }, |
| "0.9": { |
| "n": 43, |
| "coverage": 0.30714285714285716, |
| "accuracy": 0.7441860465116279 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 6, |
| "confidence": 0.2996381148700478, |
| "accuracy": 0.3333333333333333 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.37458102430107054, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 10, |
| "confidence": 0.43827573330895736, |
| "accuracy": 0.2 |
| }, |
| { |
| "n": 11, |
| "confidence": 0.5135863804562275, |
| "accuracy": 0.6363636363636364 |
| }, |
| { |
| "n": 7, |
| "confidence": 0.5655142857142857, |
| "accuracy": 0.42857142857142855 |
| }, |
| { |
| "n": 13, |
| "confidence": 0.6327408621914118, |
| "accuracy": 0.46153846153846156 |
| }, |
| { |
| "n": 9, |
| "confidence": 0.6960221081986655, |
| "accuracy": 0.6666666666666666 |
| }, |
| { |
| "n": 9, |
| "confidence": 0.7785007062507501, |
| "accuracy": 0.6666666666666666 |
| }, |
| { |
| "n": 19, |
| "confidence": 0.8315686740694261, |
| "accuracy": 0.7368421052631579 |
| }, |
| { |
| "n": 19, |
| "confidence": 0.8988784093221, |
| "accuracy": 0.6842105263157895 |
| }, |
| { |
| "n": 33, |
| "confidence": 0.984123302889007, |
| "accuracy": 0.7272727272727273 |
| } |
| ] |
| }, |
| "newsfin_long": { |
| "n": 56, |
| "accuracy": 0.6607142857142857, |
| "accuracy_ci95": [ |
| 0.5357142857142857, |
| 0.7857142857142857 |
| ], |
| "nll": 1.2032614087003177, |
| "brier_onehot": 0.5529698397175744, |
| "ece15": 0.19426640345296486, |
| "mean_confidence": 0.784147753253207, |
| "uniform_accuracy": 0.24032936018230133, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 41, |
| "coverage": 0.7321428571428571, |
| "accuracy": 0.6829268292682927 |
| }, |
| "0.8": { |
| "n": 34, |
| "coverage": 0.6071428571428571, |
| "accuracy": 0.6764705882352942 |
| }, |
| "0.9": { |
| "n": 21, |
| "coverage": 0.375, |
| "accuracy": 0.7142857142857143 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 3, |
| "confidence": 0.36054520785411875, |
| "accuracy": 0.6666666666666666 |
| }, |
| { |
| "n": 2, |
| "confidence": 0.42155, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 3, |
| "confidence": 0.496984198359848, |
| "accuracy": 0.3333333333333333 |
| }, |
| { |
| "n": 2, |
| "confidence": 0.5759784578457847, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 6, |
| "confidence": 0.6315058727460587, |
| "accuracy": 0.6666666666666666 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.690932906709329, |
| "accuracy": 0.75 |
| }, |
| { |
| "n": 2, |
| "confidence": 0.7695876537653765, |
| "accuracy": 1.0 |
| }, |
| { |
| "n": 11, |
| "confidence": 0.8321180001198024, |
| "accuracy": 0.5454545454545454 |
| }, |
| { |
| "n": 6, |
| "confidence": 0.9037815511951489, |
| "accuracy": 0.6666666666666666 |
| }, |
| { |
| "n": 17, |
| "confidence": 0.9809823275595868, |
| "accuracy": 0.7647058823529411 |
| } |
| ] |
| }, |
| "heldout_tasks": { |
| "n": 300, |
| "accuracy": 0.66, |
| "accuracy_ci95": [ |
| 0.6066666666666667, |
| 0.7133333333333334 |
| ], |
| "nll": 1.5088246130436413, |
| "brier_onehot": 0.49109099242960164, |
| "ece15": 0.1363256368468856, |
| "mean_confidence": 0.7706782853152053, |
| "uniform_accuracy": 0.3491236190667541, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 195, |
| "coverage": 0.65, |
| "accuracy": 0.7897435897435897 |
| }, |
| "0.8": { |
| "n": 160, |
| "coverage": 0.5333333333333333, |
| "accuracy": 0.81875 |
| }, |
| "0.9": { |
| "n": 136, |
| "coverage": 0.4533333333333333, |
| "accuracy": 0.8235294117647058 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 1, |
| "confidence": 0.3066, |
| "accuracy": 0.0 |
| }, |
| { |
| "n": 15, |
| "confidence": 0.381084709529047, |
| "accuracy": 0.3333333333333333 |
| }, |
| { |
| "n": 26, |
| "confidence": 0.4301559573447903, |
| "accuracy": 0.23076923076923078 |
| }, |
| { |
| "n": 26, |
| "confidence": 0.5058806642403066, |
| "accuracy": 0.6538461538461539 |
| }, |
| { |
| "n": 20, |
| "confidence": 0.5698708586837585, |
| "accuracy": 0.45 |
| }, |
| { |
| "n": 20, |
| "confidence": 0.6283789555242396, |
| "accuracy": 0.4 |
| }, |
| { |
| "n": 15, |
| "confidence": 0.7039680905242808, |
| "accuracy": 0.6 |
| }, |
| { |
| "n": 17, |
| "confidence": 0.7685221145643976, |
| "accuracy": 0.7647058823529411 |
| }, |
| { |
| "n": 16, |
| "confidence": 0.83745649416744, |
| "accuracy": 0.6875 |
| }, |
| { |
| "n": 30, |
| "confidence": 0.9053593478879268, |
| "accuracy": 0.8 |
| }, |
| { |
| "n": 114, |
| "confidence": 0.988545481205944, |
| "accuracy": 0.8421052631578947 |
| } |
| ] |
| }, |
| "typed_decisions": { |
| "n": 2000, |
| "accuracy": 0.364, |
| "accuracy_ci95": [ |
| 0.339, |
| 0.3885 |
| ], |
| "nll": 1.340352049822895, |
| "brier_onehot": 0.7689268871867527, |
| "ece15": 0.21537093573642535, |
| "mean_confidence": 0.5793709357364253, |
| "uniform_accuracy": 0.3175, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 608, |
| "coverage": 0.304, |
| "accuracy": 0.45230263157894735 |
| }, |
| "0.8": { |
| "n": 314, |
| "coverage": 0.157, |
| "accuracy": 0.4745222929936306 |
| }, |
| "0.9": { |
| "n": 102, |
| "coverage": 0.051, |
| "accuracy": 0.4803921568627451 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 13, |
| "confidence": 0.25357506365936605, |
| "accuracy": 0.23076923076923078 |
| }, |
| { |
| "n": 122, |
| "confidence": 0.30950061571517834, |
| "accuracy": 0.23770491803278687 |
| }, |
| { |
| "n": 228, |
| "confidence": 0.3670941177030421, |
| "accuracy": 0.2719298245614035 |
| }, |
| { |
| "n": 233, |
| "confidence": 0.4363279138876397, |
| "accuracy": 0.2875536480686695 |
| }, |
| { |
| "n": 256, |
| "confidence": 0.5028284181113388, |
| "accuracy": 0.30859375 |
| }, |
| { |
| "n": 337, |
| "confidence": 0.5673213889715166, |
| "accuracy": 0.3857566765578635 |
| }, |
| { |
| "n": 256, |
| "confidence": 0.6312426961373134, |
| "accuracy": 0.3828125 |
| }, |
| { |
| "n": 131, |
| "confidence": 0.6972084159331299, |
| "accuracy": 0.42748091603053434 |
| }, |
| { |
| "n": 110, |
| "confidence": 0.765639844893953, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 131, |
| "confidence": 0.831922665422135, |
| "accuracy": 0.48091603053435117 |
| }, |
| { |
| "n": 152, |
| "confidence": 0.8988913804538349, |
| "accuracy": 0.4605263157894737 |
| }, |
| { |
| "n": 31, |
| "confidence": 0.9563516129032258, |
| "accuracy": 0.5161290322580645 |
| } |
| ] |
| } |
| }, |
| "laya_td": { |
| "newsfin_short": { |
| "n": 140, |
| "accuracy": 0.6142857142857143, |
| "accuracy_ci95": [ |
| 0.5357142857142857, |
| 0.6928571428571428 |
| ], |
| "nll": 1.2190687244899345, |
| "brier_onehot": 0.5204963383039098, |
| "ece15": 0.10012283564261254, |
| "mean_confidence": 0.6159351724446068, |
| "uniform_accuracy": 0.2794763973450739, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 57, |
| "coverage": 0.40714285714285714, |
| "accuracy": 0.7719298245614035 |
| }, |
| "0.8": { |
| "n": 24, |
| "coverage": 0.17142857142857143, |
| "accuracy": 0.8333333333333334 |
| }, |
| "0.9": { |
| "n": 15, |
| "coverage": 0.10714285714285714, |
| "accuracy": 0.8666666666666667 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 4, |
| "confidence": 0.2333194916155514, |
| "accuracy": 0.0 |
| }, |
| { |
| "n": 6, |
| "confidence": 0.302599998681, |
| "accuracy": 0.16666666666666666 |
| }, |
| { |
| "n": 13, |
| "confidence": 0.36711329999243336, |
| "accuracy": 0.38461538461538464 |
| }, |
| { |
| "n": 11, |
| "confidence": 0.4237506192347789, |
| "accuracy": 0.45454545454545453 |
| }, |
| { |
| "n": 18, |
| "confidence": 0.511127863340612, |
| "accuracy": 0.6111111111111112 |
| }, |
| { |
| "n": 18, |
| "confidence": 0.5625467793025789, |
| "accuracy": 0.6111111111111112 |
| }, |
| { |
| "n": 14, |
| "confidence": 0.6292863952966607, |
| "accuracy": 0.7142857142857143 |
| }, |
| { |
| "n": 16, |
| "confidence": 0.7084717264693747, |
| "accuracy": 0.8125 |
| }, |
| { |
| "n": 16, |
| "confidence": 0.767652998875487, |
| "accuracy": 0.625 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.8280340331933613, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 7, |
| "confidence": 0.8859190199163888, |
| "accuracy": 1.0 |
| }, |
| { |
| "n": 13, |
| "confidence": 0.9831229609827629, |
| "accuracy": 0.8461538461538461 |
| } |
| ] |
| }, |
| "newsfin_long": { |
| "n": 56, |
| "accuracy": 0.625, |
| "accuracy_ci95": [ |
| 0.5, |
| 0.75 |
| ], |
| "nll": 0.9473697717806606, |
| "brier_onehot": 0.5052564707694435, |
| "ece15": 0.1282473784340939, |
| "mean_confidence": 0.6203433290814203, |
| "uniform_accuracy": 0.24032936018230133, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 23, |
| "coverage": 0.4107142857142857, |
| "accuracy": 0.7391304347826086 |
| }, |
| "0.8": { |
| "n": 9, |
| "coverage": 0.16071428571428573, |
| "accuracy": 0.8888888888888888 |
| }, |
| "0.9": { |
| "n": 8, |
| "coverage": 0.14285714285714285, |
| "accuracy": 1.0 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 1, |
| "confidence": 0.26117388261173874, |
| "accuracy": 0.0 |
| }, |
| { |
| "n": 3, |
| "confidence": 0.30080999099909994, |
| "accuracy": 1.0 |
| }, |
| { |
| "n": 7, |
| "confidence": 0.36791477390129057, |
| "accuracy": 0.42857142857142855 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.4233358232121082, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 2, |
| "confidence": 0.5034250124987502, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 7, |
| "confidence": 0.5632139915755685, |
| "accuracy": 0.42857142857142855 |
| }, |
| { |
| "n": 10, |
| "confidence": 0.6319262866286629, |
| "accuracy": 0.6 |
| }, |
| { |
| "n": 5, |
| "confidence": 0.70236, |
| "accuracy": 0.8 |
| }, |
| { |
| "n": 8, |
| "confidence": 0.7650717596759675, |
| "accuracy": 0.625 |
| }, |
| { |
| "n": 1, |
| "confidence": 0.8101810181018102, |
| "accuracy": 0.0 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.9228529563843856, |
| "accuracy": 1.0 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.9760745273582075, |
| "accuracy": 1.0 |
| } |
| ] |
| }, |
| "heldout_tasks": { |
| "n": 300, |
| "accuracy": 0.63, |
| "accuracy_ci95": [ |
| 0.5766666666666667, |
| 0.6833333333333333 |
| ], |
| "nll": 0.8284287990601404, |
| "brier_onehot": 0.4631184306750694, |
| "ece15": 0.06414849953652801, |
| "mean_confidence": 0.6360782473450751, |
| "uniform_accuracy": 0.3491236190667541, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 147, |
| "coverage": 0.49, |
| "accuracy": 0.8367346938775511 |
| }, |
| "0.8": { |
| "n": 93, |
| "coverage": 0.31, |
| "accuracy": 0.8494623655913979 |
| }, |
| "0.9": { |
| "n": 33, |
| "coverage": 0.11, |
| "accuracy": 0.9393939393939394 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 4, |
| "confidence": 0.2544439619109396, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 21, |
| "confidence": 0.30978139390048054, |
| "accuracy": 0.3333333333333333 |
| }, |
| { |
| "n": 31, |
| "confidence": 0.3617844818117397, |
| "accuracy": 0.25806451612903225 |
| }, |
| { |
| "n": 21, |
| "confidence": 0.4331388408231027, |
| "accuracy": 0.42857142857142855 |
| }, |
| { |
| "n": 35, |
| "confidence": 0.5037856980053429, |
| "accuracy": 0.45714285714285713 |
| }, |
| { |
| "n": 29, |
| "confidence": 0.5615874261913189, |
| "accuracy": 0.5862068965517241 |
| }, |
| { |
| "n": 18, |
| "confidence": 0.6337426574404175, |
| "accuracy": 0.5555555555555556 |
| }, |
| { |
| "n": 23, |
| "confidence": 0.6971773777962333, |
| "accuracy": 0.8695652173913043 |
| }, |
| { |
| "n": 25, |
| "confidence": 0.7676468601541085, |
| "accuracy": 0.84 |
| }, |
| { |
| "n": 42, |
| "confidence": 0.8360288136827827, |
| "accuracy": 0.7857142857142857 |
| }, |
| { |
| "n": 27, |
| "confidence": 0.8914703703703704, |
| "accuracy": 0.8148148148148148 |
| }, |
| { |
| "n": 24, |
| "confidence": 0.9689162707922456, |
| "accuracy": 1.0 |
| } |
| ] |
| }, |
| "typed_decisions": { |
| "n": 2000, |
| "accuracy": 0.7425, |
| "accuracy_ci95": [ |
| 0.7225, |
| 0.762 |
| ], |
| "nll": 0.7436633069434331, |
| "brier_onehot": 0.4235755392133792, |
| "ece15": 0.2097095930770926, |
| "mean_confidence": 0.5356001013082533, |
| "uniform_accuracy": 0.3175, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 392, |
| "coverage": 0.196, |
| "accuracy": 0.9617346938775511 |
| }, |
| "0.8": { |
| "n": 129, |
| "coverage": 0.0645, |
| "accuracy": 0.9844961240310077 |
| }, |
| "0.9": { |
| "n": 0, |
| "coverage": 0.0, |
| "accuracy": null |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 1, |
| "confidence": 0.2657, |
| "accuracy": 1.0 |
| }, |
| { |
| "n": 34, |
| "confidence": 0.31793218780429244, |
| "accuracy": 0.23529411764705882 |
| }, |
| { |
| "n": 289, |
| "confidence": 0.3751108461445502, |
| "accuracy": 0.5155709342560554 |
| }, |
| { |
| "n": 387, |
| "confidence": 0.4333358872661366, |
| "accuracy": 0.6382428940568475 |
| }, |
| { |
| "n": 454, |
| "confidence": 0.4990018080263838, |
| "accuracy": 0.7400881057268722 |
| }, |
| { |
| "n": 269, |
| "confidence": 0.5663064150458634, |
| "accuracy": 0.7843866171003717 |
| }, |
| { |
| "n": 206, |
| "confidence": 0.6269887756403439, |
| "accuracy": 0.9077669902912622 |
| }, |
| { |
| "n": 129, |
| "confidence": 0.6995473861386212, |
| "accuracy": 0.9534883720930233 |
| }, |
| { |
| "n": 102, |
| "confidence": 0.7637360536255958, |
| "accuracy": 0.9411764705882353 |
| }, |
| { |
| "n": 122, |
| "confidence": 0.8339088022933011, |
| "accuracy": 0.9836065573770492 |
| }, |
| { |
| "n": 7, |
| "confidence": 0.8706124126698384, |
| "accuracy": 1.0 |
| } |
| ] |
| } |
| }, |
| "jevlike": { |
| "newsfin_short": { |
| "n": 140, |
| "accuracy": 0.6357142857142857, |
| "accuracy_ci95": [ |
| 0.5571428571428572, |
| 0.7142857142857143 |
| ], |
| "nll": 0.8810642049018346, |
| "brier_onehot": 0.4594364556189868, |
| "ece15": 0.09626927118430638, |
| "mean_confidence": 0.7003433504563511, |
| "uniform_accuracy": 0.2794763973450739, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 74, |
| "coverage": 0.5285714285714286, |
| "accuracy": 0.8243243243243243 |
| }, |
| "0.8": { |
| "n": 51, |
| "coverage": 0.36428571428571427, |
| "accuracy": 0.9019607843137255 |
| }, |
| "0.9": { |
| "n": 36, |
| "coverage": 0.2571428571428571, |
| "accuracy": 0.9166666666666666 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 4, |
| "confidence": 0.31435249081859906, |
| "accuracy": 0.0 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.3683047456063695, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 13, |
| "confidence": 0.43412605262834336, |
| "accuracy": 0.3076923076923077 |
| }, |
| { |
| "n": 14, |
| "confidence": 0.5018749093199865, |
| "accuracy": 0.42857142857142855 |
| }, |
| { |
| "n": 17, |
| "confidence": 0.5625792631911706, |
| "accuracy": 0.5294117647058824 |
| }, |
| { |
| "n": 15, |
| "confidence": 0.624756897657645, |
| "accuracy": 0.5333333333333333 |
| }, |
| { |
| "n": 6, |
| "confidence": 0.6991859315065998, |
| "accuracy": 0.8333333333333334 |
| }, |
| { |
| "n": 16, |
| "confidence": 0.765267463088596, |
| "accuracy": 0.5625 |
| }, |
| { |
| "n": 11, |
| "confidence": 0.8288046343252776, |
| "accuracy": 0.9090909090909091 |
| }, |
| { |
| "n": 11, |
| "confidence": 0.9037337492339783, |
| "accuracy": 0.7272727272727273 |
| }, |
| { |
| "n": 29, |
| "confidence": 0.9729322257523271, |
| "accuracy": 0.9655172413793104 |
| } |
| ] |
| }, |
| "newsfin_long": { |
| "n": 56, |
| "accuracy": 0.7142857142857143, |
| "accuracy_ci95": [ |
| 0.5892857142857143, |
| 0.8214285714285714 |
| ], |
| "nll": 0.7747561322704637, |
| "brier_onehot": 0.3982446040756354, |
| "ece15": 0.1139029663017493, |
| "mean_confidence": 0.7618069910792377, |
| "uniform_accuracy": 0.24032936018230133, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 38, |
| "coverage": 0.6785714285714286, |
| "accuracy": 0.8157894736842105 |
| }, |
| "0.8": { |
| "n": 33, |
| "coverage": 0.5892857142857143, |
| "accuracy": 0.8484848484848485 |
| }, |
| "0.9": { |
| "n": 26, |
| "coverage": 0.4642857142857143, |
| "accuracy": 0.9230769230769231 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 1, |
| "confidence": 0.25787782972278783, |
| "accuracy": 0.0 |
| }, |
| { |
| "n": 2, |
| "confidence": 0.30044849334548607, |
| "accuracy": 0.0 |
| }, |
| { |
| "n": 1, |
| "confidence": 0.33464981710949976, |
| "accuracy": 1.0 |
| }, |
| { |
| "n": 5, |
| "confidence": 0.4328088221147037, |
| "accuracy": 0.6 |
| }, |
| { |
| "n": 2, |
| "confidence": 0.5085217603938686, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 6, |
| "confidence": 0.5693763805464213, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.6606546915216647, |
| "accuracy": 0.75 |
| }, |
| { |
| "n": 2, |
| "confidence": 0.689887285232544, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.8263626768434467, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 10, |
| "confidence": 0.9067653114584996, |
| "accuracy": 0.9 |
| }, |
| { |
| "n": 19, |
| "confidence": 0.9723644101980917, |
| "accuracy": 0.8947368421052632 |
| } |
| ] |
| }, |
| "heldout_tasks": { |
| "n": 300, |
| "accuracy": 0.58, |
| "accuracy_ci95": [ |
| 0.5266666666666666, |
| 0.6366666666666667 |
| ], |
| "nll": 0.8911869889628542, |
| "brier_onehot": 0.5134231622791497, |
| "ece15": 0.08311700140649199, |
| "mean_confidence": 0.6243801023474437, |
| "uniform_accuracy": 0.3491236190667541, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 137, |
| "coverage": 0.45666666666666667, |
| "accuracy": 0.7445255474452555 |
| }, |
| "0.8": { |
| "n": 64, |
| "coverage": 0.21333333333333335, |
| "accuracy": 0.8125 |
| }, |
| "0.9": { |
| "n": 37, |
| "coverage": 0.12333333333333334, |
| "accuracy": 0.8648648648648649 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 1, |
| "confidence": 0.23789036628440569, |
| "accuracy": 1.0 |
| }, |
| { |
| "n": 21, |
| "confidence": 0.29895297878815114, |
| "accuracy": 0.23809523809523808 |
| }, |
| { |
| "n": 28, |
| "confidence": 0.3638488940938339, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 28, |
| "confidence": 0.4302174204297628, |
| "accuracy": 0.25 |
| }, |
| { |
| "n": 26, |
| "confidence": 0.5017499667404475, |
| "accuracy": 0.4230769230769231 |
| }, |
| { |
| "n": 33, |
| "confidence": 0.5623847003664137, |
| "accuracy": 0.5151515151515151 |
| }, |
| { |
| "n": 36, |
| "confidence": 0.6323279372286387, |
| "accuracy": 0.6666666666666666 |
| }, |
| { |
| "n": 30, |
| "confidence": 0.700325317035411, |
| "accuracy": 0.6666666666666666 |
| }, |
| { |
| "n": 33, |
| "confidence": 0.7652276990151226, |
| "accuracy": 0.696969696969697 |
| }, |
| { |
| "n": 18, |
| "confidence": 0.8330853245824539, |
| "accuracy": 0.7777777777777778 |
| }, |
| { |
| "n": 23, |
| "confidence": 0.9067225341857484, |
| "accuracy": 0.7391304347826086 |
| }, |
| { |
| "n": 23, |
| "confidence": 0.960167099210642, |
| "accuracy": 0.9130434782608695 |
| } |
| ] |
| }, |
| "typed_decisions": { |
| "n": 2000, |
| "accuracy": 0.4595, |
| "accuracy_ci95": [ |
| 0.433, |
| 0.485 |
| ], |
| "nll": 1.3444350869309427, |
| "brier_onehot": 0.7480423473427633, |
| "ece15": 0.23062323931638637, |
| "mean_confidence": 0.683465700725639, |
| "uniform_accuracy": 0.3175, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 1103, |
| "coverage": 0.5515, |
| "accuracy": 0.5058930190389845 |
| }, |
| "0.8": { |
| "n": 870, |
| "coverage": 0.435, |
| "accuracy": 0.5149425287356322 |
| }, |
| "0.9": { |
| "n": 506, |
| "coverage": 0.253, |
| "accuracy": 0.5731225296442688 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 2, |
| "confidence": 0.2636414220883421, |
| "accuracy": 0.0 |
| }, |
| { |
| "n": 112, |
| "confidence": 0.31072671180670275, |
| "accuracy": 0.3392857142857143 |
| }, |
| { |
| "n": 220, |
| "confidence": 0.36609577130409987, |
| "accuracy": 0.38181818181818183 |
| }, |
| { |
| "n": 183, |
| "confidence": 0.43270138063245134, |
| "accuracy": 0.366120218579235 |
| }, |
| { |
| "n": 161, |
| "confidence": 0.4997503451031316, |
| "accuracy": 0.4472049689440994 |
| }, |
| { |
| "n": 126, |
| "confidence": 0.5668673187578344, |
| "accuracy": 0.46825396825396826 |
| }, |
| { |
| "n": 119, |
| "confidence": 0.6321133013126154, |
| "accuracy": 0.4369747899159664 |
| }, |
| { |
| "n": 111, |
| "confidence": 0.7002285366333068, |
| "accuracy": 0.4864864864864865 |
| }, |
| { |
| "n": 96, |
| "confidence": 0.7659850098182762, |
| "accuracy": 0.46875 |
| }, |
| { |
| "n": 203, |
| "confidence": 0.8370369211558824, |
| "accuracy": 0.4729064039408867 |
| }, |
| { |
| "n": 347, |
| "confidence": 0.9023447644701712, |
| "accuracy": 0.3919308357348703 |
| }, |
| { |
| "n": 320, |
| "confidence": 0.970245866206789, |
| "accuracy": 0.675 |
| } |
| ] |
| } |
| }, |
| "omni": { |
| "newsfin_short": { |
| "n": 140, |
| "accuracy": 0.6642857142857143, |
| "accuracy_ci95": [ |
| 0.5857142857142857, |
| 0.7428571428571429 |
| ], |
| "nll": 1.0239277480868136, |
| "brier_onehot": 0.4833709544993766, |
| "ece15": 0.086806222512248, |
| "mean_confidence": 0.6900754779093492, |
| "uniform_accuracy": 0.2794763973450739, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 86, |
| "coverage": 0.6142857142857143, |
| "accuracy": 0.7906976744186046 |
| }, |
| "0.8": { |
| "n": 56, |
| "coverage": 0.4, |
| "accuracy": 0.8392857142857143 |
| }, |
| "0.9": { |
| "n": 36, |
| "coverage": 0.2571428571428571, |
| "accuracy": 0.8611111111111112 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 2, |
| "confidence": 0.16850247623873293, |
| "accuracy": 0.0 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.25145491106610773, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 3, |
| "confidence": 0.30153553466505506, |
| "accuracy": 1.0 |
| }, |
| { |
| "n": 11, |
| "confidence": 0.36396977029923866, |
| "accuracy": 0.36363636363636365 |
| }, |
| { |
| "n": 10, |
| "confidence": 0.4277883482274337, |
| "accuracy": 0.4 |
| }, |
| { |
| "n": 7, |
| "confidence": 0.5038134981024585, |
| "accuracy": 0.42857142857142855 |
| }, |
| { |
| "n": 10, |
| "confidence": 0.5570586830396977, |
| "accuracy": 0.6 |
| }, |
| { |
| "n": 18, |
| "confidence": 0.6415175415281524, |
| "accuracy": 0.5555555555555556 |
| }, |
| { |
| "n": 9, |
| "confidence": 0.6942038665711675, |
| "accuracy": 0.7777777777777778 |
| }, |
| { |
| "n": 10, |
| "confidence": 0.7485815357473065, |
| "accuracy": 0.7 |
| }, |
| { |
| "n": 11, |
| "confidence": 0.825007934222517, |
| "accuracy": 0.8181818181818182 |
| }, |
| { |
| "n": 19, |
| "confidence": 0.8981023578758742, |
| "accuracy": 0.7894736842105263 |
| }, |
| { |
| "n": 26, |
| "confidence": 0.98324251744931, |
| "accuracy": 0.8846153846153846 |
| } |
| ] |
| }, |
| "newsfin_long": { |
| "n": 56, |
| "accuracy": 0.5535714285714286, |
| "accuracy_ci95": [ |
| 0.4107142857142857, |
| 0.6785714285714286 |
| ], |
| "nll": 1.0747286532578382, |
| "brier_onehot": 0.5316587234647131, |
| "ece15": 0.20183825996408222, |
| "mean_confidence": 0.6893829250347027, |
| "uniform_accuracy": 0.24032936018230133, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 32, |
| "coverage": 0.5714285714285714, |
| "accuracy": 0.78125 |
| }, |
| "0.8": { |
| "n": 23, |
| "coverage": 0.4107142857142857, |
| "accuracy": 0.7391304347826086 |
| }, |
| "0.9": { |
| "n": 13, |
| "coverage": 0.23214285714285715, |
| "accuracy": 0.9230769230769231 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 1, |
| "confidence": 0.17508739445476842, |
| "accuracy": 0.0 |
| }, |
| { |
| "n": 3, |
| "confidence": 0.30233710435052524, |
| "accuracy": 0.3333333333333333 |
| }, |
| { |
| "n": 6, |
| "confidence": 0.3677328588099678, |
| "accuracy": 0.16666666666666666 |
| }, |
| { |
| "n": 2, |
| "confidence": 0.44699478884245114, |
| "accuracy": 0.0 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.4922646316138709, |
| "accuracy": 0.0 |
| }, |
| { |
| "n": 5, |
| "confidence": 0.5582380460237041, |
| "accuracy": 0.4 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.622083597856155, |
| "accuracy": 0.75 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.6889762293752948, |
| "accuracy": 1.0 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.7732806664163974, |
| "accuracy": 0.75 |
| }, |
| { |
| "n": 6, |
| "confidence": 0.8408541777219157, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.8780159919507924, |
| "accuracy": 0.5 |
| }, |
| { |
| "n": 13, |
| "confidence": 0.9821660460455572, |
| "accuracy": 0.9230769230769231 |
| } |
| ] |
| }, |
| "heldout_tasks": { |
| "n": 300, |
| "accuracy": 0.6333333333333333, |
| "accuracy_ci95": [ |
| 0.58, |
| 0.69 |
| ], |
| "nll": 0.8258969806486628, |
| "brier_onehot": 0.4569257718777433, |
| "ece15": 0.09554911465107167, |
| "mean_confidence": 0.6315795431338078, |
| "uniform_accuracy": 0.3491236190667541, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 148, |
| "coverage": 0.49333333333333335, |
| "accuracy": 0.7972972972972973 |
| }, |
| "0.8": { |
| "n": 61, |
| "coverage": 0.20333333333333334, |
| "accuracy": 0.8688524590163934 |
| }, |
| "0.9": { |
| "n": 26, |
| "coverage": 0.08666666666666667, |
| "accuracy": 0.9615384615384616 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 1, |
| "confidence": 0.1046529915301365, |
| "accuracy": 0.0 |
| }, |
| { |
| "n": 4, |
| "confidence": 0.2577561216819676, |
| "accuracy": 0.75 |
| }, |
| { |
| "n": 11, |
| "confidence": 0.302752906130526, |
| "accuracy": 0.2727272727272727 |
| }, |
| { |
| "n": 27, |
| "confidence": 0.3719780866552427, |
| "accuracy": 0.4444444444444444 |
| }, |
| { |
| "n": 17, |
| "confidence": 0.43146238975131995, |
| "accuracy": 0.4117647058823529 |
| }, |
| { |
| "n": 37, |
| "confidence": 0.49454230114799697, |
| "accuracy": 0.35135135135135137 |
| }, |
| { |
| "n": 36, |
| "confidence": 0.5579886897891916, |
| "accuracy": 0.7222222222222222 |
| }, |
| { |
| "n": 29, |
| "confidence": 0.6272471234022386, |
| "accuracy": 0.41379310344827586 |
| }, |
| { |
| "n": 29, |
| "confidence": 0.6959158351866087, |
| "accuracy": 0.8275862068965517 |
| }, |
| { |
| "n": 48, |
| "confidence": 0.7644648380498819, |
| "accuracy": 0.7708333333333334 |
| }, |
| { |
| "n": 27, |
| "confidence": 0.8256353707236136, |
| "accuracy": 0.7777777777777778 |
| }, |
| { |
| "n": 17, |
| "confidence": 0.9039215980455362, |
| "accuracy": 0.9411764705882353 |
| }, |
| { |
| "n": 17, |
| "confidence": 0.9717127266067751, |
| "accuracy": 0.9411764705882353 |
| } |
| ] |
| }, |
| "typed_decisions": { |
| "n": 2000, |
| "accuracy": 0.4765, |
| "accuracy_ci95": [ |
| 0.4545, |
| 0.499 |
| ], |
| "nll": 1.0863653374222986, |
| "brier_onehot": 0.6166074547470112, |
| "ece15": 0.11012610270598122, |
| "mean_confidence": 0.5241069191428419, |
| "uniform_accuracy": 0.3175, |
| "confidence_thresholds": { |
| "0.65": { |
| "n": 523, |
| "coverage": 0.2615, |
| "accuracy": 0.7074569789674953 |
| }, |
| "0.8": { |
| "n": 251, |
| "coverage": 0.1255, |
| "accuracy": 0.6573705179282868 |
| }, |
| "0.9": { |
| "n": 100, |
| "coverage": 0.05, |
| "accuracy": 0.72 |
| } |
| }, |
| "reliability": [ |
| { |
| "n": 10, |
| "confidence": 0.25256444635115816, |
| "accuracy": 0.3 |
| }, |
| { |
| "n": 189, |
| "confidence": 0.30877873898245745, |
| "accuracy": 0.2698412698412698 |
| }, |
| { |
| "n": 472, |
| "confidence": 0.36585330883687267, |
| "accuracy": 0.2817796610169492 |
| }, |
| { |
| "n": 302, |
| "confidence": 0.42887444845445033, |
| "accuracy": 0.38741721854304634 |
| }, |
| { |
| "n": 243, |
| "confidence": 0.49993474017946105, |
| "accuracy": 0.3868312757201646 |
| }, |
| { |
| "n": 155, |
| "confidence": 0.5580824847934738, |
| "accuracy": 0.6645161290322581 |
| }, |
| { |
| "n": 218, |
| "confidence": 0.6316491082827173, |
| "accuracy": 0.7889908256880734 |
| }, |
| { |
| "n": 77, |
| "confidence": 0.6980893665549133, |
| "accuracy": 0.8441558441558441 |
| }, |
| { |
| "n": 83, |
| "confidence": 0.7678636543006925, |
| "accuracy": 0.6024096385542169 |
| }, |
| { |
| "n": 112, |
| "confidence": 0.8273256031580205, |
| "accuracy": 0.6160714285714286 |
| }, |
| { |
| "n": 69, |
| "confidence": 0.8956754913922487, |
| "accuracy": 0.6086956521739131 |
| }, |
| { |
| "n": 70, |
| "confidence": 0.9641727620937258, |
| "accuracy": 0.7714285714285715 |
| } |
| ] |
| } |
| } |
| }, |
| "speed": { |
| "short_1q": { |
| "n": 20, |
| "p50_ms": 26.517087999991418, |
| "p95_ms": 29.029050500071207, |
| "mean_ms": 27.113317650002955, |
| "min_ms": 24.377534999985073, |
| "max_ms": 29.310716000054526, |
| "samples_ms": [ |
| 26.25009999997019, |
| 25.432290000026114, |
| 26.320786000042062, |
| 28.980330999957005, |
| 26.646911999932854, |
| 28.12136200009263, |
| 25.81902299993999, |
| 26.38726400004998, |
| 25.392123000074207, |
| 28.523082999981852, |
| 26.326767000000473, |
| 28.44737800000985, |
| 28.463398999974743, |
| 29.310716000054526, |
| 28.692237999848658, |
| 24.377534999985073, |
| 28.48372800008292, |
| 25.53521799995906, |
| 29.014226000072085, |
| 25.741874000004827 |
| ], |
| "warmup_calls": 3, |
| "first_shape_call_ms": 27.415586999950392, |
| "questions_per_call": 1, |
| "window_counts": [ |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1 |
| ] |
| }, |
| "short_5q": { |
| "n": 20, |
| "p50_ms": 60.07641500002592, |
| "p95_ms": 65.5484159998764, |
| "mean_ms": 58.98180429998092, |
| "min_ms": 51.847296999994796, |
| "max_ms": 66.96372600003997, |
| "samples_ms": [ |
| 56.865289999905144, |
| 53.42559599989727, |
| 54.647222000085094, |
| 60.657620000029056, |
| 61.10556999988148, |
| 55.25912699999935, |
| 53.47630400001435, |
| 52.484391000007236, |
| 51.847296999994796, |
| 60.60912699990695, |
| 59.04960399993797, |
| 61.89990600000783, |
| 61.19320799984962, |
| 65.47392599986779, |
| 63.0494520000866, |
| 55.42915399996673, |
| 66.96372600003997, |
| 63.058288999854994, |
| 63.59757400014132, |
| 59.54370300014489 |
| ], |
| "warmup_calls": 3, |
| "first_shape_call_ms": 54.20621600001141, |
| "questions_per_call": 5, |
| "window_counts": [ |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1 |
| ] |
| }, |
| "short_10q": { |
| "n": 20, |
| "p50_ms": 104.81810399994629, |
| "p95_ms": 119.18314979989191, |
| "mean_ms": 105.92105819998778, |
| "min_ms": 88.4976249999454, |
| "max_ms": 120.43251000000055, |
| "samples_ms": [ |
| 92.77988300004836, |
| 104.6406630000547, |
| 96.89195900000414, |
| 119.11739399988619, |
| 105.96081699986826, |
| 103.6471459999575, |
| 98.7587220001842, |
| 98.4291290001238, |
| 88.4976249999454, |
| 114.96733699982542, |
| 109.29184199994779, |
| 104.99554499983788, |
| 110.70990500002154, |
| 115.22976200012636, |
| 118.5596349998832, |
| 95.96929799999998, |
| 120.43251000000055, |
| 104.30120299997725, |
| 115.14932800014321, |
| 100.09146099991995 |
| ], |
| "warmup_calls": 3, |
| "first_shape_call_ms": 93.11735100004626, |
| "questions_per_call": 10, |
| "window_counts": [ |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1 |
| ] |
| }, |
| "short_50q": { |
| "n": 20, |
| "p50_ms": 682.1585460000961, |
| "p95_ms": 754.3870993999008, |
| "mean_ms": 689.1575670499719, |
| "min_ms": 618.0243280000468, |
| "max_ms": 763.9201289998709, |
| "samples_ms": [ |
| 674.0678559999651, |
| 678.1810669999686, |
| 618.0243280000468, |
| 753.8853609999023, |
| 707.519105000074, |
| 670.8609360000537, |
| 680.0984779999908, |
| 671.1490820000563, |
| 641.9182600000113, |
| 688.8830089999374, |
| 693.8877620000312, |
| 684.2186140002013, |
| 674.6635569998034, |
| 738.8550079999732, |
| 707.0535409998229, |
| 627.7758280000398, |
| 747.2571369999059, |
| 698.2282959997974, |
| 763.9201289998709, |
| 662.7039869999862 |
| ], |
| "warmup_calls": 3, |
| "first_shape_call_ms": 596.6050259999065, |
| "questions_per_call": 50, |
| "window_counts": [ |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1 |
| ] |
| }, |
| "long_1q": { |
| "n": 20, |
| "p50_ms": 123.90287249991161, |
| "p95_ms": 131.76431719999755, |
| "mean_ms": 124.6096985999884, |
| "min_ms": 119.37196000008043, |
| "max_ms": 131.96851399993648, |
| "samples_ms": [ |
| 127.72182899993823, |
| 126.77874800010613, |
| 123.88399099995695, |
| 123.92175399986627, |
| 125.15549199997622, |
| 131.75357000000076, |
| 130.19190099998923, |
| 131.96851399993648, |
| 127.03383000007307, |
| 119.37196000008043, |
| 123.77765799988083, |
| 120.07203400003164, |
| 119.416425000054, |
| 119.82972900000277, |
| 120.96001899999465, |
| 120.67090499999722, |
| 120.97485899994354, |
| 120.59792499985633, |
| 128.58475899997757, |
| 129.5280700001058 |
| ], |
| "warmup_calls": 3, |
| "first_shape_call_ms": 122.09491500016156, |
| "questions_per_call": 1, |
| "window_counts": [ |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1 |
| ] |
| }, |
| "long_5q": { |
| "n": 20, |
| "p50_ms": 685.7518550001487, |
| "p95_ms": 728.8032338999642, |
| "mean_ms": 696.9230963500081, |
| "min_ms": 674.529241000073, |
| "max_ms": 733.069986000146, |
| "samples_ms": [ |
| 678.4065869999267, |
| 675.5267930000173, |
| 680.8560730000863, |
| 675.0147320001361, |
| 674.6946719999869, |
| 680.0351919998775, |
| 715.0993089999247, |
| 724.1639889998623, |
| 722.2430469998926, |
| 724.2127150000215, |
| 733.069986000146, |
| 728.5786679999546, |
| 719.8392059999605, |
| 713.666480000029, |
| 690.3053800001544, |
| 681.198330000143, |
| 674.529241000073, |
| 675.8001450000393, |
| 680.2929479999875, |
| 690.9284339999431 |
| ], |
| "warmup_calls": 3, |
| "first_shape_call_ms": 690.0589629999558, |
| "questions_per_call": 5, |
| "window_counts": [ |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1 |
| ] |
| }, |
| "long_10q": { |
| "n": 20, |
| "p50_ms": 1321.0661100000607, |
| "p95_ms": 1336.965719349837, |
| "mean_ms": 1321.2527787499992, |
| "min_ms": 1311.405681999986, |
| "max_ms": 1341.6646919999948, |
| "samples_ms": [ |
| 1333.539275000021, |
| 1322.840443000132, |
| 1312.4046030000045, |
| 1312.6979950000077, |
| 1326.8468310000117, |
| 1323.0493790001674, |
| 1317.6598679999643, |
| 1312.3101529999985, |
| 1316.093828000021, |
| 1311.405681999986, |
| 1311.9573989999935, |
| 1324.5337289999952, |
| 1341.6646919999948, |
| 1319.2917769999895, |
| 1324.1005910001604, |
| 1312.415898999916, |
| 1336.7184049998286, |
| 1325.0531029998456, |
| 1324.322876999986, |
| 1316.149045999964 |
| ], |
| "warmup_calls": 3, |
| "first_shape_call_ms": 1341.1980430000767, |
| "questions_per_call": 10, |
| "window_counts": [ |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1, |
| 1 |
| ] |
| }, |
| "throughput": { |
| "states": 16, |
| "questions": 48, |
| "repetitions": 3, |
| "batch_size_laya": 8, |
| "median_ms": 507.1860810000999, |
| "questions_per_second": 94.63982115863813, |
| "samples_ms": [ |
| 506.4106110000921, |
| 507.1860810000999, |
| 507.4538389999361 |
| ] |
| } |
| } |
| } |
|
|
