Buckets:
{
"dataset": "hotchpotch/S1MB-english-v1@10bcb8c6",
"benchmarks": 137,
"borda_benchmarks": 137,
"cohort": 217,
"rescore_check": [
{
"model": "5ak3t__scm_max_14b_v0",
"benchmarks_checked": 16,
"max_abs_diff": 1.1102230246251565e-16
},
{
"model": "5ak3t__scm_max_14b_v1",
"benchmarks_checked": 16,
"max_abs_diff": 4.440892098500626e-16
},
{
"model": "5ak3t__scm_mid_4b_v1",
"benchmarks_checked": 16,
"max_abs_diff": 5.551115123125783e-16
}
],
"ours": {
"base": {
"temps": {
"choice": 1.707683209388825,
"noul": 2.4105066990845945,
"score": 1.2847580560947693
},
"raw": {
"adjusted": {
"aegis2-noul-test-v1": 0.42052313883299797,
"laya--ag-news-choice-test-v1": 0.7910447761194032,
"aqua-rat-choice-test-v1": 0.042253521126760604,
"arc-choice-test-v1": 0.8823529411764708,
"arct-choice-test-v1": 0.2765957446808511,
"argument-quality-choice-test-v1": 0.28571428571428575,
"argument-quality-noul-test-v1": 0.09289526462740527,
"babi-nli-noul-test-v1": 0.10000000000000009,
"banking77-choice-test-v1": 0.631578947368421,
"bbq-choice-test-v1": 0.8999999999999999,
"boardgameqa-choice-test-v1": 0.25000000000000006,
"canttalk-noul-test-v1": 0.7200000000000002,
"civil-comments-noul-test-v1": 0.3041383169192424,
"cladder-noul-test-v1": 0.040000000000000036,
"clinc-choice-test-v1": 0.7931034482758621,
"contract-nli-choice-test-v1": 0.5173535791757051,
"corr2cause-choice-test-v1": 0.0,
"creak-noul-test-v1": 0.72,
"crows-pairs-choice-test-v1": 0.43749999999999994,
"dbpedia-choice-test-v1": 0.9775280898876404,
"defeasible-nli-choice-test-v1": 0.41304347826086946,
"laya--enron-spam-noul-test-v1": 0.76,
"esci-choice-test-v1": 0.0821917808219178,
"ethics-choice-test-v1": 0.5000000000000001,
"ethics-noul-test-v1": 0.1595238095238094,
"ethos-noul-test-v1": 0.45999999999999996,
"few-nerd-choice-test-v1": 0.6451612903225806,
"fol-nli-choice-test-v1": 0.16393442622950816,
"followir-core17-noul-test-v1": 0.26398601398601396,
"followir-news21-noul-test-v1": 0.03333333333333344,
"followir-robust04-noul-test-v1": 0.4736842105263157,
"go-emotions-noul-test-v1": 0.3630486674285902,
"gretel-pii-choice-test-v1": 0.6688311688311688,
"gsm8k-choice-test-v1": 0.12857142857142861,
"hans-noul-test-v1": 0.5,
"hatecheck-noul-test-v1": 0.72,
"hh-rlhf-choice-test-v1": 0.07692307692307698,
"hwu64-choice-test-v1": 0.797872340425532,
"impli-noul-test-v1": 0.28,
"lexcomp-noul-test-v1": 0.3799999999999999,
"logical-entailment-noul-test-v1": 0.040000000000000036,
"lonli-choice-test-v1": 0.540983606557377,
"massive-choice-test-v1": 0.7806215722120659,
"miqa-choice-test-v1": 0.7272727272727273,
"mtop-choice-test-v1": 0.74873660804528,
"nlsat-noul-test-v1": 0.0,
"open-jev--amount-extraction-control-v1-choice-test-v1": 0.5294117647058824,
"open-jev--amount-extraction-control-v1-noul-test-v1": 0.0,
"open-jev--browser-control-v1-choice-test-v1": 1.0,
"open-jev--citation-control-v1-choice-test-v1": 0.6363636363636365,
"open-jev--context-retention-control-v1-noul-test-v1": 0.0,
"open-jev--customer-control-v1-choice-test-v1": 0.8260869565217391,
"open-jev--customer-control-v1-noul-test-v1": 0.75,
"open-jev--customer-control-v1-score-test-v1": 0.39788644560460573,
"open-jev--drone-control-v1-choice-test-v1": 0.0,
"open-jev--drone-control-v1-noul-test-v1": 0.0,
"open-jev--drone-control-v1-score-test-v1": 0.0,
"open-jev--email-selection-control-v1-choice-test-v1": 0.0,
"open-jev--email-selection-control-v1-noul-test-v1": 0.5,
"open-jev--entity-alignment-control-v1-noul-test-v1": 0.0,
"open-jev--entity-alignment-control-v1-score-test-v1": 0.0,
"open-jev--ir-control-v1-choice-test-v1": 0.6799999999999998,
"open-jev--ir-control-v1-noul-test-v1": 0.5714285714285714,
"open-jev--ir-control-v1-score-test-v1": 0.0,
"open-jev--mailroom-control-v1-choice-test-v1": 1.0,
"open-jev--mailroom-control-v1-noul-test-v1": 0.7305555555555556,
"open-jev--painting-geometry-v1-choice-test-v1": 0.5263157894736842,
"open-jev--painting-geometry-v1-noul-test-v1": 0.1333333333333333,
"open-jev--painting-geometry-v1-score-test-v1": 0.13055902085337423,
"open-jev--phone-extraction-control-v1-choice-test-v1": 0.391304347826087,
"open-jev--phone-extraction-control-v1-noul-test-v1": 0.6000000000000001,
"open-jev--reasoning-control-v1-choice-test-v1": 0.050000000000000044,
"open-jev--reasoning-control-v1-noul-test-v1": 0.05514705882352944,
"open-jev--reasoning-control-v1-score-test-v1": 0.17067740907990736,
"open-jev--silent-failure-control-v1-noul-test-v1": 0.06000000000000005,
"open-jev--snake-v1-choice-test-v1": 0.0,
"open-jev--snake-v1-noul-test-v1": 0.0,
"open-jev--sponsor-segment-control-v1-choice-test-v1": 0.923076923076923,
"open-jev--tic-tac-toe-v1-choice-test-v1": 0.030303030303030165,
"open-jev--vizdoom-basic-v1-noul-test-v1": 0.06617647058823528,
"open-jev--vizdoom-basic-v1-score-test-v1": 0.1997340624720444,
"open-jev--workflow-controls-v1--agent-trace-observability-noul-test-v1": 0.2027972027972027,
"open-jev--workflow-controls-v1--customer-service-noul-test-v1": 0.36250536250536247,
"open-jev--workflow-controls-v1--invoice-processing-noul-test-v1": 0.02564102564102555,
"open-jev--workflow-controls-v1--security-incidents-noul-test-v1": 0.35811126694717155,
"openbookqa-choice-test-v1": 0.7361111111111112,
"patent-similarity-score-test-v1": 0.3279628341067954,
"paws-noul-test-v1": 0.5800000000000001,
"plane-noul-test-v1": 0.0,
"poem-sentiment-choice-test-v1": 0.0,
"qasper-noul-test-v1": 0.6599999999999999,
"quartz-choice-test-v1": 0.47619047619047633,
"robust-lr-choice-test-v1": 0.18333333333333332,
"ruletaker-noul-test-v1": 0.26,
"s1mb-generalization-contextual-choice-choice-test-v1": 0.9484978540772532,
"s1mb-generalization-contextual-noul-noul-test-v1": 0.8199999999999998,
"s1mb-generalization-contextual-score-score-test-v1": 0.5638258899221804,
"s1mb-generalization-diverse-choice-choice-test-v1": 0.86784140969163,
"s1mb-generalization-diverse-noul-noul-test-v1": 0.8399999999999999,
"s1mb-generalization-diverse-score-score-test-v1": 0.612037228445995,
"scicite-choice-test-v1": 0.532258064516129,
"scitail-noul-test-v1": 0.8599999999999999,
"scone-choice-test-v1": 0.56,
"sdoh-nli-noul-test-v1": 0.8999999999999999,
"sgd-choice-test-v1": 0.23728813559322043,
"snli-choice-test-v1": 0.5967741935483871,
"spartqa-choice-test-v1": 0.0,
"stepgame-choice-test-v1": 0.25,
"synthetic-relevance--nanobeir--nanoarguana-noul-test-v1": 0.1322072402534249,
"synthetic-relevance--nanobeir--nanoarguana-score-test-v1": 0.07273280228740797,
"synthetic-relevance--nanobeir--nanodbpedia-noul-test-v1": 0.3597289535156989,
"synthetic-relevance--nanobeir--nanodbpedia-score-test-v1": 0.2845582024743629,
"synthetic-relevance--nanobeir--nanofiqa2018-noul-test-v1": 0.29607812321130966,
"synthetic-relevance--nanobeir--nanofiqa2018-score-test-v1": 0.26667314457785307,
"synthetic-relevance--nanobeir--nanohotpotqa-noul-test-v1": 0.18512359296340297,
"synthetic-relevance--nanobeir--nanohotpotqa-score-test-v1": 0.20288698878246914,
"synthetic-relevance--nanobeir--nanomsmarco-noul-test-v1": 0.48737468820639496,
"synthetic-relevance--nanobeir--nanomsmarco-score-test-v1": 0.3007731311195775,
"synthetic-relevance--nanobeir--nanonfcorpus-noul-test-v1": 0.34625575841540024,
"synthetic-relevance--nanobeir--nanonfcorpus-score-test-v1": 0.20170169853082132,
"synthetic-relevance--nanobeir--nanonq-noul-test-v1": 0.4997183632167408,
"synthetic-relevance--nanobeir--nanonq-score-test-v1": 0.038876627340836856,
"synthetic-relevance--nanobeir--nanoscidocs-noul-test-v1": 0.14743786589533547,
"synthetic-relevance--nanobeir--nanoscidocs-score-test-v1": 0.1920301308546687,
"synthetic-relevance--nanobeir--nanotouche2020-noul-test-v1": 0.2508413593010428,
"synthetic-relevance--nanobeir--nanotouche2020-score-test-v1": 0.41153895739532786,
"synthetic-relevance--nanocoir-noul-test-v1": 0.28739000806659165,
"synthetic-relevance--nanocoir-score-test-v1": 0.0,
"temporal-nli-choice-test-v1": 0.08064516129032256,
"tracie-noul-test-v1": 0.0,
"laya--typed-decisions-choice-test-v1": 0.22748534707752452,
"laya--typed-decisions-noul-test-v1": 0.3635856680955216,
"laya--typed-decisions-score-test-v1": 0.2128120162362931,
"ud-ewt-choice-test-v1": 0.18302180685358257,
"winobias-choice-test-v1": 0.21739130434782605,
"winowhy-noul-test-v1": 0.3999999999999999,
"wiqa-choice-test-v1": 0.3015873015873017
},
"noul": 0.3376706817616705,
"choice": 0.4570255576401406,
"score": 0.21844126619450097,
"task_avg": 0.3377125018654374
},
"calibrated": {
"adjusted": {
"aegis2-noul-test-v1": 0.42052313883299797,
"laya--ag-news-choice-test-v1": 0.7910447761194032,
"aqua-rat-choice-test-v1": 0.042253521126760604,
"arc-choice-test-v1": 0.8823529411764708,
"arct-choice-test-v1": 0.2765957446808511,
"argument-quality-choice-test-v1": 0.28571428571428575,
"argument-quality-noul-test-v1": 0.09289526462740527,
"babi-nli-noul-test-v1": 0.10000000000000009,
"banking77-choice-test-v1": 0.631578947368421,
"bbq-choice-test-v1": 0.8999999999999999,
"boardgameqa-choice-test-v1": 0.25000000000000006,
"canttalk-noul-test-v1": 0.7200000000000002,
"civil-comments-noul-test-v1": 0.3041383169192424,
"cladder-noul-test-v1": 0.040000000000000036,
"clinc-choice-test-v1": 0.7931034482758621,
"contract-nli-choice-test-v1": 0.5173535791757051,
"corr2cause-choice-test-v1": 0.0,
"creak-noul-test-v1": 0.72,
"crows-pairs-choice-test-v1": 0.43749999999999994,
"dbpedia-choice-test-v1": 0.9775280898876404,
"defeasible-nli-choice-test-v1": 0.41304347826086946,
"laya--enron-spam-noul-test-v1": 0.76,
"esci-choice-test-v1": 0.0821917808219178,
"ethics-choice-test-v1": 0.5000000000000001,
"ethics-noul-test-v1": 0.1595238095238094,
"ethos-noul-test-v1": 0.45999999999999996,
"few-nerd-choice-test-v1": 0.6451612903225806,
"fol-nli-choice-test-v1": 0.16393442622950816,
"followir-core17-noul-test-v1": 0.26398601398601396,
"followir-news21-noul-test-v1": 0.03333333333333344,
"followir-robust04-noul-test-v1": 0.4736842105263157,
"go-emotions-noul-test-v1": 0.3630486674285902,
"gretel-pii-choice-test-v1": 0.6688311688311688,
"gsm8k-choice-test-v1": 0.12857142857142861,
"hans-noul-test-v1": 0.5,
"hatecheck-noul-test-v1": 0.72,
"hh-rlhf-choice-test-v1": 0.07692307692307698,
"hwu64-choice-test-v1": 0.797872340425532,
"impli-noul-test-v1": 0.28,
"lexcomp-noul-test-v1": 0.3799999999999999,
"logical-entailment-noul-test-v1": 0.040000000000000036,
"lonli-choice-test-v1": 0.540983606557377,
"massive-choice-test-v1": 0.7806215722120659,
"miqa-choice-test-v1": 0.7272727272727273,
"mtop-choice-test-v1": 0.74873660804528,
"nlsat-noul-test-v1": 0.0,
"open-jev--amount-extraction-control-v1-choice-test-v1": 0.5294117647058824,
"open-jev--amount-extraction-control-v1-noul-test-v1": 0.0,
"open-jev--browser-control-v1-choice-test-v1": 1.0,
"open-jev--citation-control-v1-choice-test-v1": 0.6363636363636365,
"open-jev--context-retention-control-v1-noul-test-v1": 0.0,
"open-jev--customer-control-v1-choice-test-v1": 0.8260869565217391,
"open-jev--customer-control-v1-noul-test-v1": 0.75,
"open-jev--customer-control-v1-score-test-v1": 0.3893950629833327,
"open-jev--drone-control-v1-choice-test-v1": 0.0,
"open-jev--drone-control-v1-noul-test-v1": 0.0,
"open-jev--drone-control-v1-score-test-v1": 0.0,
"open-jev--email-selection-control-v1-choice-test-v1": 0.0,
"open-jev--email-selection-control-v1-noul-test-v1": 0.5,
"open-jev--entity-alignment-control-v1-noul-test-v1": 0.0,
"open-jev--entity-alignment-control-v1-score-test-v1": 0.0,
"open-jev--ir-control-v1-choice-test-v1": 0.6799999999999998,
"open-jev--ir-control-v1-noul-test-v1": 0.5714285714285714,
"open-jev--ir-control-v1-score-test-v1": 0.0,
"open-jev--mailroom-control-v1-choice-test-v1": 1.0,
"open-jev--mailroom-control-v1-noul-test-v1": 0.7305555555555556,
"open-jev--painting-geometry-v1-choice-test-v1": 0.5263157894736842,
"open-jev--painting-geometry-v1-noul-test-v1": 0.1333333333333333,
"open-jev--painting-geometry-v1-score-test-v1": 0.11001724500712606,
"open-jev--phone-extraction-control-v1-choice-test-v1": 0.391304347826087,
"open-jev--phone-extraction-control-v1-noul-test-v1": 0.6000000000000001,
"open-jev--reasoning-control-v1-choice-test-v1": 0.050000000000000044,
"open-jev--reasoning-control-v1-noul-test-v1": 0.05514705882352944,
"open-jev--reasoning-control-v1-score-test-v1": 0.12618636261103344,
"open-jev--silent-failure-control-v1-noul-test-v1": 0.06000000000000005,
"open-jev--snake-v1-choice-test-v1": 0.0,
"open-jev--snake-v1-noul-test-v1": 0.0,
"open-jev--sponsor-segment-control-v1-choice-test-v1": 0.923076923076923,
"open-jev--tic-tac-toe-v1-choice-test-v1": 0.030303030303030165,
"open-jev--vizdoom-basic-v1-noul-test-v1": 0.06617647058823528,
"open-jev--vizdoom-basic-v1-score-test-v1": 0.1553597082025594,
"open-jev--workflow-controls-v1--agent-trace-observability-noul-test-v1": 0.2027972027972027,
"open-jev--workflow-controls-v1--customer-service-noul-test-v1": 0.36250536250536247,
"open-jev--workflow-controls-v1--invoice-processing-noul-test-v1": 0.02564102564102555,
"open-jev--workflow-controls-v1--security-incidents-noul-test-v1": 0.35811126694717155,
"openbookqa-choice-test-v1": 0.7361111111111112,
"patent-similarity-score-test-v1": 0.2934810047042089,
"paws-noul-test-v1": 0.5800000000000001,
"plane-noul-test-v1": 0.0,
"poem-sentiment-choice-test-v1": 0.0,
"qasper-noul-test-v1": 0.6599999999999999,
"quartz-choice-test-v1": 0.47619047619047633,
"robust-lr-choice-test-v1": 0.18333333333333332,
"ruletaker-noul-test-v1": 0.26,
"s1mb-generalization-contextual-choice-choice-test-v1": 0.9484978540772532,
"s1mb-generalization-contextual-noul-noul-test-v1": 0.8199999999999998,
"s1mb-generalization-contextual-score-score-test-v1": 0.5348292656817324,
"s1mb-generalization-diverse-choice-choice-test-v1": 0.86784140969163,
"s1mb-generalization-diverse-noul-noul-test-v1": 0.8399999999999999,
"s1mb-generalization-diverse-score-score-test-v1": 0.5785965969396042,
"scicite-choice-test-v1": 0.532258064516129,
"scitail-noul-test-v1": 0.8599999999999999,
"scone-choice-test-v1": 0.56,
"sdoh-nli-noul-test-v1": 0.8999999999999999,
"sgd-choice-test-v1": 0.23728813559322043,
"snli-choice-test-v1": 0.5967741935483871,
"spartqa-choice-test-v1": 0.0,
"stepgame-choice-test-v1": 0.25,
"synthetic-relevance--nanobeir--nanoarguana-noul-test-v1": 0.1322072402534249,
"synthetic-relevance--nanobeir--nanoarguana-score-test-v1": 0.009914357195622725,
"synthetic-relevance--nanobeir--nanodbpedia-noul-test-v1": 0.3597289535156989,
"synthetic-relevance--nanobeir--nanodbpedia-score-test-v1": 0.2412115992486591,
"synthetic-relevance--nanobeir--nanofiqa2018-noul-test-v1": 0.29607812321130966,
"synthetic-relevance--nanobeir--nanofiqa2018-score-test-v1": 0.23325885856088935,
"synthetic-relevance--nanobeir--nanohotpotqa-noul-test-v1": 0.18512359296340297,
"synthetic-relevance--nanobeir--nanohotpotqa-score-test-v1": 0.10831095033900406,
"synthetic-relevance--nanobeir--nanomsmarco-noul-test-v1": 0.48737468820639496,
"synthetic-relevance--nanobeir--nanomsmarco-score-test-v1": 0.2474660472587129,
"synthetic-relevance--nanobeir--nanonfcorpus-noul-test-v1": 0.34625575841540024,
"synthetic-relevance--nanobeir--nanonfcorpus-score-test-v1": 0.13921800746762414,
"synthetic-relevance--nanobeir--nanonq-noul-test-v1": 0.4997183632167408,
"synthetic-relevance--nanobeir--nanonq-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanoscidocs-noul-test-v1": 0.14743786589533547,
"synthetic-relevance--nanobeir--nanoscidocs-score-test-v1": 0.11267844076262346,
"synthetic-relevance--nanobeir--nanotouche2020-noul-test-v1": 0.2508413593010428,
"synthetic-relevance--nanobeir--nanotouche2020-score-test-v1": 0.40182339219261376,
"synthetic-relevance--nanocoir-noul-test-v1": 0.28739000806659165,
"synthetic-relevance--nanocoir-score-test-v1": 0.0,
"temporal-nli-choice-test-v1": 0.08064516129032256,
"tracie-noul-test-v1": 0.0,
"laya--typed-decisions-choice-test-v1": 0.22748534707752452,
"laya--typed-decisions-noul-test-v1": 0.3635856680955216,
"laya--typed-decisions-score-test-v1": 0.28352172500829476,
"ud-ewt-choice-test-v1": 0.18302180685358257,
"winobias-choice-test-v1": 0.21739130434782605,
"winowhy-noul-test-v1": 0.3999999999999999,
"wiqa-choice-test-v1": 0.3015873015873017
},
"noul": 0.3376706817616705,
"choice": 0.4570255576401406,
"score": 0.18882231543636388,
"task_avg": 0.3278395182793917
}
},
"base_2b": {
"temps": {
"choice": 4.78807442670338,
"noul": 19.999999999982688,
"score": 9.956643342471382
},
"raw": {
"adjusted": {
"aegis2-noul-test-v1": 0.15675873422352282,
"laya--ag-news-choice-test-v1": 0.8507462686567167,
"aqua-rat-choice-test-v1": 0.0,
"arc-choice-test-v1": 0.6323529411764707,
"arct-choice-test-v1": 0.34042553191489344,
"argument-quality-choice-test-v1": 0.0,
"argument-quality-noul-test-v1": 0.0,
"babi-nli-noul-test-v1": 0.3799999999999999,
"banking77-choice-test-v1": 0.5263157894736842,
"bbq-choice-test-v1": 0.4999999999999999,
"boardgameqa-choice-test-v1": 0.04687500000000004,
"canttalk-noul-test-v1": 0.48,
"civil-comments-noul-test-v1": 0.20250058010226324,
"cladder-noul-test-v1": 0.0,
"clinc-choice-test-v1": 0.6827586206896552,
"contract-nli-choice-test-v1": 0.08351409978308025,
"corr2cause-choice-test-v1": 0.0,
"creak-noul-test-v1": 0.10000000000000009,
"crows-pairs-choice-test-v1": 0.14583333333333323,
"dbpedia-choice-test-v1": 0.8988764044943821,
"defeasible-nli-choice-test-v1": 0.0,
"laya--enron-spam-noul-test-v1": 0.08000000000000007,
"esci-choice-test-v1": 0.027397260273972553,
"ethics-choice-test-v1": 0.0,
"ethics-noul-test-v1": 0.22857142857142865,
"ethos-noul-test-v1": 0.3999999999999999,
"few-nerd-choice-test-v1": 0.5232974910394266,
"fol-nli-choice-test-v1": 0.0,
"followir-core17-noul-test-v1": 0.2709790209790208,
"followir-news21-noul-test-v1": 0.006666666666666599,
"followir-robust04-noul-test-v1": 0.4294853154986915,
"go-emotions-noul-test-v1": 0.40216633455360284,
"gretel-pii-choice-test-v1": 0.4707792207792207,
"gsm8k-choice-test-v1": 0.07142857142857141,
"hans-noul-test-v1": 0.43999999999999995,
"hatecheck-noul-test-v1": 0.3799999999999999,
"hh-rlhf-choice-test-v1": 0.07692307692307698,
"hwu64-choice-test-v1": 0.5957446808510639,
"impli-noul-test-v1": 0.1200000000000001,
"lexcomp-noul-test-v1": 0.0,
"logical-entailment-noul-test-v1": 0.020000000000000018,
"lonli-choice-test-v1": 0.278688524590164,
"massive-choice-test-v1": 0.6325411334552102,
"miqa-choice-test-v1": 0.6136363636363635,
"mtop-choice-test-v1": 0.571710127349909,
"nlsat-noul-test-v1": 0.0,
"open-jev--amount-extraction-control-v1-choice-test-v1": 0.49019607843137253,
"open-jev--amount-extraction-control-v1-noul-test-v1": 0.125,
"open-jev--browser-control-v1-choice-test-v1": 0.9782608695652174,
"open-jev--citation-control-v1-choice-test-v1": 0.31818181818181823,
"open-jev--context-retention-control-v1-noul-test-v1": 0.020000000000000018,
"open-jev--customer-control-v1-choice-test-v1": 0.8260869565217391,
"open-jev--customer-control-v1-noul-test-v1": 0.5661764705882353,
"open-jev--customer-control-v1-score-test-v1": 0.1920571977220239,
"open-jev--drone-control-v1-choice-test-v1": 0.0,
"open-jev--drone-control-v1-noul-test-v1": 0.0,
"open-jev--drone-control-v1-score-test-v1": 0.0,
"open-jev--email-selection-control-v1-choice-test-v1": 0.11304347826086972,
"open-jev--email-selection-control-v1-noul-test-v1": 0.11538461538461542,
"open-jev--entity-alignment-control-v1-noul-test-v1": 0.0,
"open-jev--entity-alignment-control-v1-score-test-v1": 0.0,
"open-jev--ir-control-v1-choice-test-v1": 0.64,
"open-jev--ir-control-v1-noul-test-v1": 0.2142857142857144,
"open-jev--ir-control-v1-score-test-v1": 0.0,
"open-jev--mailroom-control-v1-choice-test-v1": 0.8333333333333334,
"open-jev--mailroom-control-v1-noul-test-v1": 0.5027777777777778,
"open-jev--painting-geometry-v1-choice-test-v1": 0.05263157894736847,
"open-jev--painting-geometry-v1-noul-test-v1": 0.19999999999999996,
"open-jev--painting-geometry-v1-score-test-v1": 0.2059003284578289,
"open-jev--phone-extraction-control-v1-choice-test-v1": 0.13043478260869568,
"open-jev--phone-extraction-control-v1-noul-test-v1": 0.0,
"open-jev--reasoning-control-v1-choice-test-v1": 0.050000000000000044,
"open-jev--reasoning-control-v1-noul-test-v1": 0.40441176470588225,
"open-jev--reasoning-control-v1-score-test-v1": 0.01836116151128586,
"open-jev--silent-failure-control-v1-noul-test-v1": 0.0,
"open-jev--snake-v1-choice-test-v1": 0.0,
"open-jev--snake-v1-noul-test-v1": 0.03556187766714092,
"open-jev--sponsor-segment-control-v1-choice-test-v1": 0.5256410256410257,
"open-jev--tic-tac-toe-v1-choice-test-v1": 0.022727272727272582,
"open-jev--vizdoom-basic-v1-noul-test-v1": 0.0,
"open-jev--vizdoom-basic-v1-score-test-v1": 0.0,
"open-jev--workflow-controls-v1--agent-trace-observability-noul-test-v1": 0.0862470862470861,
"open-jev--workflow-controls-v1--customer-service-noul-test-v1": 0.5057915057915059,
"open-jev--workflow-controls-v1--invoice-processing-noul-test-v1": 0.0,
"open-jev--workflow-controls-v1--security-incidents-noul-test-v1": 0.22907900888265553,
"openbookqa-choice-test-v1": 0.25,
"patent-similarity-score-test-v1": 0.0,
"paws-noul-test-v1": 0.06000000000000005,
"plane-noul-test-v1": 0.21999999999999997,
"poem-sentiment-choice-test-v1": 0.0,
"qasper-noul-test-v1": 0.48,
"quartz-choice-test-v1": 0.23809523809523828,
"robust-lr-choice-test-v1": 0.0,
"ruletaker-noul-test-v1": 0.0,
"s1mb-generalization-contextual-choice-choice-test-v1": 0.8326180257510731,
"s1mb-generalization-contextual-noul-noul-test-v1": 0.7,
"s1mb-generalization-contextual-score-score-test-v1": 0.09813882814177977,
"s1mb-generalization-diverse-choice-choice-test-v1": 0.7753303964757708,
"s1mb-generalization-diverse-noul-noul-test-v1": 0.45999999999999996,
"s1mb-generalization-diverse-score-score-test-v1": 0.24718416049574388,
"scicite-choice-test-v1": 0.2580645161290323,
"scitail-noul-test-v1": 0.19999999999999996,
"scone-choice-test-v1": 0.26,
"sdoh-nli-noul-test-v1": 0.15999999999999992,
"sgd-choice-test-v1": 0.22033898305084754,
"snli-choice-test-v1": 0.16129032258064513,
"spartqa-choice-test-v1": 0.0,
"stepgame-choice-test-v1": 0.09523809523809523,
"synthetic-relevance--nanobeir--nanoarguana-noul-test-v1": 0.25962328735435114,
"synthetic-relevance--nanobeir--nanoarguana-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanodbpedia-noul-test-v1": 0.4192016643984635,
"synthetic-relevance--nanobeir--nanodbpedia-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanofiqa2018-noul-test-v1": 0.40944448001614253,
"synthetic-relevance--nanobeir--nanofiqa2018-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanohotpotqa-noul-test-v1": 0.32422648339838234,
"synthetic-relevance--nanobeir--nanohotpotqa-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanomsmarco-noul-test-v1": 0.4970835009297585,
"synthetic-relevance--nanobeir--nanomsmarco-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanonfcorpus-noul-test-v1": 0.3791955899378503,
"synthetic-relevance--nanobeir--nanonfcorpus-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanonq-noul-test-v1": 0.5139497440345342,
"synthetic-relevance--nanobeir--nanonq-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanoscidocs-noul-test-v1": 0.32055155471466357,
"synthetic-relevance--nanobeir--nanoscidocs-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanotouche2020-noul-test-v1": 0.36999631490467055,
"synthetic-relevance--nanobeir--nanotouche2020-score-test-v1": 0.0,
"synthetic-relevance--nanocoir-noul-test-v1": 0.44900362569846064,
"synthetic-relevance--nanocoir-score-test-v1": 0.0,
"temporal-nli-choice-test-v1": 0.0,
"tracie-noul-test-v1": 0.0,
"laya--typed-decisions-choice-test-v1": 0.17588299359671944,
"laya--typed-decisions-noul-test-v1": 0.17023925810898288,
"laya--typed-decisions-score-test-v1": 0.0,
"ud-ewt-choice-test-v1": 0.0,
"winobias-choice-test-v1": 0.0,
"winowhy-noul-test-v1": 0.0,
"wiqa-choice-test-v1": 0.015873015873015886
},
"noul": 0.2287179560241029,
"choice": 0.2953177758045324,
"score": 0.03626865125374582,
"task_avg": 0.186768127694127
},
"calibrated": {
"adjusted": {
"aegis2-noul-test-v1": 0.15675873422352282,
"laya--ag-news-choice-test-v1": 0.8507462686567167,
"aqua-rat-choice-test-v1": 0.0,
"arc-choice-test-v1": 0.6323529411764707,
"arct-choice-test-v1": 0.34042553191489344,
"argument-quality-choice-test-v1": 0.0,
"argument-quality-noul-test-v1": 0.0,
"babi-nli-noul-test-v1": 0.3799999999999999,
"banking77-choice-test-v1": 0.5263157894736842,
"bbq-choice-test-v1": 0.4999999999999999,
"boardgameqa-choice-test-v1": 0.04687500000000004,
"canttalk-noul-test-v1": 0.48,
"civil-comments-noul-test-v1": 0.20250058010226324,
"cladder-noul-test-v1": 0.0,
"clinc-choice-test-v1": 0.6827586206896552,
"contract-nli-choice-test-v1": 0.08351409978308025,
"corr2cause-choice-test-v1": 0.0,
"creak-noul-test-v1": 0.10000000000000009,
"crows-pairs-choice-test-v1": 0.14583333333333323,
"dbpedia-choice-test-v1": 0.8988764044943821,
"defeasible-nli-choice-test-v1": 0.0,
"laya--enron-spam-noul-test-v1": 0.08000000000000007,
"esci-choice-test-v1": 0.027397260273972553,
"ethics-choice-test-v1": 0.0,
"ethics-noul-test-v1": 0.22857142857142865,
"ethos-noul-test-v1": 0.3999999999999999,
"few-nerd-choice-test-v1": 0.5232974910394266,
"fol-nli-choice-test-v1": 0.0,
"followir-core17-noul-test-v1": 0.2709790209790208,
"followir-news21-noul-test-v1": 0.006666666666666599,
"followir-robust04-noul-test-v1": 0.4294853154986915,
"go-emotions-noul-test-v1": 0.40216633455360284,
"gretel-pii-choice-test-v1": 0.4707792207792207,
"gsm8k-choice-test-v1": 0.07142857142857141,
"hans-noul-test-v1": 0.43999999999999995,
"hatecheck-noul-test-v1": 0.3799999999999999,
"hh-rlhf-choice-test-v1": 0.07692307692307698,
"hwu64-choice-test-v1": 0.5957446808510639,
"impli-noul-test-v1": 0.1200000000000001,
"lexcomp-noul-test-v1": 0.0,
"logical-entailment-noul-test-v1": 0.020000000000000018,
"lonli-choice-test-v1": 0.278688524590164,
"massive-choice-test-v1": 0.6325411334552102,
"miqa-choice-test-v1": 0.6136363636363635,
"mtop-choice-test-v1": 0.571710127349909,
"nlsat-noul-test-v1": 0.0,
"open-jev--amount-extraction-control-v1-choice-test-v1": 0.49019607843137253,
"open-jev--amount-extraction-control-v1-noul-test-v1": 0.125,
"open-jev--browser-control-v1-choice-test-v1": 0.9782608695652174,
"open-jev--citation-control-v1-choice-test-v1": 0.31818181818181823,
"open-jev--context-retention-control-v1-noul-test-v1": 0.020000000000000018,
"open-jev--customer-control-v1-choice-test-v1": 0.8260869565217391,
"open-jev--customer-control-v1-noul-test-v1": 0.5661764705882353,
"open-jev--customer-control-v1-score-test-v1": 0.02160936177215622,
"open-jev--drone-control-v1-choice-test-v1": 0.0,
"open-jev--drone-control-v1-noul-test-v1": 0.0,
"open-jev--drone-control-v1-score-test-v1": 0.0,
"open-jev--email-selection-control-v1-choice-test-v1": 0.11304347826086972,
"open-jev--email-selection-control-v1-noul-test-v1": 0.11538461538461542,
"open-jev--entity-alignment-control-v1-noul-test-v1": 0.0,
"open-jev--entity-alignment-control-v1-score-test-v1": 0.0,
"open-jev--ir-control-v1-choice-test-v1": 0.64,
"open-jev--ir-control-v1-noul-test-v1": 0.2142857142857144,
"open-jev--ir-control-v1-score-test-v1": 0.0,
"open-jev--mailroom-control-v1-choice-test-v1": 0.8333333333333334,
"open-jev--mailroom-control-v1-noul-test-v1": 0.5027777777777778,
"open-jev--painting-geometry-v1-choice-test-v1": 0.05263157894736847,
"open-jev--painting-geometry-v1-noul-test-v1": 0.19999999999999996,
"open-jev--painting-geometry-v1-score-test-v1": 0.03270065534985134,
"open-jev--phone-extraction-control-v1-choice-test-v1": 0.13043478260869568,
"open-jev--phone-extraction-control-v1-noul-test-v1": 0.0,
"open-jev--reasoning-control-v1-choice-test-v1": 0.050000000000000044,
"open-jev--reasoning-control-v1-noul-test-v1": 0.40441176470588225,
"open-jev--reasoning-control-v1-score-test-v1": 0.0,
"open-jev--silent-failure-control-v1-noul-test-v1": 0.0,
"open-jev--snake-v1-choice-test-v1": 0.0,
"open-jev--snake-v1-noul-test-v1": 0.03556187766714092,
"open-jev--sponsor-segment-control-v1-choice-test-v1": 0.5256410256410257,
"open-jev--tic-tac-toe-v1-choice-test-v1": 0.022727272727272582,
"open-jev--vizdoom-basic-v1-noul-test-v1": 0.0,
"open-jev--vizdoom-basic-v1-score-test-v1": 0.0,
"open-jev--workflow-controls-v1--agent-trace-observability-noul-test-v1": 0.0862470862470861,
"open-jev--workflow-controls-v1--customer-service-noul-test-v1": 0.5057915057915059,
"open-jev--workflow-controls-v1--invoice-processing-noul-test-v1": 0.0,
"open-jev--workflow-controls-v1--security-incidents-noul-test-v1": 0.22907900888265553,
"openbookqa-choice-test-v1": 0.25,
"patent-similarity-score-test-v1": 0.0,
"paws-noul-test-v1": 0.06000000000000005,
"plane-noul-test-v1": 0.21999999999999997,
"poem-sentiment-choice-test-v1": 0.0,
"qasper-noul-test-v1": 0.48,
"quartz-choice-test-v1": 0.23809523809523828,
"robust-lr-choice-test-v1": 0.0,
"ruletaker-noul-test-v1": 0.0,
"s1mb-generalization-contextual-choice-choice-test-v1": 0.8326180257510731,
"s1mb-generalization-contextual-noul-noul-test-v1": 0.7,
"s1mb-generalization-contextual-score-score-test-v1": 0.08664620462166517,
"s1mb-generalization-diverse-choice-choice-test-v1": 0.7753303964757708,
"s1mb-generalization-diverse-noul-noul-test-v1": 0.45999999999999996,
"s1mb-generalization-diverse-score-score-test-v1": 0.1398067606433141,
"scicite-choice-test-v1": 0.2580645161290323,
"scitail-noul-test-v1": 0.19999999999999996,
"scone-choice-test-v1": 0.26,
"sdoh-nli-noul-test-v1": 0.15999999999999992,
"sgd-choice-test-v1": 0.22033898305084754,
"snli-choice-test-v1": 0.16129032258064513,
"spartqa-choice-test-v1": 0.0,
"stepgame-choice-test-v1": 0.09523809523809523,
"synthetic-relevance--nanobeir--nanoarguana-noul-test-v1": 0.25962328735435114,
"synthetic-relevance--nanobeir--nanoarguana-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanodbpedia-noul-test-v1": 0.4192016643984635,
"synthetic-relevance--nanobeir--nanodbpedia-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanofiqa2018-noul-test-v1": 0.40944448001614253,
"synthetic-relevance--nanobeir--nanofiqa2018-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanohotpotqa-noul-test-v1": 0.32422648339838234,
"synthetic-relevance--nanobeir--nanohotpotqa-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanomsmarco-noul-test-v1": 0.4970835009297585,
"synthetic-relevance--nanobeir--nanomsmarco-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanonfcorpus-noul-test-v1": 0.3791955899378503,
"synthetic-relevance--nanobeir--nanonfcorpus-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanonq-noul-test-v1": 0.5139497440345342,
"synthetic-relevance--nanobeir--nanonq-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanoscidocs-noul-test-v1": 0.32055155471466357,
"synthetic-relevance--nanobeir--nanoscidocs-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanotouche2020-noul-test-v1": 0.36999631490467055,
"synthetic-relevance--nanobeir--nanotouche2020-score-test-v1": 0.04578404341519271,
"synthetic-relevance--nanocoir-noul-test-v1": 0.44900362569846064,
"synthetic-relevance--nanocoir-score-test-v1": 0.0,
"temporal-nli-choice-test-v1": 0.0,
"tracie-noul-test-v1": 0.0,
"laya--typed-decisions-choice-test-v1": 0.17588299359671944,
"laya--typed-decisions-noul-test-v1": 0.17023925810898288,
"laya--typed-decisions-score-test-v1": 0.09552482346032742,
"ud-ewt-choice-test-v1": 0.0,
"winobias-choice-test-v1": 0.0,
"winowhy-noul-test-v1": 0.0,
"wiqa-choice-test-v1": 0.015873015873015886
},
"noul": 0.2287179560241029,
"choice": 0.2953177758045324,
"score": 0.020098659488690808,
"task_avg": 0.1813781304391087
}
},
"rl60": {
"temps": {
"choice": 1.1599743376333564,
"noul": 0.7983784191901301,
"score": 0.7276446912498098
},
"raw": {
"adjusted": {
"aegis2-noul-test-v1": 0.5602707152002926,
"laya--ag-news-choice-test-v1": 0.8358208955223883,
"aqua-rat-choice-test-v1": 0.21126760563380287,
"arc-choice-test-v1": 0.9558823529411764,
"arct-choice-test-v1": 0.5319148936170213,
"argument-quality-choice-test-v1": 0.6938775510204082,
"argument-quality-noul-test-v1": 0.12934047939043625,
"babi-nli-noul-test-v1": 0.6400000000000001,
"banking77-choice-test-v1": 0.6736842105263157,
"bbq-choice-test-v1": 0.9166666666666666,
"boardgameqa-choice-test-v1": 0.23437500000000003,
"canttalk-noul-test-v1": 0.5800000000000001,
"civil-comments-noul-test-v1": 0.2915103139204662,
"cladder-noul-test-v1": 0.48,
"clinc-choice-test-v1": 0.786206896551724,
"contract-nli-choice-test-v1": 0.5466377440347072,
"corr2cause-choice-test-v1": 0.11666666666666659,
"creak-noul-test-v1": 0.6400000000000001,
"crows-pairs-choice-test-v1": 0.6041666666666667,
"dbpedia-choice-test-v1": 0.9662921348314606,
"defeasible-nli-choice-test-v1": 0.5652173913043479,
"laya--enron-spam-noul-test-v1": 0.8999999999999999,
"esci-choice-test-v1": 0.2602739726027397,
"ethics-choice-test-v1": 0.6666666666666665,
"ethics-noul-test-v1": 0.38928571428571423,
"ethos-noul-test-v1": 0.6400000000000001,
"few-nerd-choice-test-v1": 0.6487455197132617,
"fol-nli-choice-test-v1": 0.13114754098360648,
"followir-core17-noul-test-v1": 0.29428904428904423,
"followir-news21-noul-test-v1": 0.18000000000000016,
"followir-robust04-noul-test-v1": 0.7923815062518174,
"go-emotions-noul-test-v1": 0.5142817874272971,
"gretel-pii-choice-test-v1": 0.7175324675324675,
"gsm8k-choice-test-v1": 0.5857142857142857,
"hans-noul-test-v1": 0.7000000000000002,
"hatecheck-noul-test-v1": 0.6800000000000002,
"hh-rlhf-choice-test-v1": 0.05128205128205133,
"hwu64-choice-test-v1": 0.7765957446808511,
"impli-noul-test-v1": 0.74,
"lexcomp-noul-test-v1": 0.43999999999999995,
"logical-entailment-noul-test-v1": 0.06000000000000005,
"lonli-choice-test-v1": 0.7704918032786885,
"massive-choice-test-v1": 0.829981718464351,
"miqa-choice-test-v1": 0.9318181818181818,
"mtop-choice-test-v1": 0.8001313927632908,
"nlsat-noul-test-v1": 0.06000000000000005,
"open-jev--amount-extraction-control-v1-choice-test-v1": 0.7254901960784313,
"open-jev--amount-extraction-control-v1-noul-test-v1": 0.46875,
"open-jev--browser-control-v1-choice-test-v1": 1.0,
"open-jev--citation-control-v1-choice-test-v1": 0.9848484848484849,
"open-jev--context-retention-control-v1-noul-test-v1": 0.0,
"open-jev--customer-control-v1-choice-test-v1": 0.8260869565217391,
"open-jev--customer-control-v1-noul-test-v1": 0.47058823529411775,
"open-jev--customer-control-v1-score-test-v1": 0.4798837052234861,
"open-jev--drone-control-v1-choice-test-v1": 0.0,
"open-jev--drone-control-v1-noul-test-v1": 1.0,
"open-jev--drone-control-v1-score-test-v1": 0.0,
"open-jev--email-selection-control-v1-choice-test-v1": 0.11304347826086972,
"open-jev--email-selection-control-v1-noul-test-v1": 0.6153846153846154,
"open-jev--entity-alignment-control-v1-noul-test-v1": 0.8333333333333335,
"open-jev--entity-alignment-control-v1-score-test-v1": 0.09181388236706145,
"open-jev--ir-control-v1-choice-test-v1": 0.84,
"open-jev--ir-control-v1-noul-test-v1": 0.8571428571428572,
"open-jev--ir-control-v1-score-test-v1": 0.0,
"open-jev--mailroom-control-v1-choice-test-v1": 1.0,
"open-jev--mailroom-control-v1-noul-test-v1": 0.8111111111111111,
"open-jev--painting-geometry-v1-choice-test-v1": 0.8947368421052632,
"open-jev--painting-geometry-v1-noul-test-v1": 0.2666666666666666,
"open-jev--painting-geometry-v1-score-test-v1": 0.2945765737278052,
"open-jev--phone-extraction-control-v1-choice-test-v1": 0.45652173913043476,
"open-jev--phone-extraction-control-v1-noul-test-v1": 0.8999999999999999,
"open-jev--reasoning-control-v1-choice-test-v1": 0.55,
"open-jev--reasoning-control-v1-noul-test-v1": 0.40808823529411775,
"open-jev--reasoning-control-v1-score-test-v1": 0.48436402001289314,
"open-jev--silent-failure-control-v1-noul-test-v1": 0.6000000000000001,
"open-jev--snake-v1-choice-test-v1": 0.0,
"open-jev--snake-v1-noul-test-v1": 0.027738264580369876,
"open-jev--sponsor-segment-control-v1-choice-test-v1": 1.0,
"open-jev--tic-tac-toe-v1-choice-test-v1": 0.03787878787878775,
"open-jev--vizdoom-basic-v1-noul-test-v1": 0.7058823529411766,
"open-jev--vizdoom-basic-v1-score-test-v1": 0.4617796281661025,
"open-jev--workflow-controls-v1--agent-trace-observability-noul-test-v1": 0.3659673659673659,
"open-jev--workflow-controls-v1--customer-service-noul-test-v1": 0.5628485628485629,
"open-jev--workflow-controls-v1--invoice-processing-noul-test-v1": 0.6801176965111391,
"open-jev--workflow-controls-v1--security-incidents-noul-test-v1": 0.7840112201963534,
"openbookqa-choice-test-v1": 0.8611111111111112,
"patent-similarity-score-test-v1": 0.3018724918435256,
"paws-noul-test-v1": 0.6400000000000001,
"plane-noul-test-v1": 0.06000000000000005,
"poem-sentiment-choice-test-v1": 0.0,
"qasper-noul-test-v1": 0.54,
"quartz-choice-test-v1": 0.6428571428571428,
"robust-lr-choice-test-v1": 0.65,
"ruletaker-noul-test-v1": 0.48,
"s1mb-generalization-contextual-choice-choice-test-v1": 0.9613733905579399,
"s1mb-generalization-contextual-noul-noul-test-v1": 0.8999999999999999,
"s1mb-generalization-contextual-score-score-test-v1": 0.7084916866224619,
"s1mb-generalization-diverse-choice-choice-test-v1": 0.9207048458149779,
"s1mb-generalization-diverse-noul-noul-test-v1": 0.8799999999999999,
"s1mb-generalization-diverse-score-score-test-v1": 0.7178847338130516,
"scicite-choice-test-v1": 0.5806451612903225,
"scitail-noul-test-v1": 0.6799999999999999,
"scone-choice-test-v1": 0.72,
"sdoh-nli-noul-test-v1": 0.8199999999999998,
"sgd-choice-test-v1": 0.22033898305084754,
"snli-choice-test-v1": 0.7096774193548386,
"spartqa-choice-test-v1": 0.0967741935483871,
"stepgame-choice-test-v1": 0.33333333333333337,
"synthetic-relevance--nanobeir--nanoarguana-noul-test-v1": 0.3056378575842238,
"synthetic-relevance--nanobeir--nanoarguana-score-test-v1": 0.19497590804151588,
"synthetic-relevance--nanobeir--nanodbpedia-noul-test-v1": 0.3978836709428111,
"synthetic-relevance--nanobeir--nanodbpedia-score-test-v1": 0.3903572387473163,
"synthetic-relevance--nanobeir--nanofiqa2018-noul-test-v1": 0.44083812096059116,
"synthetic-relevance--nanobeir--nanofiqa2018-score-test-v1": 0.3280920645399601,
"synthetic-relevance--nanobeir--nanohotpotqa-noul-test-v1": 0.1557554160303185,
"synthetic-relevance--nanobeir--nanohotpotqa-score-test-v1": 0.31119288161182757,
"synthetic-relevance--nanobeir--nanomsmarco-noul-test-v1": 0.5616846942591427,
"synthetic-relevance--nanobeir--nanomsmarco-score-test-v1": 0.41704488490408065,
"synthetic-relevance--nanobeir--nanonfcorpus-noul-test-v1": 0.3462396114391635,
"synthetic-relevance--nanobeir--nanonfcorpus-score-test-v1": 0.26812714454978337,
"synthetic-relevance--nanobeir--nanonq-noul-test-v1": 0.5649185247217279,
"synthetic-relevance--nanobeir--nanonq-score-test-v1": 0.26908745017293123,
"synthetic-relevance--nanobeir--nanoscidocs-noul-test-v1": 0.16736832720824957,
"synthetic-relevance--nanobeir--nanoscidocs-score-test-v1": 0.21189142390970628,
"synthetic-relevance--nanobeir--nanotouche2020-noul-test-v1": 0.4545782796020945,
"synthetic-relevance--nanobeir--nanotouche2020-score-test-v1": 0.3189426005270901,
"synthetic-relevance--nanocoir-noul-test-v1": 0.512296962815457,
"synthetic-relevance--nanocoir-score-test-v1": 0.07479742192783478,
"temporal-nli-choice-test-v1": 0.22580645161290325,
"tracie-noul-test-v1": 0.18000000000000016,
"laya--typed-decisions-choice-test-v1": 0.30864086833403337,
"laya--typed-decisions-noul-test-v1": 0.35900076645519086,
"laya--typed-decisions-score-test-v1": 0.5457901932960199,
"ud-ewt-choice-test-v1": 0.21651090342679127,
"winobias-choice-test-v1": 0.5652173913043479,
"winowhy-noul-test-v1": 0.3999999999999999,
"wiqa-choice-test-v1": 0.3492063492063492
},
"noul": 0.5070371577975563,
"choice": 0.5719274044054583,
"score": 0.327188854000212,
"task_avg": 0.4687178054010756
},
"calibrated": {
"adjusted": {
"aegis2-noul-test-v1": 0.5602707152002926,
"laya--ag-news-choice-test-v1": 0.8358208955223883,
"aqua-rat-choice-test-v1": 0.21126760563380287,
"arc-choice-test-v1": 0.9558823529411764,
"arct-choice-test-v1": 0.5319148936170213,
"argument-quality-choice-test-v1": 0.6938775510204082,
"argument-quality-noul-test-v1": 0.12934047939043625,
"babi-nli-noul-test-v1": 0.6400000000000001,
"banking77-choice-test-v1": 0.6736842105263157,
"bbq-choice-test-v1": 0.9166666666666666,
"boardgameqa-choice-test-v1": 0.23437500000000003,
"canttalk-noul-test-v1": 0.5800000000000001,
"civil-comments-noul-test-v1": 0.2915103139204662,
"cladder-noul-test-v1": 0.48,
"clinc-choice-test-v1": 0.786206896551724,
"contract-nli-choice-test-v1": 0.5466377440347072,
"corr2cause-choice-test-v1": 0.11666666666666659,
"creak-noul-test-v1": 0.6400000000000001,
"crows-pairs-choice-test-v1": 0.6041666666666667,
"dbpedia-choice-test-v1": 0.9662921348314606,
"defeasible-nli-choice-test-v1": 0.5652173913043479,
"laya--enron-spam-noul-test-v1": 0.8999999999999999,
"esci-choice-test-v1": 0.2602739726027397,
"ethics-choice-test-v1": 0.6666666666666665,
"ethics-noul-test-v1": 0.38928571428571423,
"ethos-noul-test-v1": 0.6400000000000001,
"few-nerd-choice-test-v1": 0.6487455197132617,
"fol-nli-choice-test-v1": 0.13114754098360648,
"followir-core17-noul-test-v1": 0.29428904428904423,
"followir-news21-noul-test-v1": 0.18000000000000016,
"followir-robust04-noul-test-v1": 0.7923815062518174,
"go-emotions-noul-test-v1": 0.5142817874272971,
"gretel-pii-choice-test-v1": 0.7175324675324675,
"gsm8k-choice-test-v1": 0.5857142857142857,
"hans-noul-test-v1": 0.7000000000000002,
"hatecheck-noul-test-v1": 0.6800000000000002,
"hh-rlhf-choice-test-v1": 0.05128205128205133,
"hwu64-choice-test-v1": 0.7765957446808511,
"impli-noul-test-v1": 0.74,
"lexcomp-noul-test-v1": 0.43999999999999995,
"logical-entailment-noul-test-v1": 0.06000000000000005,
"lonli-choice-test-v1": 0.7704918032786885,
"massive-choice-test-v1": 0.829981718464351,
"miqa-choice-test-v1": 0.9318181818181818,
"mtop-choice-test-v1": 0.8001313927632908,
"nlsat-noul-test-v1": 0.06000000000000005,
"open-jev--amount-extraction-control-v1-choice-test-v1": 0.7254901960784313,
"open-jev--amount-extraction-control-v1-noul-test-v1": 0.46875,
"open-jev--browser-control-v1-choice-test-v1": 1.0,
"open-jev--citation-control-v1-choice-test-v1": 0.9848484848484849,
"open-jev--context-retention-control-v1-noul-test-v1": 0.0,
"open-jev--customer-control-v1-choice-test-v1": 0.8260869565217391,
"open-jev--customer-control-v1-noul-test-v1": 0.47058823529411775,
"open-jev--customer-control-v1-score-test-v1": 0.5010736256437411,
"open-jev--drone-control-v1-choice-test-v1": 0.0,
"open-jev--drone-control-v1-noul-test-v1": 1.0,
"open-jev--drone-control-v1-score-test-v1": 0.0,
"open-jev--email-selection-control-v1-choice-test-v1": 0.11304347826086972,
"open-jev--email-selection-control-v1-noul-test-v1": 0.6153846153846154,
"open-jev--entity-alignment-control-v1-noul-test-v1": 0.8333333333333335,
"open-jev--entity-alignment-control-v1-score-test-v1": 0.10768792068745892,
"open-jev--ir-control-v1-choice-test-v1": 0.84,
"open-jev--ir-control-v1-noul-test-v1": 0.8571428571428572,
"open-jev--ir-control-v1-score-test-v1": 0.0,
"open-jev--mailroom-control-v1-choice-test-v1": 1.0,
"open-jev--mailroom-control-v1-noul-test-v1": 0.8111111111111111,
"open-jev--painting-geometry-v1-choice-test-v1": 0.8947368421052632,
"open-jev--painting-geometry-v1-noul-test-v1": 0.2666666666666666,
"open-jev--painting-geometry-v1-score-test-v1": 0.31163678271733497,
"open-jev--phone-extraction-control-v1-choice-test-v1": 0.45652173913043476,
"open-jev--phone-extraction-control-v1-noul-test-v1": 0.8999999999999999,
"open-jev--reasoning-control-v1-choice-test-v1": 0.55,
"open-jev--reasoning-control-v1-noul-test-v1": 0.40808823529411775,
"open-jev--reasoning-control-v1-score-test-v1": 0.5116676314844353,
"open-jev--silent-failure-control-v1-noul-test-v1": 0.6000000000000001,
"open-jev--snake-v1-choice-test-v1": 0.0,
"open-jev--snake-v1-noul-test-v1": 0.027738264580369876,
"open-jev--sponsor-segment-control-v1-choice-test-v1": 1.0,
"open-jev--tic-tac-toe-v1-choice-test-v1": 0.03787878787878775,
"open-jev--vizdoom-basic-v1-noul-test-v1": 0.7058823529411766,
"open-jev--vizdoom-basic-v1-score-test-v1": 0.5611449747196239,
"open-jev--workflow-controls-v1--agent-trace-observability-noul-test-v1": 0.3659673659673659,
"open-jev--workflow-controls-v1--customer-service-noul-test-v1": 0.5628485628485629,
"open-jev--workflow-controls-v1--invoice-processing-noul-test-v1": 0.6801176965111391,
"open-jev--workflow-controls-v1--security-incidents-noul-test-v1": 0.7840112201963534,
"openbookqa-choice-test-v1": 0.8611111111111112,
"patent-similarity-score-test-v1": 0.290246823046062,
"paws-noul-test-v1": 0.6400000000000001,
"plane-noul-test-v1": 0.06000000000000005,
"poem-sentiment-choice-test-v1": 0.0,
"qasper-noul-test-v1": 0.54,
"quartz-choice-test-v1": 0.6428571428571428,
"robust-lr-choice-test-v1": 0.65,
"ruletaker-noul-test-v1": 0.48,
"s1mb-generalization-contextual-choice-choice-test-v1": 0.9613733905579399,
"s1mb-generalization-contextual-noul-noul-test-v1": 0.8999999999999999,
"s1mb-generalization-contextual-score-score-test-v1": 0.7493735516889689,
"s1mb-generalization-diverse-choice-choice-test-v1": 0.9207048458149779,
"s1mb-generalization-diverse-noul-noul-test-v1": 0.8799999999999999,
"s1mb-generalization-diverse-score-score-test-v1": 0.7605669662438349,
"scicite-choice-test-v1": 0.5806451612903225,
"scitail-noul-test-v1": 0.6799999999999999,
"scone-choice-test-v1": 0.72,
"sdoh-nli-noul-test-v1": 0.8199999999999998,
"sgd-choice-test-v1": 0.22033898305084754,
"snli-choice-test-v1": 0.7096774193548386,
"spartqa-choice-test-v1": 0.0967741935483871,
"stepgame-choice-test-v1": 0.33333333333333337,
"synthetic-relevance--nanobeir--nanoarguana-noul-test-v1": 0.3056378575842238,
"synthetic-relevance--nanobeir--nanoarguana-score-test-v1": 0.24287287966932491,
"synthetic-relevance--nanobeir--nanodbpedia-noul-test-v1": 0.3978836709428111,
"synthetic-relevance--nanobeir--nanodbpedia-score-test-v1": 0.4117364590263355,
"synthetic-relevance--nanobeir--nanofiqa2018-noul-test-v1": 0.44083812096059116,
"synthetic-relevance--nanobeir--nanofiqa2018-score-test-v1": 0.347061893737288,
"synthetic-relevance--nanobeir--nanohotpotqa-noul-test-v1": 0.1557554160303185,
"synthetic-relevance--nanobeir--nanohotpotqa-score-test-v1": 0.3533716425449749,
"synthetic-relevance--nanobeir--nanomsmarco-noul-test-v1": 0.5616846942591427,
"synthetic-relevance--nanobeir--nanomsmarco-score-test-v1": 0.4421745077469317,
"synthetic-relevance--nanobeir--nanonfcorpus-noul-test-v1": 0.3462396114391635,
"synthetic-relevance--nanobeir--nanonfcorpus-score-test-v1": 0.30330704037715905,
"synthetic-relevance--nanobeir--nanonq-noul-test-v1": 0.5649185247217279,
"synthetic-relevance--nanobeir--nanonq-score-test-v1": 0.34563416909402134,
"synthetic-relevance--nanobeir--nanoscidocs-noul-test-v1": 0.16736832720824957,
"synthetic-relevance--nanobeir--nanoscidocs-score-test-v1": 0.2551052742071359,
"synthetic-relevance--nanobeir--nanotouche2020-noul-test-v1": 0.4545782796020945,
"synthetic-relevance--nanobeir--nanotouche2020-score-test-v1": 0.29566497078210885,
"synthetic-relevance--nanocoir-noul-test-v1": 0.512296962815457,
"synthetic-relevance--nanocoir-score-test-v1": 0.21403263359189162,
"temporal-nli-choice-test-v1": 0.22580645161290325,
"tracie-noul-test-v1": 0.18000000000000016,
"laya--typed-decisions-choice-test-v1": 0.30864086833403337,
"laya--typed-decisions-noul-test-v1": 0.35900076645519086,
"laya--typed-decisions-score-test-v1": 0.5067348569558621,
"ud-ewt-choice-test-v1": 0.21651090342679127,
"winobias-choice-test-v1": 0.5652173913043479,
"winowhy-noul-test-v1": 0.3999999999999999,
"wiqa-choice-test-v1": 0.3492063492063492
},
"noul": 0.5070371577975563,
"choice": 0.5719274044054583,
"score": 0.3576711716173568,
"task_avg": 0.47887857794012384
}
},
"rl60_routed30": {
"temps": null,
"raw": {
"adjusted": {
"aegis2-noul-test-v1": 0.5754527162977867,
"laya--ag-news-choice-test-v1": 0.8507462686567167,
"aqua-rat-choice-test-v1": 0.6619718309859156,
"arc-choice-test-v1": 0.9852941176470588,
"arct-choice-test-v1": 0.7446808510638298,
"argument-quality-choice-test-v1": 0.7755102040816326,
"argument-quality-noul-test-v1": 0.09000837414976615,
"babi-nli-noul-test-v1": 0.8,
"banking77-choice-test-v1": 0.7157894736842105,
"bbq-choice-test-v1": 0.95,
"boardgameqa-choice-test-v1": 0.65625,
"canttalk-noul-test-v1": 0.6400000000000001,
"civil-comments-noul-test-v1": 0.312200496400324,
"cladder-noul-test-v1": 0.6200000000000001,
"clinc-choice-test-v1": 0.8896551724137931,
"contract-nli-choice-test-v1": 0.6052060737527115,
"corr2cause-choice-test-v1": 0.23333333333333336,
"creak-noul-test-v1": 0.7200000000000002,
"crows-pairs-choice-test-v1": 0.6874999999999999,
"dbpedia-choice-test-v1": 0.9662921348314606,
"defeasible-nli-choice-test-v1": 0.6956521739130435,
"laya--enron-spam-noul-test-v1": 0.8999999999999999,
"esci-choice-test-v1": 0.2328767123287671,
"ethics-choice-test-v1": 0.3333333333333334,
"ethics-noul-test-v1": 0.5571428571428569,
"ethos-noul-test-v1": 0.7,
"few-nerd-choice-test-v1": 0.7025089605734768,
"fol-nli-choice-test-v1": 0.540983606557377,
"followir-core17-noul-test-v1": 0.3665501165501166,
"followir-news21-noul-test-v1": 0.2466666666666666,
"followir-robust04-noul-test-v1": 0.7813317824949113,
"go-emotions-noul-test-v1": 0.5453700644558837,
"gretel-pii-choice-test-v1": 0.8051948051948051,
"gsm8k-choice-test-v1": 0.9428571428571428,
"hans-noul-test-v1": 0.76,
"hatecheck-noul-test-v1": 0.72,
"hh-rlhf-choice-test-v1": 0.0,
"hwu64-choice-test-v1": 0.8404255319148937,
"impli-noul-test-v1": 0.78,
"lexcomp-noul-test-v1": 0.43999999999999995,
"logical-entailment-noul-test-v1": 0.6800000000000002,
"lonli-choice-test-v1": 0.8360655737704918,
"massive-choice-test-v1": 0.8409506398537477,
"miqa-choice-test-v1": 0.8636363636363635,
"mtop-choice-test-v1": 0.8172629876692945,
"nlsat-noul-test-v1": 0.1200000000000001,
"open-jev--amount-extraction-control-v1-choice-test-v1": 0.7254901960784313,
"open-jev--amount-extraction-control-v1-noul-test-v1": 0.53125,
"open-jev--browser-control-v1-choice-test-v1": 1.0,
"open-jev--citation-control-v1-choice-test-v1": 0.9545454545454545,
"open-jev--context-retention-control-v1-noul-test-v1": 0.0,
"open-jev--customer-control-v1-choice-test-v1": 0.8260869565217391,
"open-jev--customer-control-v1-noul-test-v1": 0.47058823529411775,
"open-jev--customer-control-v1-score-test-v1": 0.4999230199817669,
"open-jev--drone-control-v1-choice-test-v1": 0.34210526315789475,
"open-jev--drone-control-v1-noul-test-v1": 1.0,
"open-jev--drone-control-v1-score-test-v1": 0.0,
"open-jev--email-selection-control-v1-choice-test-v1": 0.3217391304347826,
"open-jev--email-selection-control-v1-noul-test-v1": 0.6923076923076925,
"open-jev--entity-alignment-control-v1-noul-test-v1": 0.9444444444444444,
"open-jev--entity-alignment-control-v1-score-test-v1": 0.21129200822082306,
"open-jev--ir-control-v1-choice-test-v1": 0.84,
"open-jev--ir-control-v1-noul-test-v1": 0.9285714285714286,
"open-jev--ir-control-v1-score-test-v1": 0.24326190557901484,
"open-jev--mailroom-control-v1-choice-test-v1": 1.0,
"open-jev--mailroom-control-v1-noul-test-v1": 0.8111111111111111,
"open-jev--painting-geometry-v1-choice-test-v1": 0.8947368421052632,
"open-jev--painting-geometry-v1-noul-test-v1": 0.2666666666666666,
"open-jev--painting-geometry-v1-score-test-v1": 0.5976683560167559,
"open-jev--phone-extraction-control-v1-choice-test-v1": 0.5434782608695653,
"open-jev--phone-extraction-control-v1-noul-test-v1": 0.8999999999999999,
"open-jev--reasoning-control-v1-choice-test-v1": 0.7000000000000001,
"open-jev--reasoning-control-v1-noul-test-v1": 0.5845588235294117,
"open-jev--reasoning-control-v1-score-test-v1": 0.7667997990055253,
"open-jev--silent-failure-control-v1-noul-test-v1": 0.6600000000000001,
"open-jev--snake-v1-choice-test-v1": 0.154639175257732,
"open-jev--snake-v1-noul-test-v1": 0.8918918918918919,
"open-jev--sponsor-segment-control-v1-choice-test-v1": 0.9487179487179487,
"open-jev--tic-tac-toe-v1-choice-test-v1": 0.5454545454545453,
"open-jev--vizdoom-basic-v1-noul-test-v1": 0.8823529411764706,
"open-jev--vizdoom-basic-v1-score-test-v1": 0.7063524968418744,
"open-jev--workflow-controls-v1--agent-trace-observability-noul-test-v1": 0.3659673659673659,
"open-jev--workflow-controls-v1--customer-service-noul-test-v1": 0.6598026598026596,
"open-jev--workflow-controls-v1--invoice-processing-noul-test-v1": 0.8226145439260193,
"open-jev--workflow-controls-v1--security-incidents-noul-test-v1": 0.7260402057035997,
"openbookqa-choice-test-v1": 0.888888888888889,
"patent-similarity-score-test-v1": 0.37933688115458797,
"paws-noul-test-v1": 0.7,
"plane-noul-test-v1": 0.08000000000000007,
"poem-sentiment-choice-test-v1": 0.0,
"qasper-noul-test-v1": 0.6200000000000001,
"quartz-choice-test-v1": 0.7142857142857143,
"robust-lr-choice-test-v1": 0.7,
"ruletaker-noul-test-v1": 0.6599999999999999,
"s1mb-generalization-contextual-choice-choice-test-v1": 1.0,
"s1mb-generalization-contextual-noul-noul-test-v1": 0.94,
"s1mb-generalization-contextual-score-score-test-v1": 0.8558560747528754,
"s1mb-generalization-diverse-choice-choice-test-v1": 0.9471365638766519,
"s1mb-generalization-diverse-noul-noul-test-v1": 0.9199999999999999,
"s1mb-generalization-diverse-score-score-test-v1": 0.849707901053516,
"scicite-choice-test-v1": 0.5967741935483871,
"scitail-noul-test-v1": 0.76,
"scone-choice-test-v1": 0.98,
"sdoh-nli-noul-test-v1": 0.8399999999999999,
"sgd-choice-test-v1": 0.30508474576271183,
"snli-choice-test-v1": 0.7096774193548386,
"spartqa-choice-test-v1": 0.532258064516129,
"stepgame-choice-test-v1": 0.488095238095238,
"synthetic-relevance--nanobeir--nanoarguana-noul-test-v1": 0.4134112329279427,
"synthetic-relevance--nanobeir--nanoarguana-score-test-v1": 0.14842430453822486,
"synthetic-relevance--nanobeir--nanodbpedia-noul-test-v1": 0.3902108553958745,
"synthetic-relevance--nanobeir--nanodbpedia-score-test-v1": 0.42357205529278996,
"synthetic-relevance--nanobeir--nanofiqa2018-noul-test-v1": 0.4472720711071787,
"synthetic-relevance--nanobeir--nanofiqa2018-score-test-v1": 0.25044390721168586,
"synthetic-relevance--nanobeir--nanohotpotqa-noul-test-v1": 0.2594264381469542,
"synthetic-relevance--nanobeir--nanohotpotqa-score-test-v1": 0.4839618133977178,
"synthetic-relevance--nanobeir--nanomsmarco-noul-test-v1": 0.5663302744040744,
"synthetic-relevance--nanobeir--nanomsmarco-score-test-v1": 0.338247053950411,
"synthetic-relevance--nanobeir--nanonfcorpus-noul-test-v1": 0.3508253526903282,
"synthetic-relevance--nanobeir--nanonfcorpus-score-test-v1": 0.2566050750253225,
"synthetic-relevance--nanobeir--nanonq-noul-test-v1": 0.5853724999543066,
"synthetic-relevance--nanobeir--nanonq-score-test-v1": 0.29342713634055395,
"synthetic-relevance--nanobeir--nanoscidocs-noul-test-v1": 0.24720904201048044,
"synthetic-relevance--nanobeir--nanoscidocs-score-test-v1": 0.2761282715556823,
"synthetic-relevance--nanobeir--nanotouche2020-noul-test-v1": 0.442376047517552,
"synthetic-relevance--nanobeir--nanotouche2020-score-test-v1": 0.18507053455759748,
"synthetic-relevance--nanocoir-noul-test-v1": 0.6313993273825997,
"synthetic-relevance--nanocoir-score-test-v1": 0.0827347435762017,
"temporal-nli-choice-test-v1": 0.7580645161290323,
"tracie-noul-test-v1": 0.26,
"laya--typed-decisions-choice-test-v1": 0.2767724141428169,
"laya--typed-decisions-noul-test-v1": 0.4353576361529514,
"laya--typed-decisions-score-test-v1": 0.504422923959694,
"ud-ewt-choice-test-v1": 0.6308411214953271,
"winobias-choice-test-v1": 0.717391304347826,
"winowhy-noul-test-v1": 0.3400000000000001,
"wiqa-choice-test-v1": 0.33333333333333326
},
"noul": 0.5827471502074819,
"choice": 0.6763083260170817,
"score": 0.39777315533393437,
"task_avg": 0.5522762105194993
}
},
"v1_joint": {
"temps": {
"choice": 1.2843740495491411,
"noul": 1.1776347713530488,
"score": 1.0535480782391846
},
"raw": {
"adjusted": {
"aegis2-noul-test-v1": 0.4433875983171758,
"laya--ag-news-choice-test-v1": 0.7910447761194032,
"aqua-rat-choice-test-v1": 0.028169014084507067,
"arc-choice-test-v1": 0.9705882352941176,
"arct-choice-test-v1": 0.48936170212765956,
"argument-quality-choice-test-v1": 0.6326530612244897,
"argument-quality-noul-test-v1": 0.11079818480422476,
"babi-nli-noul-test-v1": 0.41999999999999993,
"banking77-choice-test-v1": 0.6631578947368422,
"bbq-choice-test-v1": 0.95,
"boardgameqa-choice-test-v1": 0.375,
"canttalk-noul-test-v1": 0.72,
"civil-comments-noul-test-v1": 0.314524306059917,
"cladder-noul-test-v1": 0.26,
"clinc-choice-test-v1": 0.8275862068965518,
"contract-nli-choice-test-v1": 0.6106290672451193,
"corr2cause-choice-test-v1": 0.0,
"creak-noul-test-v1": 0.74,
"crows-pairs-choice-test-v1": 0.5416666666666667,
"dbpedia-choice-test-v1": 0.9775280898876404,
"defeasible-nli-choice-test-v1": 0.41304347826086946,
"laya--enron-spam-noul-test-v1": 0.8999999999999999,
"esci-choice-test-v1": 0.17808219178082194,
"ethics-choice-test-v1": 0.3333333333333334,
"ethics-noul-test-v1": 0.43452380952380953,
"ethos-noul-test-v1": 0.72,
"few-nerd-choice-test-v1": 0.6093189964157706,
"fol-nli-choice-test-v1": 0.11475409836065575,
"followir-core17-noul-test-v1": 0.35897435897435903,
"followir-news21-noul-test-v1": 0.2333333333333334,
"followir-robust04-noul-test-v1": 0.8200058156440826,
"go-emotions-noul-test-v1": 0.48444169078029087,
"gretel-pii-choice-test-v1": 0.7727272727272728,
"gsm8k-choice-test-v1": 0.24285714285714285,
"hans-noul-test-v1": 0.7000000000000002,
"hatecheck-noul-test-v1": 0.7799999999999998,
"hh-rlhf-choice-test-v1": 0.10256410256410266,
"hwu64-choice-test-v1": 0.797872340425532,
"impli-noul-test-v1": 0.54,
"lexcomp-noul-test-v1": 0.3999999999999999,
"logical-entailment-noul-test-v1": 0.20000000000000018,
"lonli-choice-test-v1": 0.6229508196721312,
"massive-choice-test-v1": 0.8244972577696527,
"miqa-choice-test-v1": 0.8181818181818182,
"mtop-choice-test-v1": 0.7772892662219527,
"nlsat-noul-test-v1": 0.0,
"open-jev--amount-extraction-control-v1-choice-test-v1": 0.7254901960784313,
"open-jev--amount-extraction-control-v1-noul-test-v1": 0.6875,
"open-jev--browser-control-v1-choice-test-v1": 0.6086956521739129,
"open-jev--citation-control-v1-choice-test-v1": 0.7121212121212123,
"open-jev--context-retention-control-v1-noul-test-v1": 0.06000000000000005,
"open-jev--customer-control-v1-choice-test-v1": 0.8260869565217391,
"open-jev--customer-control-v1-noul-test-v1": 1.0,
"open-jev--customer-control-v1-score-test-v1": 0.4592296552398998,
"open-jev--drone-control-v1-choice-test-v1": 0.0,
"open-jev--drone-control-v1-noul-test-v1": 0.9411764705882353,
"open-jev--drone-control-v1-score-test-v1": 0.0,
"open-jev--email-selection-control-v1-choice-test-v1": 0.37391304347826093,
"open-jev--email-selection-control-v1-noul-test-v1": 0.42307692307692313,
"open-jev--entity-alignment-control-v1-noul-test-v1": 0.7115384615384615,
"open-jev--entity-alignment-control-v1-score-test-v1": 0.10561298468116609,
"open-jev--ir-control-v1-choice-test-v1": 0.6799999999999998,
"open-jev--ir-control-v1-noul-test-v1": 0.7857142857142856,
"open-jev--ir-control-v1-score-test-v1": 0.0,
"open-jev--mailroom-control-v1-choice-test-v1": 1.0,
"open-jev--mailroom-control-v1-noul-test-v1": 0.7722222222222221,
"open-jev--painting-geometry-v1-choice-test-v1": 0.47368421052631576,
"open-jev--painting-geometry-v1-noul-test-v1": 0.06666666666666665,
"open-jev--painting-geometry-v1-score-test-v1": 0.0,
"open-jev--phone-extraction-control-v1-choice-test-v1": 0.6739130434782609,
"open-jev--phone-extraction-control-v1-noul-test-v1": 0.9090909090909092,
"open-jev--reasoning-control-v1-choice-test-v1": 0.55,
"open-jev--reasoning-control-v1-noul-test-v1": 0.4007352941176472,
"open-jev--reasoning-control-v1-score-test-v1": 0.4748872860767298,
"open-jev--silent-failure-control-v1-noul-test-v1": 0.72,
"open-jev--snake-v1-choice-test-v1": 0.18556701030927833,
"open-jev--snake-v1-noul-test-v1": 0.1813655761024182,
"open-jev--sponsor-segment-control-v1-choice-test-v1": 0.9487179487179487,
"open-jev--tic-tac-toe-v1-choice-test-v1": 0.0,
"open-jev--vizdoom-basic-v1-noul-test-v1": 0.8235294117647058,
"open-jev--vizdoom-basic-v1-score-test-v1": 0.4303267488828373,
"open-jev--workflow-controls-v1--agent-trace-observability-noul-test-v1": 0.3543123543123543,
"open-jev--workflow-controls-v1--customer-service-noul-test-v1": 0.5851565851565852,
"open-jev--workflow-controls-v1--invoice-processing-noul-test-v1": 0.7057587221521646,
"open-jev--workflow-controls-v1--security-incidents-noul-test-v1": 0.5409069658719026,
"openbookqa-choice-test-v1": 0.7638888888888888,
"patent-similarity-score-test-v1": 0.3330279158835684,
"paws-noul-test-v1": 0.6400000000000001,
"plane-noul-test-v1": 0.1200000000000001,
"poem-sentiment-choice-test-v1": 0.0,
"qasper-noul-test-v1": 0.6400000000000001,
"quartz-choice-test-v1": 0.7142857142857143,
"robust-lr-choice-test-v1": 0.3166666666666666,
"ruletaker-noul-test-v1": 0.3600000000000001,
"s1mb-generalization-contextual-choice-choice-test-v1": 0.9613733905579399,
"s1mb-generalization-contextual-noul-noul-test-v1": 0.8799999999999999,
"s1mb-generalization-contextual-score-score-test-v1": 0.624034874401848,
"s1mb-generalization-diverse-choice-choice-test-v1": 0.881057268722467,
"s1mb-generalization-diverse-noul-noul-test-v1": 0.8799999999999999,
"s1mb-generalization-diverse-score-score-test-v1": 0.7195231998977583,
"scicite-choice-test-v1": 0.4999999999999999,
"scitail-noul-test-v1": 0.6799999999999999,
"scone-choice-test-v1": 0.54,
"sdoh-nli-noul-test-v1": 0.8399999999999999,
"sgd-choice-test-v1": 0.2542372881355933,
"snli-choice-test-v1": 0.6612903225806452,
"spartqa-choice-test-v1": 0.0967741935483871,
"stepgame-choice-test-v1": 0.27380952380952384,
"synthetic-relevance--nanobeir--nanoarguana-noul-test-v1": 0.19926157464108507,
"synthetic-relevance--nanobeir--nanoarguana-score-test-v1": 0.12693432570770635,
"synthetic-relevance--nanobeir--nanodbpedia-noul-test-v1": 0.41770258066944344,
"synthetic-relevance--nanobeir--nanodbpedia-score-test-v1": 0.37295431133360935,
"synthetic-relevance--nanobeir--nanofiqa2018-noul-test-v1": 0.39002621310810626,
"synthetic-relevance--nanobeir--nanofiqa2018-score-test-v1": 0.34722306717996154,
"synthetic-relevance--nanobeir--nanohotpotqa-noul-test-v1": 0.15598167317155953,
"synthetic-relevance--nanobeir--nanohotpotqa-score-test-v1": 0.17705811713353992,
"synthetic-relevance--nanobeir--nanomsmarco-noul-test-v1": 0.5384503681124295,
"synthetic-relevance--nanobeir--nanomsmarco-score-test-v1": 0.4019849723419704,
"synthetic-relevance--nanobeir--nanonfcorpus-noul-test-v1": 0.3663022294130087,
"synthetic-relevance--nanobeir--nanonfcorpus-score-test-v1": 0.22829536601785572,
"synthetic-relevance--nanobeir--nanonq-noul-test-v1": 0.5156445494559176,
"synthetic-relevance--nanobeir--nanonq-score-test-v1": 0.2591265118565217,
"synthetic-relevance--nanobeir--nanoscidocs-noul-test-v1": 0.17144951511129158,
"synthetic-relevance--nanobeir--nanoscidocs-score-test-v1": 0.16315568062226238,
"synthetic-relevance--nanobeir--nanotouche2020-noul-test-v1": 0.32832924999283186,
"synthetic-relevance--nanobeir--nanotouche2020-score-test-v1": 0.40920848113560715,
"synthetic-relevance--nanocoir-noul-test-v1": 0.4870635794503144,
"synthetic-relevance--nanocoir-score-test-v1": 0.0,
"temporal-nli-choice-test-v1": 0.24193548387096778,
"tracie-noul-test-v1": 0.0,
"laya--typed-decisions-choice-test-v1": 0.3138313776226675,
"laya--typed-decisions-noul-test-v1": 0.3480258402550176,
"laya--typed-decisions-score-test-v1": 0.40457483081738554,
"ud-ewt-choice-test-v1": 0.29595015576323985,
"winobias-choice-test-v1": 0.41304347826086946,
"winowhy-noul-test-v1": 0.3400000000000001,
"wiqa-choice-test-v1": 0.36507936507936506
},
"noul": 0.5007962257490454,
"choice": 0.5230924425272342,
"score": 0.2874837299623918,
"task_avg": 0.4371241327462238
},
"calibrated": {
"adjusted": {
"aegis2-noul-test-v1": 0.4433875983171758,
"laya--ag-news-choice-test-v1": 0.7910447761194032,
"aqua-rat-choice-test-v1": 0.028169014084507067,
"arc-choice-test-v1": 0.9705882352941176,
"arct-choice-test-v1": 0.48936170212765956,
"argument-quality-choice-test-v1": 0.6326530612244897,
"argument-quality-noul-test-v1": 0.11079818480422476,
"babi-nli-noul-test-v1": 0.41999999999999993,
"banking77-choice-test-v1": 0.6631578947368422,
"bbq-choice-test-v1": 0.95,
"boardgameqa-choice-test-v1": 0.375,
"canttalk-noul-test-v1": 0.72,
"civil-comments-noul-test-v1": 0.314524306059917,
"cladder-noul-test-v1": 0.26,
"clinc-choice-test-v1": 0.8275862068965518,
"contract-nli-choice-test-v1": 0.6106290672451193,
"corr2cause-choice-test-v1": 0.0,
"creak-noul-test-v1": 0.74,
"crows-pairs-choice-test-v1": 0.5416666666666667,
"dbpedia-choice-test-v1": 0.9775280898876404,
"defeasible-nli-choice-test-v1": 0.41304347826086946,
"laya--enron-spam-noul-test-v1": 0.8999999999999999,
"esci-choice-test-v1": 0.17808219178082194,
"ethics-choice-test-v1": 0.3333333333333334,
"ethics-noul-test-v1": 0.43452380952380953,
"ethos-noul-test-v1": 0.72,
"few-nerd-choice-test-v1": 0.6093189964157706,
"fol-nli-choice-test-v1": 0.11475409836065575,
"followir-core17-noul-test-v1": 0.35897435897435903,
"followir-news21-noul-test-v1": 0.2333333333333334,
"followir-robust04-noul-test-v1": 0.8200058156440826,
"go-emotions-noul-test-v1": 0.48444169078029087,
"gretel-pii-choice-test-v1": 0.7727272727272728,
"gsm8k-choice-test-v1": 0.24285714285714285,
"hans-noul-test-v1": 0.7000000000000002,
"hatecheck-noul-test-v1": 0.7799999999999998,
"hh-rlhf-choice-test-v1": 0.10256410256410266,
"hwu64-choice-test-v1": 0.797872340425532,
"impli-noul-test-v1": 0.54,
"lexcomp-noul-test-v1": 0.3999999999999999,
"logical-entailment-noul-test-v1": 0.20000000000000018,
"lonli-choice-test-v1": 0.6229508196721312,
"massive-choice-test-v1": 0.8244972577696527,
"miqa-choice-test-v1": 0.8181818181818182,
"mtop-choice-test-v1": 0.7772892662219527,
"nlsat-noul-test-v1": 0.0,
"open-jev--amount-extraction-control-v1-choice-test-v1": 0.7254901960784313,
"open-jev--amount-extraction-control-v1-noul-test-v1": 0.6875,
"open-jev--browser-control-v1-choice-test-v1": 0.6086956521739129,
"open-jev--citation-control-v1-choice-test-v1": 0.7121212121212123,
"open-jev--context-retention-control-v1-noul-test-v1": 0.06000000000000005,
"open-jev--customer-control-v1-choice-test-v1": 0.8260869565217391,
"open-jev--customer-control-v1-noul-test-v1": 1.0,
"open-jev--customer-control-v1-score-test-v1": 0.4550389617256263,
"open-jev--drone-control-v1-choice-test-v1": 0.0,
"open-jev--drone-control-v1-noul-test-v1": 0.9411764705882353,
"open-jev--drone-control-v1-score-test-v1": 0.0,
"open-jev--email-selection-control-v1-choice-test-v1": 0.37391304347826093,
"open-jev--email-selection-control-v1-noul-test-v1": 0.42307692307692313,
"open-jev--entity-alignment-control-v1-noul-test-v1": 0.7115384615384615,
"open-jev--entity-alignment-control-v1-score-test-v1": 0.10354883642807688,
"open-jev--ir-control-v1-choice-test-v1": 0.6799999999999998,
"open-jev--ir-control-v1-noul-test-v1": 0.7857142857142856,
"open-jev--ir-control-v1-score-test-v1": 0.0,
"open-jev--mailroom-control-v1-choice-test-v1": 1.0,
"open-jev--mailroom-control-v1-noul-test-v1": 0.7722222222222221,
"open-jev--painting-geometry-v1-choice-test-v1": 0.47368421052631576,
"open-jev--painting-geometry-v1-noul-test-v1": 0.06666666666666665,
"open-jev--painting-geometry-v1-score-test-v1": 0.0,
"open-jev--phone-extraction-control-v1-choice-test-v1": 0.6739130434782609,
"open-jev--phone-extraction-control-v1-noul-test-v1": 0.9090909090909092,
"open-jev--reasoning-control-v1-choice-test-v1": 0.55,
"open-jev--reasoning-control-v1-noul-test-v1": 0.4007352941176472,
"open-jev--reasoning-control-v1-score-test-v1": 0.46228880266144246,
"open-jev--silent-failure-control-v1-noul-test-v1": 0.72,
"open-jev--snake-v1-choice-test-v1": 0.18556701030927833,
"open-jev--snake-v1-noul-test-v1": 0.1813655761024182,
"open-jev--sponsor-segment-control-v1-choice-test-v1": 0.9487179487179487,
"open-jev--tic-tac-toe-v1-choice-test-v1": 0.0,
"open-jev--vizdoom-basic-v1-noul-test-v1": 0.8235294117647058,
"open-jev--vizdoom-basic-v1-score-test-v1": 0.4189118599502557,
"open-jev--workflow-controls-v1--agent-trace-observability-noul-test-v1": 0.3543123543123543,
"open-jev--workflow-controls-v1--customer-service-noul-test-v1": 0.5851565851565852,
"open-jev--workflow-controls-v1--invoice-processing-noul-test-v1": 0.7057587221521646,
"open-jev--workflow-controls-v1--security-incidents-noul-test-v1": 0.5409069658719026,
"openbookqa-choice-test-v1": 0.7638888888888888,
"patent-similarity-score-test-v1": 0.3266830724535009,
"paws-noul-test-v1": 0.6400000000000001,
"plane-noul-test-v1": 0.1200000000000001,
"poem-sentiment-choice-test-v1": 0.0,
"qasper-noul-test-v1": 0.6400000000000001,
"quartz-choice-test-v1": 0.7142857142857143,
"robust-lr-choice-test-v1": 0.3166666666666666,
"ruletaker-noul-test-v1": 0.3600000000000001,
"s1mb-generalization-contextual-choice-choice-test-v1": 0.9613733905579399,
"s1mb-generalization-contextual-noul-noul-test-v1": 0.8799999999999999,
"s1mb-generalization-contextual-score-score-test-v1": 0.6149834321085637,
"s1mb-generalization-diverse-choice-choice-test-v1": 0.881057268722467,
"s1mb-generalization-diverse-noul-noul-test-v1": 0.8799999999999999,
"s1mb-generalization-diverse-score-score-test-v1": 0.7096966733728995,
"scicite-choice-test-v1": 0.4999999999999999,
"scitail-noul-test-v1": 0.6799999999999999,
"scone-choice-test-v1": 0.54,
"sdoh-nli-noul-test-v1": 0.8399999999999999,
"sgd-choice-test-v1": 0.2542372881355933,
"snli-choice-test-v1": 0.6612903225806452,
"spartqa-choice-test-v1": 0.0967741935483871,
"stepgame-choice-test-v1": 0.27380952380952384,
"synthetic-relevance--nanobeir--nanoarguana-noul-test-v1": 0.19926157464108507,
"synthetic-relevance--nanobeir--nanoarguana-score-test-v1": 0.11167728663667009,
"synthetic-relevance--nanobeir--nanodbpedia-noul-test-v1": 0.41770258066944344,
"synthetic-relevance--nanobeir--nanodbpedia-score-test-v1": 0.3649886179620183,
"synthetic-relevance--nanobeir--nanofiqa2018-noul-test-v1": 0.39002621310810626,
"synthetic-relevance--nanobeir--nanofiqa2018-score-test-v1": 0.34097038999966456,
"synthetic-relevance--nanobeir--nanohotpotqa-noul-test-v1": 0.15598167317155953,
"synthetic-relevance--nanobeir--nanohotpotqa-score-test-v1": 0.15821570935090346,
"synthetic-relevance--nanobeir--nanomsmarco-noul-test-v1": 0.5384503681124295,
"synthetic-relevance--nanobeir--nanomsmarco-score-test-v1": 0.39363766129605826,
"synthetic-relevance--nanobeir--nanonfcorpus-noul-test-v1": 0.3663022294130087,
"synthetic-relevance--nanobeir--nanonfcorpus-score-test-v1": 0.21236216453239287,
"synthetic-relevance--nanobeir--nanonq-noul-test-v1": 0.5156445494559176,
"synthetic-relevance--nanobeir--nanonq-score-test-v1": 0.229478884673753,
"synthetic-relevance--nanobeir--nanoscidocs-noul-test-v1": 0.17144951511129158,
"synthetic-relevance--nanobeir--nanoscidocs-score-test-v1": 0.14839491556145668,
"synthetic-relevance--nanobeir--nanotouche2020-noul-test-v1": 0.32832924999283186,
"synthetic-relevance--nanobeir--nanotouche2020-score-test-v1": 0.4100041834166628,
"synthetic-relevance--nanocoir-noul-test-v1": 0.4870635794503144,
"synthetic-relevance--nanocoir-score-test-v1": 0.0,
"temporal-nli-choice-test-v1": 0.24193548387096778,
"tracie-noul-test-v1": 0.0,
"laya--typed-decisions-choice-test-v1": 0.3138313776226675,
"laya--typed-decisions-noul-test-v1": 0.3480258402550176,
"laya--typed-decisions-score-test-v1": 0.41387338748058744,
"ud-ewt-choice-test-v1": 0.29595015576323985,
"winobias-choice-test-v1": 0.41304347826086946,
"winowhy-noul-test-v1": 0.3400000000000001,
"wiqa-choice-test-v1": 0.36507936507936506
},
"noul": 0.5007962257490454,
"choice": 0.5230924425272342,
"score": 0.27975023045764447,
"task_avg": 0.4345462995779747
}
},
"v2_2b_ro": {
"temps": {
"choice": 1.0772703967295125,
"noul": 0.6149937401183327,
"score": 0.7973321069676444
},
"raw": {
"adjusted": {
"aegis2-noul-test-v1": 0.44539967075178355,
"laya--ag-news-choice-test-v1": 0.8656716417910449,
"aqua-rat-choice-test-v1": 0.042253521126760604,
"arc-choice-test-v1": 0.7205882352941178,
"arct-choice-test-v1": 0.17021276595744672,
"argument-quality-choice-test-v1": 0.673469387755102,
"argument-quality-noul-test-v1": 0.13869566972747505,
"babi-nli-noul-test-v1": 0.6200000000000001,
"banking77-choice-test-v1": 0.6105263157894737,
"bbq-choice-test-v1": 0.7666666666666666,
"boardgameqa-choice-test-v1": 0.09375,
"canttalk-noul-test-v1": 0.6400000000000001,
"civil-comments-noul-test-v1": 0.20946003197816698,
"cladder-noul-test-v1": 0.020000000000000018,
"clinc-choice-test-v1": 0.7724137931034483,
"contract-nli-choice-test-v1": 0.4511930585683298,
"corr2cause-choice-test-v1": 0.0,
"creak-noul-test-v1": 0.48,
"crows-pairs-choice-test-v1": 0.27083333333333337,
"dbpedia-choice-test-v1": 0.9550561797752809,
"defeasible-nli-choice-test-v1": 0.19565217391304343,
"laya--enron-spam-noul-test-v1": 0.74,
"esci-choice-test-v1": 0.17808219178082194,
"ethics-choice-test-v1": 0.6666666666666665,
"ethics-noul-test-v1": 0.3321428571428571,
"ethos-noul-test-v1": 0.6200000000000001,
"few-nerd-choice-test-v1": 0.5878136200716847,
"fol-nli-choice-test-v1": 0.14754098360655732,
"followir-core17-noul-test-v1": 0.3152680652680653,
"followir-news21-noul-test-v1": 0.3799999999999999,
"followir-robust04-noul-test-v1": 0.7176504797906369,
"go-emotions-noul-test-v1": 0.4876902085539958,
"gretel-pii-choice-test-v1": 0.6753246753246753,
"gsm8k-choice-test-v1": 0.15714285714285714,
"hans-noul-test-v1": 0.48,
"hatecheck-noul-test-v1": 0.52,
"hh-rlhf-choice-test-v1": 0.0,
"hwu64-choice-test-v1": 0.7659574468085106,
"impli-noul-test-v1": 0.54,
"lexcomp-noul-test-v1": 0.3599999999999999,
"logical-entailment-noul-test-v1": 0.1399999999999999,
"lonli-choice-test-v1": 0.6229508196721312,
"massive-choice-test-v1": 0.7477148080438757,
"miqa-choice-test-v1": 0.7727272727272727,
"mtop-choice-test-v1": 0.7144734182332727,
"nlsat-noul-test-v1": 0.0,
"open-jev--amount-extraction-control-v1-choice-test-v1": 0.7254901960784313,
"open-jev--amount-extraction-control-v1-noul-test-v1": 0.15625,
"open-jev--browser-control-v1-choice-test-v1": 0.6086956521739129,
"open-jev--citation-control-v1-choice-test-v1": 0.7272727272727273,
"open-jev--context-retention-control-v1-noul-test-v1": 0.0,
"open-jev--customer-control-v1-choice-test-v1": 0.8260869565217391,
"open-jev--customer-control-v1-noul-test-v1": 0.8198529411764706,
"open-jev--customer-control-v1-score-test-v1": 0.37576589136321537,
"open-jev--drone-control-v1-choice-test-v1": 0.0,
"open-jev--drone-control-v1-noul-test-v1": 0.9411764705882353,
"open-jev--drone-control-v1-score-test-v1": 0.0,
"open-jev--email-selection-control-v1-choice-test-v1": 0.21739130434782614,
"open-jev--email-selection-control-v1-noul-test-v1": 0.6153846153846154,
"open-jev--entity-alignment-control-v1-noul-test-v1": 0.3632478632478633,
"open-jev--entity-alignment-control-v1-score-test-v1": 0.0,
"open-jev--ir-control-v1-choice-test-v1": 0.7999999999999998,
"open-jev--ir-control-v1-noul-test-v1": 0.7857142857142856,
"open-jev--ir-control-v1-score-test-v1": 0.0,
"open-jev--mailroom-control-v1-choice-test-v1": 0.9166666666666667,
"open-jev--mailroom-control-v1-noul-test-v1": 0.786111111111111,
"open-jev--painting-geometry-v1-choice-test-v1": 0.5263157894736842,
"open-jev--painting-geometry-v1-noul-test-v1": 0.4761904761904763,
"open-jev--painting-geometry-v1-score-test-v1": 0.0,
"open-jev--phone-extraction-control-v1-choice-test-v1": 0.5217391304347825,
"open-jev--phone-extraction-control-v1-noul-test-v1": 0.30000000000000004,
"open-jev--reasoning-control-v1-choice-test-v1": 0.5000000000000001,
"open-jev--reasoning-control-v1-noul-test-v1": 0.5257352941176472,
"open-jev--reasoning-control-v1-score-test-v1": 0.4566635975353456,
"open-jev--silent-failure-control-v1-noul-test-v1": 0.7999999999999998,
"open-jev--snake-v1-choice-test-v1": 0.09278350515463916,
"open-jev--snake-v1-noul-test-v1": 0.0,
"open-jev--sponsor-segment-control-v1-choice-test-v1": 0.9615384615384615,
"open-jev--tic-tac-toe-v1-choice-test-v1": 0.0,
"open-jev--vizdoom-basic-v1-noul-test-v1": 0.0,
"open-jev--vizdoom-basic-v1-score-test-v1": 0.0,
"open-jev--workflow-controls-v1--agent-trace-observability-noul-test-v1": 0.07925407925407923,
"open-jev--workflow-controls-v1--customer-service-noul-test-v1": 0.5216645216645217,
"open-jev--workflow-controls-v1--invoice-processing-noul-test-v1": 0.4237074401008827,
"open-jev--workflow-controls-v1--security-incidents-noul-test-v1": 0.8597475455820476,
"openbookqa-choice-test-v1": 0.513888888888889,
"patent-similarity-score-test-v1": 0.31252524547010985,
"paws-noul-test-v1": 0.5800000000000001,
"plane-noul-test-v1": 0.15999999999999992,
"poem-sentiment-choice-test-v1": 0.0,
"qasper-noul-test-v1": 0.5,
"quartz-choice-test-v1": 0.5714285714285714,
"robust-lr-choice-test-v1": 0.2833333333333332,
"ruletaker-noul-test-v1": 0.19999999999999996,
"s1mb-generalization-contextual-choice-choice-test-v1": 0.9356223175965664,
"s1mb-generalization-contextual-noul-noul-test-v1": 0.8599999999999999,
"s1mb-generalization-contextual-score-score-test-v1": 0.5807637701678208,
"s1mb-generalization-diverse-choice-choice-test-v1": 0.8281938325991189,
"s1mb-generalization-diverse-noul-noul-test-v1": 0.8199999999999998,
"s1mb-generalization-diverse-score-score-test-v1": 0.542853674942144,
"scicite-choice-test-v1": 0.5806451612903225,
"scitail-noul-test-v1": 0.3599999999999999,
"scone-choice-test-v1": 0.45999999999999996,
"sdoh-nli-noul-test-v1": 0.7799999999999998,
"sgd-choice-test-v1": 0.28813559322033894,
"snli-choice-test-v1": 0.435483870967742,
"spartqa-choice-test-v1": 0.0,
"stepgame-choice-test-v1": 0.16666666666666666,
"synthetic-relevance--nanobeir--nanoarguana-noul-test-v1": 0.24323236341159382,
"synthetic-relevance--nanobeir--nanoarguana-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanodbpedia-noul-test-v1": 0.3514200661946152,
"synthetic-relevance--nanobeir--nanodbpedia-score-test-v1": 0.338086317475911,
"synthetic-relevance--nanobeir--nanofiqa2018-noul-test-v1": 0.39382915107171046,
"synthetic-relevance--nanobeir--nanofiqa2018-score-test-v1": 0.3121162821875584,
"synthetic-relevance--nanobeir--nanohotpotqa-noul-test-v1": 0.15317608462017107,
"synthetic-relevance--nanobeir--nanohotpotqa-score-test-v1": 0.25432868872792525,
"synthetic-relevance--nanobeir--nanomsmarco-noul-test-v1": 0.5433401142954075,
"synthetic-relevance--nanobeir--nanomsmarco-score-test-v1": 0.4092907693647627,
"synthetic-relevance--nanobeir--nanonfcorpus-noul-test-v1": 0.31028836884860755,
"synthetic-relevance--nanobeir--nanonfcorpus-score-test-v1": 0.28151358723979114,
"synthetic-relevance--nanobeir--nanonq-noul-test-v1": 0.4650246992966558,
"synthetic-relevance--nanobeir--nanonq-score-test-v1": 0.18347955342122924,
"synthetic-relevance--nanobeir--nanoscidocs-noul-test-v1": 0.10519955225802624,
"synthetic-relevance--nanobeir--nanoscidocs-score-test-v1": 0.11690566856922147,
"synthetic-relevance--nanobeir--nanotouche2020-noul-test-v1": 0.43438990432557967,
"synthetic-relevance--nanobeir--nanotouche2020-score-test-v1": 0.379596838141959,
"synthetic-relevance--nanocoir-noul-test-v1": 0.4468861308059964,
"synthetic-relevance--nanocoir-score-test-v1": 0.0,
"temporal-nli-choice-test-v1": 0.20967741935483872,
"tracie-noul-test-v1": 0.040000000000000036,
"laya--typed-decisions-choice-test-v1": 0.25768202501015314,
"laya--typed-decisions-noul-test-v1": 0.342119756005856,
"laya--typed-decisions-score-test-v1": 0.260692037594918,
"ud-ewt-choice-test-v1": 0.16043613707165108,
"winobias-choice-test-v1": 0.1086956521739129,
"winowhy-noul-test-v1": 0.19999999999999996,
"wiqa-choice-test-v1": 0.19047619047619047
},
"noul": 0.42252965794032943,
"choice": 0.4568250505770796,
"score": 0.22878961534294814,
"task_avg": 0.36938144128678574
},
"calibrated": {
"adjusted": {
"aegis2-noul-test-v1": 0.44539967075178355,
"laya--ag-news-choice-test-v1": 0.8656716417910449,
"aqua-rat-choice-test-v1": 0.042253521126760604,
"arc-choice-test-v1": 0.7205882352941178,
"arct-choice-test-v1": 0.17021276595744672,
"argument-quality-choice-test-v1": 0.673469387755102,
"argument-quality-noul-test-v1": 0.13869566972747505,
"babi-nli-noul-test-v1": 0.6200000000000001,
"banking77-choice-test-v1": 0.6105263157894737,
"bbq-choice-test-v1": 0.7666666666666666,
"boardgameqa-choice-test-v1": 0.09375,
"canttalk-noul-test-v1": 0.6400000000000001,
"civil-comments-noul-test-v1": 0.20946003197816698,
"cladder-noul-test-v1": 0.020000000000000018,
"clinc-choice-test-v1": 0.7724137931034483,
"contract-nli-choice-test-v1": 0.4511930585683298,
"corr2cause-choice-test-v1": 0.0,
"creak-noul-test-v1": 0.48,
"crows-pairs-choice-test-v1": 0.27083333333333337,
"dbpedia-choice-test-v1": 0.9550561797752809,
"defeasible-nli-choice-test-v1": 0.19565217391304343,
"laya--enron-spam-noul-test-v1": 0.74,
"esci-choice-test-v1": 0.17808219178082194,
"ethics-choice-test-v1": 0.6666666666666665,
"ethics-noul-test-v1": 0.3321428571428571,
"ethos-noul-test-v1": 0.6200000000000001,
"few-nerd-choice-test-v1": 0.5878136200716847,
"fol-nli-choice-test-v1": 0.14754098360655732,
"followir-core17-noul-test-v1": 0.3152680652680653,
"followir-news21-noul-test-v1": 0.3799999999999999,
"followir-robust04-noul-test-v1": 0.7176504797906369,
"go-emotions-noul-test-v1": 0.4876902085539958,
"gretel-pii-choice-test-v1": 0.6753246753246753,
"gsm8k-choice-test-v1": 0.15714285714285714,
"hans-noul-test-v1": 0.48,
"hatecheck-noul-test-v1": 0.52,
"hh-rlhf-choice-test-v1": 0.0,
"hwu64-choice-test-v1": 0.7659574468085106,
"impli-noul-test-v1": 0.54,
"lexcomp-noul-test-v1": 0.3599999999999999,
"logical-entailment-noul-test-v1": 0.1399999999999999,
"lonli-choice-test-v1": 0.6229508196721312,
"massive-choice-test-v1": 0.7477148080438757,
"miqa-choice-test-v1": 0.7727272727272727,
"mtop-choice-test-v1": 0.7144734182332727,
"nlsat-noul-test-v1": 0.0,
"open-jev--amount-extraction-control-v1-choice-test-v1": 0.7254901960784313,
"open-jev--amount-extraction-control-v1-noul-test-v1": 0.15625,
"open-jev--browser-control-v1-choice-test-v1": 0.6086956521739129,
"open-jev--citation-control-v1-choice-test-v1": 0.7272727272727273,
"open-jev--context-retention-control-v1-noul-test-v1": 0.0,
"open-jev--customer-control-v1-choice-test-v1": 0.8260869565217391,
"open-jev--customer-control-v1-noul-test-v1": 0.8198529411764706,
"open-jev--customer-control-v1-score-test-v1": 0.4187521241302211,
"open-jev--drone-control-v1-choice-test-v1": 0.0,
"open-jev--drone-control-v1-noul-test-v1": 0.9411764705882353,
"open-jev--drone-control-v1-score-test-v1": 0.0,
"open-jev--email-selection-control-v1-choice-test-v1": 0.21739130434782614,
"open-jev--email-selection-control-v1-noul-test-v1": 0.6153846153846154,
"open-jev--entity-alignment-control-v1-noul-test-v1": 0.3632478632478633,
"open-jev--entity-alignment-control-v1-score-test-v1": 0.0,
"open-jev--ir-control-v1-choice-test-v1": 0.7999999999999998,
"open-jev--ir-control-v1-noul-test-v1": 0.7857142857142856,
"open-jev--ir-control-v1-score-test-v1": 0.003762448219239989,
"open-jev--mailroom-control-v1-choice-test-v1": 0.9166666666666667,
"open-jev--mailroom-control-v1-noul-test-v1": 0.786111111111111,
"open-jev--painting-geometry-v1-choice-test-v1": 0.5263157894736842,
"open-jev--painting-geometry-v1-noul-test-v1": 0.4761904761904763,
"open-jev--painting-geometry-v1-score-test-v1": 0.0,
"open-jev--phone-extraction-control-v1-choice-test-v1": 0.5217391304347825,
"open-jev--phone-extraction-control-v1-noul-test-v1": 0.30000000000000004,
"open-jev--reasoning-control-v1-choice-test-v1": 0.5000000000000001,
"open-jev--reasoning-control-v1-noul-test-v1": 0.5257352941176472,
"open-jev--reasoning-control-v1-score-test-v1": 0.4850952557994459,
"open-jev--silent-failure-control-v1-noul-test-v1": 0.7999999999999998,
"open-jev--snake-v1-choice-test-v1": 0.09278350515463916,
"open-jev--snake-v1-noul-test-v1": 0.0,
"open-jev--sponsor-segment-control-v1-choice-test-v1": 0.9615384615384615,
"open-jev--tic-tac-toe-v1-choice-test-v1": 0.0,
"open-jev--vizdoom-basic-v1-noul-test-v1": 0.0,
"open-jev--vizdoom-basic-v1-score-test-v1": 0.0,
"open-jev--workflow-controls-v1--agent-trace-observability-noul-test-v1": 0.07925407925407923,
"open-jev--workflow-controls-v1--customer-service-noul-test-v1": 0.5216645216645217,
"open-jev--workflow-controls-v1--invoice-processing-noul-test-v1": 0.4237074401008827,
"open-jev--workflow-controls-v1--security-incidents-noul-test-v1": 0.8597475455820476,
"openbookqa-choice-test-v1": 0.513888888888889,
"patent-similarity-score-test-v1": 0.30384651191657186,
"paws-noul-test-v1": 0.5800000000000001,
"plane-noul-test-v1": 0.15999999999999992,
"poem-sentiment-choice-test-v1": 0.0,
"qasper-noul-test-v1": 0.5,
"quartz-choice-test-v1": 0.5714285714285714,
"robust-lr-choice-test-v1": 0.2833333333333332,
"ruletaker-noul-test-v1": 0.19999999999999996,
"s1mb-generalization-contextual-choice-choice-test-v1": 0.9356223175965664,
"s1mb-generalization-contextual-noul-noul-test-v1": 0.8599999999999999,
"s1mb-generalization-contextual-score-score-test-v1": 0.594086446068868,
"s1mb-generalization-diverse-choice-choice-test-v1": 0.8281938325991189,
"s1mb-generalization-diverse-noul-noul-test-v1": 0.8199999999999998,
"s1mb-generalization-diverse-score-score-test-v1": 0.5760538063599603,
"scicite-choice-test-v1": 0.5806451612903225,
"scitail-noul-test-v1": 0.3599999999999999,
"scone-choice-test-v1": 0.45999999999999996,
"sdoh-nli-noul-test-v1": 0.7799999999999998,
"sgd-choice-test-v1": 0.28813559322033894,
"snli-choice-test-v1": 0.435483870967742,
"spartqa-choice-test-v1": 0.0,
"stepgame-choice-test-v1": 0.16666666666666666,
"synthetic-relevance--nanobeir--nanoarguana-noul-test-v1": 0.24323236341159382,
"synthetic-relevance--nanobeir--nanoarguana-score-test-v1": 0.0,
"synthetic-relevance--nanobeir--nanodbpedia-noul-test-v1": 0.3514200661946152,
"synthetic-relevance--nanobeir--nanodbpedia-score-test-v1": 0.34431685707848536,
"synthetic-relevance--nanobeir--nanofiqa2018-noul-test-v1": 0.39382915107171046,
"synthetic-relevance--nanobeir--nanofiqa2018-score-test-v1": 0.3232729438947516,
"synthetic-relevance--nanobeir--nanohotpotqa-noul-test-v1": 0.15317608462017107,
"synthetic-relevance--nanobeir--nanohotpotqa-score-test-v1": 0.2757512290222319,
"synthetic-relevance--nanobeir--nanomsmarco-noul-test-v1": 0.5433401142954075,
"synthetic-relevance--nanobeir--nanomsmarco-score-test-v1": 0.42010774639672477,
"synthetic-relevance--nanobeir--nanonfcorpus-noul-test-v1": 0.31028836884860755,
"synthetic-relevance--nanobeir--nanonfcorpus-score-test-v1": 0.29726958670945025,
"synthetic-relevance--nanobeir--nanonq-noul-test-v1": 0.4650246992966558,
"synthetic-relevance--nanobeir--nanonq-score-test-v1": 0.2322138564292785,
"synthetic-relevance--nanobeir--nanoscidocs-noul-test-v1": 0.10519955225802624,
"synthetic-relevance--nanobeir--nanoscidocs-score-test-v1": 0.13122383349369227,
"synthetic-relevance--nanobeir--nanotouche2020-noul-test-v1": 0.43438990432557967,
"synthetic-relevance--nanobeir--nanotouche2020-score-test-v1": 0.3695734364304606,
"synthetic-relevance--nanocoir-noul-test-v1": 0.4468861308059964,
"synthetic-relevance--nanocoir-score-test-v1": 0.0,
"temporal-nli-choice-test-v1": 0.20967741935483872,
"tracie-noul-test-v1": 0.040000000000000036,
"laya--typed-decisions-choice-test-v1": 0.25768202501015314,
"laya--typed-decisions-noul-test-v1": 0.342119756005856,
"laya--typed-decisions-score-test-v1": 0.25422877050308035,
"ud-ewt-choice-test-v1": 0.16043613707165108,
"winobias-choice-test-v1": 0.1086956521739129,
"winowhy-noul-test-v1": 0.19999999999999996,
"wiqa-choice-test-v1": 0.19047619047619047
},
"noul": 0.42252965794032943,
"choice": 0.4568250505770796,
"score": 0.23950261202154582,
"task_avg": 0.3729524401796516
}
}
},
"leaderboard": [
{
"model": "FINAL-Bench__Darwin-27B-ZTC-v9",
"borda": 89.4549202487159,
"noul": 74.81050701609263,
"choice": 76.10256265688238,
"score": 65.69946157747638,
"task_avg": 72.2041770834838
},
{
"model": "sqwish__kettei-fat-v1.0",
"borda": 89.23695593403625,
"noul": 73.47864112605596,
"choice": 78.33753697215504,
"score": 60.677673660012786,
"task_avg": 70.83128391940792
},
{
"model": "startlux-models__StartLux-Decision-v1.1-27B-Instant",
"borda": 88.77399297107321,
"noul": 74.28070126625073,
"choice": 80.9689934716538,
"score": 60.47420173900966,
"task_avg": 71.90796549230473
},
{
"model": "punkytiger__superion_s1",
"borda": 88.68275209516082,
"noul": 74.1544173754237,
"choice": 82.76161505434226,
"score": 66.1661284515905,
"task_avg": 74.36072029378549
},
{
"model": "vllm-sr__d3",
"borda": 88.40227088402264,
"noul": 72.76979659296646,
"choice": 79.43823597745329,
"score": 62.15350100760008,
"task_avg": 71.45384452600659
},
{
"model": "PelaAI__KnowLine-31B-Gen1",
"borda": 87.92917004595832,
"noul": 71.06337531128415,
"choice": 76.01593574512474,
"score": 63.27543646305255,
"task_avg": 70.1182491731538
},
{
"model": "FINAL-Bench__Darwin-27B-ZTC-v5",
"borda": 87.44931062449307,
"noul": 74.32352318504921,
"choice": 72.25066441930817,
"score": 65.25738278053589,
"task_avg": 70.61052346163108
},
{
"model": "perplexity-ai__pplx-decider-v1.2",
"borda": 86.45241957285754,
"noul": 71.63991167602491,
"choice": 78.67069758233657,
"score": 59.98685998614592,
"task_avg": 70.0991564148358
},
{
"model": "Maincode__matilda-jev-v1.5",
"borda": 85.13787510137875,
"noul": 70.92486162821892,
"choice": 75.58328214021182,
"score": 62.653655185778035,
"task_avg": 69.72059965140292
},
{
"model": "Superwheat__Gimbal-27B",
"borda": 85.07873749662068,
"noul": 72.64888110858253,
"choice": 72.39510312615168,
"score": 58.82642293307343,
"task_avg": 67.95680238926921
},
{
"model": "perplexity-ai__pplx-decider-v1.1-27b",
"borda": 84.96384157880506,
"noul": 71.38205356140031,
"choice": 76.58026138262434,
"score": 58.41767784792135,
"task_avg": 68.79333093064866
},
{
"model": "mukaddamzaid__amos-1-large",
"borda": 84.78135982698028,
"noul": 71.47255900056145,
"choice": 77.60383254688709,
"score": 49.98509963659832,
"task_avg": 66.35383039468229
},
{
"model": "torchcast-ai__torchcast-decision-27b",
"borda": 84.41639632333064,
"noul": 70.0633938779726,
"choice": 73.80921735043513,
"score": 57.43617546121547,
"task_avg": 67.10292889654106
},
{
"model": "PelaAI__KnowLine-35B-A3B-Gen1",
"borda": 83.37557447958906,
"noul": 67.81403265812658,
"choice": 74.7408710871611,
"score": 53.3391848116189,
"task_avg": 65.29802951896886
},
{
"model": "PelaAI__KnowLine-26B-A4B-Gen2",
"borda": 83.28433360367666,
"noul": 67.5690519789062,
"choice": 72.81852030694385,
"score": 52.064070423706234,
"task_avg": 64.15054756985211
},
{
"model": "shgao__rsi-jev-v6.1-vl-27b",
"borda": 82.72168153555018,
"noul": 67.95377166171617,
"choice": 71.49713578075915,
"score": 61.83374525924009,
"task_avg": 67.09488423390513
},
{
"model": "FINAL-Bench__Darwin-27B-ZTC-v2",
"borda": 82.40909705325764,
"noul": 67.65843776727887,
"choice": 71.5072093161536,
"score": 60.2135925895139,
"task_avg": 66.45974655764878
},
{
"model": "imguangyu__metaencoder_think",
"borda": 80.96276020546095,
"noul": 68.40085704143098,
"choice": 74.68865311969675,
"score": 62.157196191909556,
"task_avg": 68.41556878434577
},
{
"model": "vllm-sr__d3-flash",
"borda": 80.75493376588267,
"noul": 66.6111302967872,
"choice": 74.73312296664545,
"score": 46.47167335902383,
"task_avg": 62.605308874152165
},
{
"model": "PelaAI__KnowLine-4B-Gen5",
"borda": 80.61300351446333,
"noul": 65.65138406936738,
"choice": 76.54750376209795,
"score": 58.83559873081363,
"task_avg": 67.01149552075964
},
{
"model": "frontier-infra__jebadiah-27b",
"borda": 80.55893484725604,
"noul": 66.06741566951077,
"choice": 67.7786419515256,
"score": 53.94896592945718,
"task_avg": 62.598341183497844
},
{
"model": "fastino__glide",
"borda": 80.21931603135981,
"noul": 68.21691573894877,
"choice": 72.21867107156177,
"score": 51.88306305451613,
"task_avg": 64.10621662167556
},
{
"model": "PelaAI__KnowLine-9B-Gen1",
"borda": 79.64314679643144,
"noul": 64.39776873096814,
"choice": 70.81963024662268,
"score": 49.367916639323994,
"task_avg": 61.528438538971606
},
{
"model": "vllm-sr__Decision-2.0-Vega-27B",
"borda": 79.23087320897544,
"noul": 66.89352629736425,
"choice": 70.21338348833062,
"score": 42.94723908397514,
"task_avg": 60.018049623223334
},
{
"model": "sqwish__kettei-chub-v1.0",
"borda": 79.06866720735333,
"noul": 67.94710871859854,
"choice": 70.70417091046505,
"score": 45.63470436445472,
"task_avg": 61.428661331172776
},
{
"model": "openjev__openjev_27b_bf16",
"borda": 78.81690997566912,
"noul": 65.21616191434015,
"choice": 67.74459368182643,
"score": 54.82763584346644,
"task_avg": 62.59613047987768
},
{
"model": "vllm-sr__d3-mini",
"borda": 78.71722087050551,
"noul": 65.03865882654557,
"choice": 71.19375863233792,
"score": 46.37568943914048,
"task_avg": 60.86936896600798
},
{
"model": "perplexity-ai__pplx-decider-flash-v1.2",
"borda": 78.65301432819679,
"noul": 63.8734478128067,
"choice": 73.27406036727218,
"score": 44.65777650755283,
"task_avg": 60.60176156254391
},
{
"model": "jaredpalmer__kev-27b",
"borda": 78.59049743173831,
"noul": 66.65087911918437,
"choice": 70.70791577670633,
"score": 43.18481114363604,
"task_avg": 60.181202013175586
},
{
"model": "autotrust__JEV-27B",
"borda": 78.21877534468776,
"noul": 67.6453099367861,
"choice": 67.72948464303629,
"score": 54.976843994708915,
"task_avg": 63.45054619151044
},
{
"model": "denis-pplx__AutoJev-27B",
"borda": 78.12753446877532,
"noul": 64.96068056423817,
"choice": 68.10599665053212,
"score": 49.33934973953349,
"task_avg": 60.80200898476793
},
{
"model": "logic65__Whittle-Reflex-25B-A3B",
"borda": 77.87408759124088,
"noul": 65.35384776913521,
"choice": 70.67223963282753,
"score": 63.10444249477316,
"task_avg": 66.37684329891196
},
{
"model": "typesafe__jev_1_14_preview",
"borda": 77.58177885915116,
"noul": 65.0558566202727,
"choice": 69.78676525630864,
"score": 46.461284134075306,
"task_avg": 60.43463533688554
},
{
"model": "lihe07__span-02",
"borda": 77.31650446066502,
"noul": 63.602764397014546,
"choice": 69.87903857921958,
"score": 51.221110149015615,
"task_avg": 61.56763770841658
},
{
"model": "PelaAI__KnowLine-26B-A4B-Gen1",
"borda": 77.14753987564207,
"noul": 61.409353277703794,
"choice": 69.64829608572035,
"score": 43.05133673689556,
"task_avg": 58.036328700106566
},
{
"model": "theunnecessarythings__JADE",
"borda": 76.11347661530145,
"noul": 61.70080095759924,
"choice": 66.57448959822455,
"score": 55.43627214509025,
"task_avg": 61.23718756697135
},
{
"model": "OmniJev__OneJev1.6-27B",
"borda": 76.06447688564481,
"noul": 63.54010027741565,
"choice": 69.88914285489403,
"score": 46.90048800520919,
"task_avg": 60.10991037917296
},
{
"model": "typesafe__jev_1_13",
"borda": 76.01040821843742,
"noul": 64.63433927735257,
"choice": 67.21804084276843,
"score": 46.918451356443185,
"task_avg": 59.59027715885473
},
{
"model": "caiovicentino1__Eikos-27B",
"borda": 75.89044336307103,
"noul": 64.1370897159886,
"choice": 67.61278489453032,
"score": 47.833150601627935,
"task_avg": 59.86100840404895
},
{
"model": "e12ex2__jev-vision-27b",
"borda": 75.07434441741007,
"noul": 64.04436182267986,
"choice": 65.93713695074054,
"score": 48.500124275408,
"task_avg": 59.49387434960946
},
{
"model": "OmniJev__OneJev1.5-27B",
"borda": 74.668829413355,
"noul": 61.09501894474635,
"choice": 69.87649691951748,
"score": 47.938519484675226,
"task_avg": 59.636678449646354
},
{
"model": "5ak3t__scm_max_14b_v1",
"borda": 74.55055420383889,
"noul": 60.51181830885981,
"choice": 66.65372054368405,
"score": 39.22091797154031,
"task_avg": 55.46215227469472
},
{
"model": "cribl__decision-1.0-27B",
"borda": 73.34583671262506,
"noul": 58.23245852051888,
"choice": 67.317861895564,
"score": 49.5166582149999,
"task_avg": 58.35565954369426
},
{
"model": "PixilabAI__Blink-v0.3-26B-A4B-NVFP4",
"borda": 73.2191132738578,
"noul": 61.68945323627374,
"choice": 66.14093862930842,
"score": 55.31595651133141,
"task_avg": 61.04878279230452
},
{
"model": "juspay__Xor-26B-A4B",
"borda": 73.01297648012977,
"noul": 62.011868472600995,
"choice": 67.03600249426962,
"score": 31.88512034587986,
"task_avg": 53.6443304375835
},
{
"model": "Cloudflare__clef",
"borda": 72.98763179237638,
"noul": 61.62730332999063,
"choice": 67.20169633565135,
"score": 39.082881095062675,
"task_avg": 55.97062692023489
},
{
"model": "PelaAI__KnowLine-4B-Gen3",
"borda": 72.67504731008384,
"noul": 59.97018440461604,
"choice": 65.48596192877402,
"score": 46.44983643291625,
"task_avg": 57.30199425543544
},
{
"model": "PelaAI__KnowLine-4B-Gen4",
"borda": 72.15970532576372,
"noul": 59.7814276733928,
"choice": 64.98264621333891,
"score": 44.39828763620016,
"task_avg": 56.38745384097729
},
{
"model": "autotrust__GEV-26B-Decide",
"borda": 71.38753717220871,
"noul": 61.897586797625806,
"choice": 66.80357393074135,
"score": 36.232251710077016,
"task_avg": 54.977804146148046
},
{
"model": "michaelfeil__rune-26b-a4b",
"borda": 70.94147066774806,
"noul": 58.56463370433087,
"choice": 64.36358265014647,
"score": 51.57246351871485,
"task_avg": 58.166893291064056
},
{
"model": "OURS:rl60_routed30",
"borda": 70.69309272776422,
"noul": 58.27471502074819,
"choice": 67.63083260170816,
"score": 39.777315533393434,
"task_avg": 55.227621051949924
},
{
"model": "PelaAI__KnowLine-4B-Gen2",
"borda": 70.31968099486345,
"noul": 58.72155491154057,
"choice": 64.0505785467449,
"score": 42.99430713187057,
"task_avg": 55.25548019671868
},
{
"model": "andyzhang232__ajev-gemma4-12b-lora5",
"borda": 70.02906190862397,
"noul": 59.947985038400475,
"choice": 61.77736117267184,
"score": 42.902281691801605,
"task_avg": 54.87587596762463
},
{
"model": "EldanRing__Winnow-12B",
"borda": 69.02034333603677,
"noul": 61.38007245783731,
"choice": 60.69557820772463,
"score": 41.61464332348616,
"task_avg": 54.5634313296827
},
{
"model": "vllm-sr__Decision-2.0-Lux-9B",
"borda": 67.5638686131387,
"noul": 55.22550956774106,
"choice": 60.745127626074826,
"score": 39.14262240520837,
"task_avg": 51.70441986634141
},
{
"model": "juspay__jev-one",
"borda": 66.69032170856988,
"noul": 56.42824286907736,
"choice": 60.78598989989204,
"score": 39.703599336592085,
"task_avg": 52.30594403518717
},
{
"model": "hotchpotch__saihai-v1-8b",
"borda": 66.44701270613676,
"noul": 59.96326955105372,
"choice": 59.50955138563523,
"score": 32.29351303767978,
"task_avg": 50.58877799145625
},
{
"model": "kirp__jpt-9b",
"borda": 66.27128953771289,
"noul": 55.27194977131644,
"choice": 61.86869369858429,
"score": 38.48880869670963,
"task_avg": 51.876484055536785
},
{
"model": "facebook__meta-encoder-cosine-t0.0184671662",
"borda": 66.1766693701,
"noul": 58.083454678598926,
"choice": 61.53247777642923,
"score": 31.041417232334425,
"task_avg": 50.21911656245419
},
{
"model": "shgao__rsi-jev-v6.1-vl-4b",
"borda": 66.1699107866991,
"noul": 58.71382196252589,
"choice": 60.155733062517605,
"score": 34.6320139533932,
"task_avg": 51.16718965947889
},
{
"model": "bespokelabs__Bespoke-Nimble-9B-v3",
"borda": 65.86915382535821,
"noul": 54.77071642710267,
"choice": 62.238226340415316,
"score": 39.52578098159997,
"task_avg": 52.17824124970599
},
{
"model": "OmniJev__OneJev1.5-9B",
"borda": 64.77088402270881,
"noul": 51.549561609277674,
"choice": 60.397532590452826,
"score": 32.61798123492535,
"task_avg": 48.188358478218625
},
{
"model": "Cloudflare__clef-flash",
"borda": 63.80947553392807,
"noul": 51.81658406094412,
"choice": 63.05238235307888,
"score": 29.317245033831874,
"task_avg": 48.062070482618296
},
{
"model": "hotchpotch__bekko-system-one-v0-400m",
"borda": 63.07447958907812,
"noul": 51.23656603425154,
"choice": 61.31612420623802,
"score": 39.248477224230605,
"task_avg": 50.60038915490672
},
{
"model": "frontier-infra__jebadiah-9b-v2",
"borda": 62.934238983509054,
"noul": 53.302174585046544,
"choice": 58.22725190376157,
"score": 34.47869550166637,
"task_avg": 48.66937399682483
},
{
"model": "Mapika__decider-35b-a3b",
"borda": 62.68417139767504,
"noul": 54.76743959294037,
"choice": 59.920146312776545,
"score": 25.355920318146346,
"task_avg": 46.681168741287756
},
{
"model": "OmniJev__OneJev1.6-9B",
"borda": 62.31582860232495,
"noul": 49.01025150665736,
"choice": 61.20405372330777,
"score": 29.028362215106267,
"task_avg": 46.41422248169047
},
{
"model": "ZefanCai__Open-Jev-27B-v1.1",
"borda": 62.29724249797241,
"noul": 51.47692283969588,
"choice": 66.91252355646577,
"score": 52.25918085490528,
"task_avg": 56.88287575035565
},
{
"model": "theaviv__Lev1-3.7B",
"borda": 62.06407137064071,
"noul": 57.96208481407391,
"choice": 60.406478185370084,
"score": 32.141977592905654,
"task_avg": 50.17018019744989
},
{
"model": "kirp__jpt-4b",
"borda": 61.964382265477184,
"noul": 55.383345312989064,
"choice": 56.752909682248124,
"score": 36.06967904698538,
"task_avg": 49.40197801407418
},
{
"model": "vllm-sr__Decision-2.0-Nox-4B",
"borda": 61.94917545282511,
"noul": 51.699137853991296,
"choice": 57.16912445060331,
"score": 37.10346511290773,
"task_avg": 48.65724247250078
},
{
"model": "vllm-sr__d3-nano",
"borda": 61.78865909705327,
"noul": 51.40453253890441,
"choice": 60.33571490530164,
"score": 30.213438941645656,
"task_avg": 47.31789546195057
},
{
"model": "frontier-infra__jebadiah-9b-v0",
"borda": 61.47438496891052,
"noul": 52.107687627842125,
"choice": 56.65043516037408,
"score": 38.22000861676978,
"task_avg": 48.99271046832867
},
{
"model": "bespokelabs__Bespoke-Nimble-9B",
"borda": 61.16349013246825,
"noul": 52.59016873865345,
"choice": 56.98874631196198,
"score": 31.783933399654117,
"task_avg": 47.12094948342318
},
{
"model": "5ak3t__scm_max_14b_v0",
"borda": 60.916801838334685,
"noul": 50.09046799789556,
"choice": 55.36828314970512,
"score": 32.3116435197021,
"task_avg": 45.92346488910093
},
{
"model": "OURS:rl60",
"borda": 60.50790754257907,
"noul": 50.70371577975563,
"choice": 57.19274044054583,
"score": 35.767117161735676,
"task_avg": 47.88785779401238
},
{
"model": "openai__gpt-6-luna-decisions",
"borda": 60.448769937821034,
"noul": 54.54085821490578,
"choice": 56.23257426488686,
"score": 36.84663806748401,
"task_avg": 49.206690182425554
},
{
"model": "mohit67890__imajev-9b",
"borda": 60.403149499864845,
"noul": 54.001619796730594,
"choice": 58.287980587192536,
"score": 23.14132659704442,
"task_avg": 45.14364232698918
},
{
"model": "empiriolabsai__aplomb-1",
"borda": 60.30346039470126,
"noul": 52.43316942953806,
"choice": 54.79683599028732,
"score": 35.009688942999766,
"task_avg": 47.41323145427504
},
{
"model": "shgao__rsi-jev-v5.0-vl-3b",
"borda": 60.07366855907002,
"noul": 56.703209225209065,
"choice": 55.45133278638838,
"score": 29.60071776434164,
"task_avg": 47.25175325864635
},
{
"model": "akhilaaa3__Jev-Omni",
"borda": 60.04156528791562,
"noul": 55.66517298237154,
"choice": 54.75495804543774,
"score": 33.380992759919344,
"task_avg": 47.93370792924287
},
{
"model": "mohit67890__imajev-4b",
"borda": 59.9249797242498,
"noul": 53.48572346871568,
"choice": 56.70130078423596,
"score": 28.40857443345301,
"task_avg": 46.19853289546822
},
{
"model": "sthanika-ai__Sieve-9B",
"borda": 59.6140848878075,
"noul": 51.240803885533225,
"choice": 57.24807805511779,
"score": 28.67039450941818,
"task_avg": 45.719758816689726
},
{
"model": "eljiwo__jiwo-4b",
"borda": 59.49074074074073,
"noul": 52.65877502202466,
"choice": 55.02139973370082,
"score": 31.00492474270615,
"task_avg": 46.22836649947721
},
{
"model": "michaljach__jet",
"borda": 59.28798323871317,
"noul": 51.024653480610084,
"choice": 53.74774297160457,
"score": 33.01172599080244,
"task_avg": 45.928040814339035
},
{
"model": "everettjf__ezjev-4b-s2",
"borda": 59.068329278183285,
"noul": 50.89605291103704,
"choice": 57.34730010900303,
"score": 34.75755858279741,
"task_avg": 47.66697053427916
},
{
"model": "alibiserikbay__JevK5-9B",
"borda": 58.72195187888618,
"noul": 52.8766435353212,
"choice": 56.67250455930102,
"score": 23.87040931557622,
"task_avg": 44.47318580339948
},
{
"model": "ZefanCai__Open-Jev-9B",
"borda": 58.61381454447151,
"noul": 55.75509897557962,
"choice": 55.51380128740718,
"score": 45.03514620643869,
"task_avg": 52.10134882314183
},
{
"model": "jsaurabh__qwen3.5-9b-jev-data-mix-v2",
"borda": 58.0308867261422,
"noul": 51.19384350959355,
"choice": 51.08361612498409,
"score": 33.53590738843635,
"task_avg": 45.27112234100466
},
{
"model": "apus-ailab__APUS-OpenJev-v1-9B",
"borda": 57.758853744255205,
"noul": 49.13784280054897,
"choice": 53.91768197623167,
"score": 31.09373253631511,
"task_avg": 44.71641910436525
},
{
"model": "PelaAI__KnowLine-2B-Gen1",
"borda": 57.04920248715871,
"noul": 48.42044984584481,
"choice": 57.3876989661614,
"score": 28.73278522706909,
"task_avg": 44.8469780130251
},
{
"model": "togethercomputer__Tev1-4B-experimental",
"borda": 56.95796161124626,
"noul": 50.728319922771725,
"choice": 53.35818184686657,
"score": 37.111928168339126,
"task_avg": 47.06614331265914
},
{
"model": "Mapika__decider-4b",
"borda": 56.854893214382294,
"noul": 52.40292363906677,
"choice": 53.992784562008985,
"score": 22.117985237066858,
"task_avg": 42.8378978127142
},
{
"model": "frontier-infra__jebadiah-4b-v2",
"borda": 56.76703163017033,
"noul": 48.858914095700776,
"choice": 54.819709035764284,
"score": 32.87631710043918,
"task_avg": 45.518313410634754
},
{
"model": "apus-ailab__APUS-OpenJev-v1-4B",
"borda": 56.481481481481474,
"noul": 50.213300414471185,
"choice": 51.4927844560767,
"score": 31.6552861011774,
"task_avg": 44.45379032390843
},
{
"model": "OmniJev__OneJev1.6-4B",
"borda": 56.371654501216575,
"noul": 46.99999602368463,
"choice": 57.112714382717165,
"score": 26.095264486295598,
"task_avg": 43.4026582975658
},
{
"model": "CountingSheep__vev-9b",
"borda": 56.246620708299545,
"noul": 49.17221842703725,
"choice": 56.61914987767809,
"score": 23.004241643862912,
"task_avg": 42.931869982859425
},
{
"model": "EldanRing__Winnow-E4B",
"borda": 55.73972695323064,
"noul": 51.540622380366926,
"choice": 52.334647835879785,
"score": 32.22216291323428,
"task_avg": 45.365811043160335
},
{
"model": "StandardThinking__StandardOne-8B",
"borda": 55.64848607731817,
"noul": 52.9291533583201,
"choice": 49.06930731501095,
"score": 26.35668319514521,
"task_avg": 42.785047956158756
},
{
"model": "jaredpalmer__kev-9b",
"borda": 55.62821032711544,
"noul": 50.20671642811111,
"choice": 56.36143243775897,
"score": 20.63580286164106,
"task_avg": 42.40131724250371
},
{
"model": "evalengine__decision-4b",
"borda": 55.239591781562574,
"noul": 50.164139724054266,
"choice": 53.17461756459602,
"score": 33.65457848947355,
"task_avg": 45.66444525937462
},
{
"model": "abhishek085__spark-s1-4b-v6",
"borda": 55.1449716139497,
"noul": 51.80307006362222,
"choice": 49.76591502898487,
"score": 33.83453084970998,
"task_avg": 45.134505314105695
},
{
"model": "OURS:v1_joint",
"borda": 54.79183563125168,
"noul": 50.07962257490453,
"choice": 52.30924425272342,
"score": 27.975023045764445,
"task_avg": 43.45462995779747
},
{
"model": "TokenRhythm__NeoHorse-Jev-4B",
"borda": 53.801703163017045,
"noul": 48.33247276234772,
"choice": 52.761591848959476,
"score": 29.700346169829544,
"task_avg": 43.59813692704558
},
{
"model": "jaredpalmer__kev-4b",
"borda": 53.64287645309544,
"noul": 48.4774594752686,
"choice": 52.034952823700706,
"score": 26.470899581663804,
"task_avg": 42.3277706268777
},
{
"model": "bespokelabs__Bespoke-Nimble-9B-v2",
"borda": 53.28129224114622,
"noul": 46.02391167595726,
"choice": 55.71719728128821,
"score": 24.460288843686584,
"task_avg": 42.06713260031068
},
{
"model": "uneeverse__unee-2b",
"borda": 53.13429305217627,
"noul": 45.59951252551379,
"choice": 52.64336596463589,
"score": 35.00483267241027,
"task_avg": 44.41590372085331
},
{
"model": "caiovicentino1__Eikos-4B",
"borda": 52.80988104893216,
"noul": 49.03191091243997,
"choice": 52.20046871910717,
"score": 22.24212386108504,
"task_avg": 41.158167830877396
},
{
"model": "flymy-ai__decision-4b-v1.1",
"borda": 52.49391727493917,
"noul": 47.715308394774105,
"choice": 48.47537367583929,
"score": 28.68205617290122,
"task_avg": 41.62424608117155
},
{
"model": "dhtocks__malkuth-4b",
"borda": 52.291159772911605,
"noul": 48.13267469024474,
"choice": 53.78319033513599,
"score": 21.089741327302296,
"task_avg": 41.001868784227675
},
{
"model": "alibiserikbay__JevK5",
"borda": 52.15598810489322,
"noul": 48.92717406282948,
"choice": 52.84174756452853,
"score": 21.648399133484318,
"task_avg": 41.139106920280774
},
{
"model": "interfaze-ai__lev",
"borda": 52.123884833738856,
"noul": 45.00380964688521,
"choice": 52.978836772077834,
"score": 23.021133906488345,
"task_avg": 40.33459344181713
},
{
"model": "HopitAI__hopper",
"borda": 51.91943768586106,
"noul": 49.92860136088426,
"choice": 49.00742580162692,
"score": 22.50867013185354,
"task_avg": 40.481565764788236
},
{
"model": "CountingSheep__vev-4b",
"borda": 51.4683022438497,
"noul": 47.938653157953674,
"choice": 51.19792526452689,
"score": 21.055236490664544,
"task_avg": 40.0639383043817
},
{
"model": "hotchpotch__bekko-system-one-v0-68m",
"borda": 51.44802649364691,
"noul": 42.913061338083594,
"choice": 51.6181937632021,
"score": 26.845989503564372,
"task_avg": 40.45908153495002
},
{
"model": "flymy-ai__decision-4b-v1.2",
"borda": 50.02027575020277,
"noul": 47.32481945457459,
"choice": 48.07060178870557,
"score": 25.472043244269898,
"task_avg": 40.28915482918335
},
{
"model": "vllm-sr__d3-lite",
"borda": 49.619829683698306,
"noul": 43.48613150148678,
"choice": 51.892808643346754,
"score": 20.68741614064444,
"task_avg": 38.68878542849266
},
{
"model": "oraculumai__Manchego",
"borda": 48.25966477426332,
"noul": 47.04604178545591,
"choice": 45.14757039190996,
"score": 19.804282017145038,
"task_avg": 37.33263139817031
},
{
"model": "hotchpotch__saihai-v1-310m",
"borda": 47.52973776696408,
"noul": 40.92421104645248,
"choice": 47.198442747238445,
"score": 33.65849860409062,
"task_avg": 40.59371746592718
},
{
"model": "AlexWortega__openjev--qwen3.5-4b-nli-v5",
"borda": 47.33711814003785,
"noul": 42.771292706774986,
"choice": 47.49208578199777,
"score": 19.105679109219206,
"task_avg": 36.456352532663985
},
{
"model": "crh225__plumb-4b",
"borda": 47.1512570965126,
"noul": 44.76249223405814,
"choice": 45.370383998009764,
"score": 21.489386935461948,
"task_avg": 37.207421055843284
},
{
"model": "wayfind__metask-jev-4b-policy-mix",
"borda": 46.61901865369019,
"noul": 44.06001452621817,
"choice": 46.15624352503484,
"score": 20.36866645662884,
"task_avg": 36.86164150262729
},
{
"model": "AmirBraham__jevstral-8b",
"borda": 46.41288185996213,
"noul": 44.108287297215774,
"choice": 48.90426519874536,
"score": 12.887721999075769,
"task_avg": 35.30009149834563
},
{
"model": "kshetrajna12__reflex-qwen3.5-4b-lora",
"borda": 45.84178156258446,
"noul": 46.897123335085354,
"choice": 44.688947310079996,
"score": 15.888206351428474,
"task_avg": 35.82475899886461
},
{
"model": "jaredpalmer__kev-8b",
"borda": 45.75391997837253,
"noul": 41.040239693696904,
"choice": 47.53947005827517,
"score": 19.445423228052107,
"task_avg": 36.008377660008065
},
{
"model": "kpiya__kas-4b",
"borda": 45.4497837253312,
"noul": 42.308812674285434,
"choice": 51.23386959437334,
"score": 14.33021359577897,
"task_avg": 35.95763195481258
},
{
"model": "OURS:v2_2b_ro",
"borda": 44.95302784536361,
"noul": 42.252965794032946,
"choice": 45.682505057707964,
"score": 23.950261202154582,
"task_avg": 37.29524401796516
},
{
"model": "uneeverse__unee-0.8b",
"borda": 43.87672343876726,
"noul": 42.43363587439376,
"choice": 41.416221928478066,
"score": 22.508777339193227,
"task_avg": 35.452878380688354
},
{
"model": "chaoliangUNSW__Jev-Style-2B-Decision-v3",
"borda": 43.42389835090567,
"noul": 39.99393767141549,
"choice": 45.94280668155854,
"score": 18.369204941297397,
"task_avg": 34.768649764757136
},
{
"model": "vllm-sr__Decision-2.0-Sol-2B",
"borda": 43.40531224655312,
"noul": 37.45424316002706,
"choice": 44.19198049340873,
"score": 18.786574596041465,
"task_avg": 33.477599416492424
},
{
"model": "pngwn__system-one-qwen3.5-4b-scorer",
"borda": 43.00993511759934,
"noul": 40.278468045993684,
"choice": 43.931154644733134,
"score": 12.705857703586062,
"task_avg": 32.305160131437624
},
{
"model": "PelaAI__KnowLine-0.8B-Gen1",
"borda": 42.572316842389796,
"noul": 37.08432082593905,
"choice": 47.15612284001329,
"score": 18.259239046267105,
"task_avg": 34.16656090407315
},
{
"model": "GestaltLabs__Jeff-1",
"borda": 42.09921600432553,
"noul": 38.370081367789524,
"choice": 38.63294657292472,
"score": 32.916423644618426,
"task_avg": 36.63981719511089
},
{
"model": "5ak3t__scm_mid_4b_v1",
"borda": 41.68525277101919,
"noul": 45.92198337424015,
"choice": 38.21709098037366,
"score": 6.303910416270853,
"task_avg": 30.147661590294888
},
{
"model": "UserMoonlight__intelif-qwen3-4b",
"borda": 41.621046228710455,
"noul": 35.931771284623764,
"choice": 44.584519858722324,
"score": 11.26538983929321,
"task_avg": 30.59389366087976
},
{
"model": "pngwn__system-one-qwen3.5-4b-scorer-v2b",
"borda": 40.71201676128682,
"noul": 40.73096581608453,
"choice": 47.49872109068542,
"score": 11.255482531382844,
"task_avg": 33.16172314605093
},
{
"model": "mstrasser__Jeff-Gemma4-E2B",
"borda": 40.28960529872938,
"noul": 37.490263602369964,
"choice": 41.97100505496683,
"score": 14.528356623257475,
"task_avg": 31.329875093531424
},
{
"model": "mstrasser__Jeff-Qwen3.5-2B",
"borda": 40.10543390105433,
"noul": 37.88530508490729,
"choice": 43.103576532513785,
"score": 15.156137149433471,
"task_avg": 32.048339588951514
},
{
"model": "OURS:base",
"borda": 40.046296296296305,
"noul": 33.76706817616705,
"choice": 45.70255576401406,
"score": 18.882231543636387,
"task_avg": 32.78395182793917
},
{
"model": "flymy-ai__decision-2b-preview",
"borda": 39.99560692078941,
"noul": 37.88320498017567,
"choice": 37.75292647800017,
"score": 17.14077374439567,
"task_avg": 30.92563506752384
},
{
"model": "davidburhans__gevva-e4b",
"borda": 39.64585022979185,
"noul": 40.546151435239565,
"choice": 42.80001198616829,
"score": 4.22575655315711,
"task_avg": 29.190639991521657
},
{
"model": "flock-io__this-that-model-1.2",
"borda": 39.24540416328738,
"noul": 35.1017754925684,
"choice": 43.488813331032325,
"score": 15.372714102370214,
"task_avg": 31.321100975323645
},
{
"model": "ZefanCai__Open-Jev-2B",
"borda": 39.123749662070814,
"noul": 38.83305311842163,
"choice": 39.82442308077451,
"score": 25.268369720547007,
"task_avg": 34.64194863991438
},
{
"model": "Mapika__decider-2b",
"borda": 39.12206001622061,
"noul": 38.57729999894551,
"choice": 39.31728653105371,
"score": 10.388387284592177,
"task_avg": 29.42765793819714
},
{
"model": "mohit67890__imajev-2b",
"borda": 38.97675047310085,
"noul": 37.70378357523272,
"choice": 43.031172624025935,
"score": 11.685794338472851,
"task_avg": 30.806916845910504
},
{
"model": "vllm-sr__d3-edge",
"borda": 38.06772100567721,
"noul": 32.99872964035195,
"choice": 41.35124914551756,
"score": 13.657641531976411,
"task_avg": 29.335873439281972
},
{
"model": "hotchpotch__bekko-system-one-v0-17m",
"borda": 37.78048121113814,
"noul": 31.43315941534458,
"choice": 35.56540565712225,
"score": 15.700463916883209,
"task_avg": 27.566342996450015
},
{
"model": "dhtocks__malkuth-2b",
"borda": 37.412138415788036,
"noul": 35.926391917345285,
"choice": 39.012742590961224,
"score": 12.353633749590351,
"task_avg": 29.09758941929896
},
{
"model": "OmniJev__OneJev1.6-0.8B",
"borda": 37.38848337388484,
"noul": 36.62715426323174,
"choice": 39.09571539244136,
"score": 14.372384217745191,
"task_avg": 30.03175129113943
},
{
"model": "davidburhans__gevva-e2b",
"borda": 37.38003514463368,
"noul": 39.150340975100036,
"choice": 39.074866297421956,
"score": 6.018645382635025,
"task_avg": 28.081284218385676
},
{
"model": "AnkitAI__TinyJev-4B",
"borda": 37.33779399837793,
"noul": 32.24141943416321,
"choice": 42.672562541753514,
"score": 12.398819911183608,
"task_avg": 29.104267295700115
},
{
"model": "kirp__jpt-0.8b",
"borda": 36.883279264666136,
"noul": 38.354356448823864,
"choice": 36.85227432583024,
"score": 11.870674563388144,
"task_avg": 29.025768446014084
},
{
"model": "eljiwo__jiwo-0.8b",
"borda": 36.06211138145445,
"noul": 36.35547403770529,
"choice": 33.71114762929817,
"score": 15.295423952393103,
"task_avg": 28.45401520646552
},
{
"model": "OmniJev__OneJev1.5-0.8B",
"borda": 34.97566909975668,
"noul": 32.303165490281074,
"choice": 37.50987898646421,
"score": 14.579994355364848,
"task_avg": 28.13101294403671
},
{
"model": "vllm-sr__Decision-2.0-Eos-0.8B",
"borda": 34.31670721816707,
"noul": 28.84799678052876,
"choice": 34.95839771196565,
"score": 15.532451501232206,
"task_avg": 26.446281997908873
},
{
"model": "carrtesy__EXAONE-4.0-1.2B-JEV",
"borda": 33.84698567180318,
"noul": 32.12673137770427,
"choice": 39.26135941492968,
"score": 8.16470117641795,
"task_avg": 26.5175973230173
},
{
"model": "mstrasser__Jeff-Qwen3.5-0.8B",
"borda": 33.05623141389564,
"noul": 31.224289193970566,
"choice": 34.695601494309024,
"score": 11.998621454259574,
"task_avg": 25.972837380846386
},
{
"model": "sthanika-ai__sieve-2b",
"borda": 32.990335225736686,
"noul": 33.799584652254325,
"choice": 35.150391005039495,
"score": 7.322320615018016,
"task_avg": 25.42409875743728
},
{
"model": "Hanno-Labs__bosun-v3.1-1.7b",
"borda": 32.69295755609627,
"noul": 28.04997349025108,
"choice": 28.081958912239397,
"score": 20.427989896267544,
"task_avg": 25.51997409958601
},
{
"model": "Mapika__decider-2b-vision",
"borda": 32.61354420113544,
"noul": 34.10575498627405,
"choice": 32.24739614073577,
"score": 10.188694403829833,
"task_avg": 25.51394851027989
},
{
"model": "AlexWortega__openjev--qwen3.5-2b-nli-v5",
"borda": 31.596377399297104,
"noul": 29.723388227975022,
"choice": 37.79538324399946,
"score": 8.112348380903406,
"task_avg": 25.21037328429263
},
{
"model": "OmniJev__OneJev-0.8B",
"borda": 31.211138145444703,
"noul": 30.463820211555774,
"choice": 30.748756859069452,
"score": 7.925767920224533,
"task_avg": 23.04611499694992
},
{
"model": "chaoliangUNSW__Jev-Style-0.8B-Decision-v3",
"borda": 31.042173560421734,
"noul": 28.184107627974807,
"choice": 28.836727605936392,
"score": 9.672658176118954,
"task_avg": 22.231164470010047
},
{
"model": "Mapika__decider-0.8b",
"borda": 30.986415247364157,
"noul": 28.9673685086067,
"choice": 32.357637367388755,
"score": 6.392707984749325,
"task_avg": 22.572571286914926
},
{
"model": "vllm-sr__Decision-2.0-Kai-0.6B",
"borda": 30.76676128683428,
"noul": 28.39671715679961,
"choice": 34.27585259456087,
"score": 6.968857069956099,
"task_avg": 23.213808940438863
},
{
"model": "AndeyTait__JevForge-0.8B",
"borda": 30.640037848067045,
"noul": 29.860761769678785,
"choice": 25.032095488629245,
"score": 15.694452283576018,
"task_avg": 23.52910318062802
},
{
"model": "togethercomputer__Tev1-0.8B-experimental",
"borda": 27.618951067856177,
"noul": 25.97343515742046,
"choice": 27.1457200719027,
"score": 8.163037094833115,
"task_avg": 20.427397441385423
},
{
"model": "OURS:base_2b",
"borda": 26.334820221681532,
"noul": 22.871795602410288,
"choice": 29.531777580453237,
"score": 2.0098659488690807,
"task_avg": 18.13781304391087
},
{
"model": "Hanno-Labs__bosun-v3.1-0.6b",
"borda": 26.1574074074074,
"noul": 24.90439276593173,
"choice": 20.491384948080473,
"score": 12.559630477597267,
"task_avg": 19.318469397203156
},
{
"model": "LMDHQ-0420__vision-jev-9b",
"borda": 25.98844282238442,
"noul": 25.295203567189844,
"choice": 25.983634193949495,
"score": 0.6645502392581939,
"task_avg": 17.31446266679918
},
{
"model": "divyanshx11__JEVision",
"borda": 25.951270613679373,
"noul": 22.40803103996612,
"choice": 26.880345865049794,
"score": 6.197743716965625,
"task_avg": 18.49537354066051
},
{
"model": "jaredpalmer__kev-0.8b",
"borda": 25.88368477967018,
"noul": 23.611325424298922,
"choice": 27.36222624924885,
"score": 5.071763261606087,
"task_avg": 18.68177164505129
},
{
"model": "iapp__OpenThai-SystemOne",
"borda": 25.415652879156532,
"noul": 22.367327037534988,
"choice": 26.22135868241336,
"score": 7.3714975885457505,
"task_avg": 18.653394436164696
},
{
"model": "Lukitaduarte__dinah-0",
"borda": 25.29737766964044,
"noul": 16.820408366046614,
"choice": 26.739988868819776,
"score": 11.361591789920888,
"task_avg": 18.307329674929093
},
{
"model": "FrontiersMind__Lumma-fev-0.6b",
"borda": 24.76344958096784,
"noul": 17.60554805520938,
"choice": 21.15225656054735,
"score": 24.3078689655222,
"task_avg": 21.021891193759643
},
{
"model": "wfzyx__von",
"borda": 24.0757637199243,
"noul": 20.150537695397656,
"choice": 23.985557398038075,
"score": 4.4792196377932,
"task_avg": 16.205104910409645
},
{
"model": "Qwen__Qwen3-Reranker-4B",
"borda": 23.89666125979994,
"noul": 17.62881435769582,
"choice": 28.913615050637947,
"score": 1.620679097735405,
"task_avg": 16.054369502023054
},
{
"model": "mohamedlotfy50__sifr-0.8b-v3",
"borda": 23.88145444714789,
"noul": 23.666774320228065,
"choice": 23.95722554543709,
"score": 1.6177606557694186,
"task_avg": 16.413920173811526
},
{
"model": "convaiinnovations__laya-typed-decisions",
"borda": 23.670248715869143,
"noul": 20.062833690097438,
"choice": 18.931880597379365,
"score": 5.994198619698101,
"task_avg": 14.996304302391636
},
{
"model": "AlexWortega__openjev--qwen3.5-0.8b-nli-v2s-long",
"borda": 23.095769126791026,
"noul": 18.902225279875033,
"choice": 23.79105791052257,
"score": 5.359923474119997,
"task_avg": 16.0177355548392
},
{
"model": "moganai__lavoir",
"borda": 22.62773722627737,
"noul": 18.93986239283059,
"choice": 18.203270080976015,
"score": 2.9992074600505156,
"task_avg": 13.380779977952376
},
{
"model": "zeroentropy__zerank-2-reranker",
"borda": 22.57704785077047,
"noul": 13.326420068720996,
"choice": 31.962061490160533,
"score": 1.4433007397282875,
"task_avg": 15.57726076620327
},
{
"model": "convaiinnovations__laya",
"borda": 22.13098134630981,
"noul": 20.192323410725066,
"choice": 16.305950464357625,
"score": 3.5838472739440608,
"task_avg": 13.360707049675586
},
{
"model": "jaredpalmer__kev-0.6b",
"borda": 22.056636928899703,
"noul": 14.688412314113597,
"choice": 24.025031451357542,
"score": 3.5063424994417294,
"task_avg": 14.07326208830429
},
{
"model": "flymy-ai__decision-fast-preview",
"borda": 21.718707758853743,
"noul": 18.22846972115562,
"choice": 23.740218206765924,
"score": 2.0278130142512443,
"task_avg": 14.665500314057597
},
{
"model": "LibertAIDAI__deem-0.8-v1",
"borda": 21.488915923222493,
"noul": 18.10126399278811,
"choice": 13.611952791316394,
"score": 2.1374360232394505,
"task_avg": 11.283550935781319
},
{
"model": "PIXELZX__XERON-0.1",
"borda": 20.6728169775615,
"noul": 15.031335732742432,
"choice": 15.554782929993724,
"score": 4.132084549379842,
"task_avg": 11.572734404038666
},
{
"model": "monotykamary__LFM2.5-2.6B-RLCD",
"borda": 20.517369559340366,
"noul": 19.154322729808044,
"choice": 13.759507629852768,
"score": 2.626183028303241,
"task_avg": 11.84667112932135
},
{
"model": "AnkitAI__tinyjev-0.6b",
"borda": 19.84995944849959,
"noul": 10.935794989960982,
"choice": 23.101353232919642,
"score": 3.250148445911695,
"task_avg": 12.429098889597439
},
{
"model": "mghafiri__qwen3.5-0.8B-decision-model",
"borda": 18.98317112733171,
"noul": 14.329981250198824,
"choice": 19.49529077682183,
"score": 3.699777909110459,
"task_avg": 12.508349978710367
},
{
"model": "mstrasser__Jeff-Qwen3.5-0.8B-Chess",
"borda": 18.844620167612867,
"noul": 14.50634308737809,
"choice": 19.390990393552094,
"score": 2.429120508652862,
"task_avg": 12.108817996527682
},
{
"model": "LMDHQ-0420__vision-jev-0.8b",
"borda": 18.663828061638288,
"noul": 20.881038870188878,
"choice": 9.818740425946064,
"score": 0.1983723251182467,
"task_avg": 10.299383873751063
},
{
"model": "convaiinnovations__laya-multilingual",
"borda": 18.525277101919436,
"noul": 14.010551486645076,
"choice": 13.030684525327096,
"score": 0.7943755407276067,
"task_avg": 9.278537184233258
},
{
"model": "isHeSatoshi__smalljev-semantic-v9",
"borda": 18.060624493106246,
"noul": 19.52428131944893,
"choice": 1.045241269905861,
"score": 2.9785716827411277,
"task_avg": 7.849364757365307
},
{
"model": "com-kotobalabs__open-jev-deberta-v3-large",
"borda": 16.974182211408486,
"noul": 6.990852278746,
"choice": 21.675329226292998,
"score": 1.4836375647889881,
"task_avg": 10.049939689942661
},
{
"model": "jaredpalmer__kev-0.5b",
"borda": 16.644701270613677,
"noul": 9.708535887222626,
"choice": 18.649924917914177,
"score": 1.7977004241041845,
"task_avg": 10.052053743080329
},
{
"model": "alibiserikbay__JevK5-Lite",
"borda": 16.11922141119221,
"noul": 3.76654702738377,
"choice": 17.7797951838272,
"score": 1.5654975915267526,
"task_avg": 7.703946600912574
},
{
"model": "franckverrot__lev-350m",
"borda": 16.117531765341983,
"noul": 5.935991628257736,
"choice": 16.405108374637887,
"score": 2.3352033016494516,
"task_avg": 8.225434434848358
},
{
"model": "Contrastive-LM__CLM-v0.1-8B",
"borda": 16.014463368477966,
"noul": 14.412694524927733,
"choice": 5.857365116987648,
"score": 1.1908580239087454,
"task_avg": 7.153639221941376
},
{
"model": "knowledgator__gliformer-large-v1",
"borda": 16.00770478507705,
"noul": 11.484628873458307,
"choice": 13.644376375042802,
"score": 0.7360329168838754,
"task_avg": 8.621679388461661
},
{
"model": "jerichosiahaya__Naluri-308M-encoder",
"borda": 15.89449851311165,
"noul": 9.151701070343478,
"choice": 12.348336262914215,
"score": 2.103617106215122,
"task_avg": 7.867884813157605
},
{
"model": "fastino__GLiNER2.5-Decide",
"borda": 15.82691267910246,
"noul": 5.400377848791741,
"choice": 18.19382225695684,
"score": 3.86811588722709,
"task_avg": 9.154105330991891
},
{
"model": "zeredy879__minojev",
"borda": 15.573465801567995,
"noul": 8.56832688483775,
"choice": 14.56405132767957,
"score": 0.6714324571009406,
"task_avg": 7.93460355653942
},
{
"model": "MoritzLaurer__ModernBERT-large-zeroshot-v2.0",
"borda": 15.48560421735604,
"noul": 7.331183111706821,
"choice": 17.666254986073167,
"score": 3.6760704109207163,
"task_avg": 9.557836169566901
},
{
"model": "fastino__gliner2.5-base-v1",
"borda": 15.304812111381457,
"noul": 8.439469009276774,
"choice": 14.47523737644395,
"score": 2.022515426402544,
"task_avg": 8.312407270707755
},
{
"model": "fastino__gliner2.5-small-v1",
"borda": 14.61036766693701,
"noul": 8.561408258769246,
"choice": 9.659802627656926,
"score": 0.8577162506172501,
"task_avg": 6.359642379014472
},
{
"model": "fastino__gliner2-large-v1",
"borda": 14.147404703974049,
"noul": 4.573740815952506,
"choice": 14.246141445359422,
"score": 3.502160455974361,
"task_avg": 7.440680905762097
},
{
"model": "DangerLabs__DM-JEPA",
"borda": 13.632062719653959,
"noul": 2.3453534688748725,
"choice": 7.5740133411342665,
"score": 0.05678259971611506,
"task_avg": 3.325383136575085
},
{
"model": "Manav2op__verdict-small",
"borda": 13.422546634225467,
"noul": 2.9023875492083064,
"choice": 14.789408809054272,
"score": 0.20211982269648562,
"task_avg": 5.964638726986355
},
{
"model": "heman10x__rlcd-modernbert-151m",
"borda": 13.177547985942146,
"noul": 2.4589584837028946,
"choice": 15.542087739556877,
"score": 1.572045506770059,
"task_avg": 6.524363910009943
},
{
"model": "notnotsamuel__LFM2.5-350M-RLCD",
"borda": 12.270208164368748,
"noul": 3.4312751472189125,
"choice": 1.3549197108162505,
"score": 1.262619370781004,
"task_avg": 2.016271409605389
},
{
"model": "mixedbread-ai__mxbai-rerank-base-v2",
"borda": 11.930589348472559,
"noul": 2.0324865897741966,
"choice": 7.946053526070839,
"score": 0.0,
"task_avg": 3.326180038615012
},
{
"model": "Alibaba-NLP__gte-reranker-modernbert-base",
"borda": 11.687280346039469,
"noul": 1.4893808110867344,
"choice": 2.654646587771311,
"score": 0.0,
"task_avg": 1.3813424662860152
},
{
"model": "fastino__gliner2.5-multi-v1",
"borda": 11.63490132468235,
"noul": 3.0257346247767076,
"choice": 6.470804851610941,
"score": 0.2718941682039498,
"task_avg": 3.256144548197199
},
{
"model": "Cactus-Compute__needle3",
"borda": 11.37638550959719,
"noul": 2.8042359077246624,
"choice": 0.6742908467164807,
"score": 0.0,
"task_avg": 1.1595089181470475
},
{
"model": "samatv256__mini-Jev",
"borda": 10.781630170316305,
"noul": 1.5711299855670875,
"choice": 5.736470461187357,
"score": 0.1576189245062373,
"task_avg": 2.4884064570868945
},
{
"model": "altslate__certo-decision-model",
"borda": 10.676872127602058,
"noul": 1.4496767390820329,
"choice": 0.33999465537549906,
"score": 0.007202716928625794,
"task_avg": 0.5989580371287192
}
]
}
{
"dataset": "hotchpotch/S1MB-english-v1@10bcb8c6",
"benchmarks": 137,
"borda_benchmarks": 137,
"cohort": 217,
"rescore_check": [
{
"model": "5ak3t__scm_max_14b_v0",
"benchmarks_checked": 16,
"max_abs_diff": 1.1102230246251565e-16
},
{
"model": "5ak3t__scm_max_14b_v1",
"benchmarks_checked": 16,
"max_abs_diff": 4.440892098500626e-16
},
{
"model": "5ak3t__scm_mid_4b_v1",
"benchmarks_checked": 16,
"max_abs_diff": 5.551115123125783e-16
}
],
"ours": {
"base": {
"temps": {
"choice": 1.707683209388825,
"noul": 2.4105066990845945,
"score": 1.2847580560947693
},
"raw": {
"adjusted": {
"aegis2-noul-test-v1": 0.42052313883299797,
"laya--ag-news-choice-test-v1": 0.7910447761194032,
"aqua-rat-choice-test-v1": 0.042253521126760604,
"arc-choice-test-v1": 0.8823529411764708,
"arct-choice-test-v1": 0.2765957446808511,
"argument-quality-choice-test-v1": 0.28571428571428575,
"argument-quality-noul-test-v1": 0.09289526462740527,
"babi-nli-noul-test-v1": 0.10000000000000009,
"banking77-choice-test-v1": 0.631578947368421,
"bbq-choice-test-v1": 0.8999999999999999,
"boardgameqa-choice-test-v1": 0.25000000000000006,
"canttalk-noul-test-v1": 0.7200000000000002,
"civil-comments-noul-test-v1": 0.3041383169192424,
"cladder-noul-test-v1": 0.040000000000000036,
"clinc-choice-test-v1": 0.7931034482758621,
"contract-nli-choice-test-v1": 0.5173535791757051,
"corr2cause-choice-test-v1": 0.0,
"creak-noul-test-v1": 0.72,
"crows-pairs-choice-test-v1": 0.43749999999999994,
"dbpedia-choice-test-v1": 0.9775280898876404,
"defeasible-nli-choice-test-v1": 0.41304347826086946,
"laya--enron-spam-noul-test-v1": 0.76,
"esci-choice-test-v1": 0.0821917808219178,
"ethics-choice-test-v1": 0.5000000000000001,
"ethics-noul-test-v1": 0.1595238095238094,
"ethos-noul-test-v1": 0.45999999999999996,
"few-nerd-choice-test-v1": 0.6451612903225806,
"fol-nli-choice-test-v1": 0.16393442622950816,
"followir-core17-noul-test-v1": 0.26398601398601396,
"followir-news21-noul-test-v1": 0.03333333333333344,
"followir-robust04-noul-test-v1": 0.4736842105263157,
"go-emotions-noul-test-v1": 0.3630486674285902,
"gretel-pii-choice-test-v1": 0.6688311688311688,
"gsm8k-choice-test-v1": 0.12857142857142861,
"hans-noul-test-v1": 0.5,
"hatecheck-noul-test-v1": 0.72,
"hh-rlhf-choice-test-v1": 0.07692307692307698,
"hwu64-choice-test-v1": 0.797872340425532,
"impli-noul-test-v1": 0.28,
"lexcomp-noul-test-v1": 0.3799999999999999,
"logical-entailment-noul-test-v1": 0.040000000000000036,
"lonli-choice-test-v1": 0.540983606557377,
"massive-choice-test-v1": 0.7806215722120659,
"miqa-choice-test-v1": 0.7272727272727273,
"mtop-choice-test-v1": 0.74873660804528,
"nlsat-noul-test-v1": 0.0,
"open-jev--amount-extraction-control-v1-choice-test-v1": 0.5294117647058824,
"open-jev--amount-extraction-control-v1-noul-test-v1": 0.0,
"open-jev--browser-control-v1-choice-test-v1": 1.0,
"open-jev--citation-control-v1-choice-test-v1": 0.6363636363636365,
"open-jev--context-retention-control-v1-noul-test-v1": 0.0,
"open-jev--customer-control-v1-choice-test-v1": 0.8260869565217391,
"open-jev--customer-control-v1-noul-test-v1": 0.75,
"open-jev--customer-control-v1-score-test-v1": 0.39788644560460573,
"open-jev--drone-control-v1-choice-test-v1": 0.0,
"open-jev--drone-control-v1-noul-test-v1": 0.0,
"open-jev--drone-control-v1-score-test-v1": 0.0,
"open-jev--email-selection-control-v1-choice-test-v1": 0.0,
"open-jev--email-selection-control-v1-noul-test-v1": 0.5,
"open-jev--entity-alignment-control-v1-noul-test-v1": 0.0,
"open-jev--entity-alignment-control-v1-score-test-v1": 0.0,
"open-jev--ir-control-v1-choice-test-v1": 0.6799999999999998,
"open-jev--ir-control-v1-noul-test-v1": 0.5714285714285714,
"open-jev--ir-control-v1-score-test-v1": 0.0,
"open-jev--mailroom-control-v1-choice-test-v1": 1.0,
"open-jev--mailroom-control-v1-noul-test-v1": 0.7305555555555556,
"open-jev--painting-geometry-v1-choice-test-v1": 0.5263157894736842,
"open-jev--painting-geometry-v1-noul-test-v1": 0.1333333333333333,
"open-jev--painting-geometry-v1-score-test-v1": 0.13055902085337423,
Xet Storage Details
- Size:
- 141 kB
- Xet hash:
- 782745c22b3866749875afc1f42488bff5d5f664940d74766192da6a07c3c3a0
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.