Buckets:

Mercity
/

Desicion-Model-Custom

Files

xet

Mercity/Desicion-Model-Custom / think-decide /results /stats.json

Pranav2748

9 days ago

download

raw

12 kB

{
 "table": {
  "S1@1": {
   "reasoning": 0.6406833333333333,
   "lookup": 0.7396,
   "other": 0.61495,
   "core17": 0.6606941176470588,
   "core14": 0.6405142857142856,
   "ece_r": 0.16933333333333334,
   "ece_l": 0.11779999999999999
  },
  "E1@1": {
   "reasoning": 0.6498666666666667,
   "lookup": 0.7410399999999999,
   "other": 0.605,
   "core17": 0.6608470588235293,
   "core14": 0.641457142857143,
   "ece_r": 0.16533333333333336,
   "ece_l": 0.13491999999999998
  },
  "E2@1": {
   "reasoning": 0.6467333333333333,
   "lookup": 0.7444600000000001,
   "other": 0.5982,
   "core17": 0.6583470588235294,
   "core14": 0.6393285714285714,
   "ece_r": 0.1665333333333333,
   "ece_l": 0.13146
  },
  "E2@2": {
   "reasoning": 0.6464666666666667,
   "lookup": 0.74448,
   "other": 0.5975333333333334,
   "core17": 0.6580235294117648,
   "core14": 0.6384142857142857,
   "ece_r": 0.16528333333333334,
   "ece_l": 0.12835999999999997
  },
  "E2@4": {
   "reasoning": 0.6481666666666667,
   "lookup": 0.74434,
   "other": 0.5982,
   "core17": 0.6588176470588236,
   "core14": 0.6396642857142857,
   "ece_r": 0.16638333333333333,
   "ece_l": 0.12762
  },
  "E2@8": {
   "reasoning": 0.6476000000000001,
   "lookup": 0.74354,
   "other": 0.5994,
   "core17": 0.6588058823529411,
   "core14": 0.6396928571428572,
   "ece_r": 0.16634999999999997,
   "ece_l": 0.12882
  },
  "E3@1": {
   "reasoning": 0.6450666666666667,
   "lookup": 0.75312,
   "other": 0.5956333333333333,
   "core17": 0.6594000000000001,
   "core14": 0.6401785714285715,
   "ece_r": 0.16726666666666667,
   "ece_l": 0.12766
  },
  "E3@2": {
   "reasoning": 0.6408166666666667,
   "lookup": 0.7527600000000001,
   "other": 0.5948666666666667,
   "core17": 0.6575235294117647,
   "core14": 0.6376142857142858,
   "ece_r": 0.16968333333333332,
   "ece_l": 0.128
  },
  "E3@4": {
   "reasoning": 0.64395,
   "lookup": 0.7541399999999999,
   "other": 0.59545,
   "core17": 0.6592411764705881,
   "core14": 0.6394142857142857,
   "ece_r": 0.16671666666666665,
   "ece_l": 0.12678
  },
  "E3@8": {
   "reasoning": 0.6447,
   "lookup": 0.75344,
   "other": 0.5963166666666666,
   "core17": 0.6596058823529412,
   "core14": 0.6401428571428571,
   "ece_r": 0.17145,
   "ece_l": 0.1268
  }
 },
 "tests": [
  {
   "diff": -0.02699730914900167,
   "lo": -0.36332414447067657,
   "hi": 0.29900863107664405,
   "a": 64.6472722005679,
   "b": 64.6742695097169,
   "n_suites": 6,
   "name": "H1 E2 K=2 vs K=1",
   "group": "reasoning",
   "arm_a": "E2@2",
   "arm_b": "E2@1"
  },
  {
   "diff": 0.004137931034475795,
   "lo": -0.3291954022988497,
   "hi": 0.33241379310344765,
   "a": 74.44873563218391,
   "b": 74.44459770114943,
   "n_suites": 5,
   "name": "H1 E2 K=2 vs K=1",
   "group": "lookup",
   "arm_a": "E2@2",
   "arm_b": "E2@1"
  },
  {
   "diff": 0.14219617683347563,
   "lo": -0.17713875633987694,
   "hi": 0.47481730443776826,
   "a": 64.81646568655037,
   "b": 64.6742695097169,
   "n_suites": 6,
   "name": "H1 E2 K=4 vs K=1",
   "group": "reasoning",
   "arm_a": "E2@4",
   "arm_b": "E2@1",
   "verdict": "not met"
  },
  {
   "diff": -0.00919540229885607,
   "lo": -0.34505747126437036,
   "hi": 0.3241379310344806,
   "a": 74.43540229885058,
   "b": 74.44459770114943,
   "n_suites": 5,
   "name": "H1 E2 K=4 vs K=1",
   "group": "lookup",
   "arm_a": "E2@4",
   "arm_b": "E2@1",
   "verdict": "ok (>= -1.0)"
  },
  {
   "diff": 0.08439235366695108,
   "lo": -0.2688539974959878,
   "hi": 0.43259550598698837,
   "a": 64.75866186338385,
   "b": 64.6742695097169,
   "n_suites": 6,
   "name": "H1 E2 K=8 vs K=1",
   "group": "reasoning",
   "arm_a": "E2@8",
   "arm_b": "E2@1",
   "verdict": "not met"
  },
  {
   "diff": -0.08919540229884726,
   "lo": -0.38574712643677955,
   "hi": 0.1724137931034475,
   "a": 74.35540229885058,
   "b": 74.44459770114943,
   "n_suites": 5,
   "name": "H1 E2 K=8 vs K=1",
   "group": "lookup",
   "arm_a": "E2@8",
   "arm_b": "E2@1",
   "verdict": "ok (>= -1.0)"
  },
  {
   "diff": -0.42427117061569763,
   "lo": -1.037234941509671,
   "hi": 0.06575557999000761,
   "a": 64.0820390344333,
   "b": 64.50631020504899,
   "n_suites": 6,
   "name": "H1 E3 K=2 vs K=1",
   "group": "reasoning",
   "arm_a": "E3@2",
   "arm_b": "E3@1"
  },
  {
   "diff": -0.03494252873561532,
   "lo": -0.26482758620689717,
   "hi": 0.18252873563218586,
   "a": 75.27678160919541,
   "b": 75.31172413793102,
   "n_suites": 5,
   "name": "H1 E3 K=2 vs K=1",
   "group": "lookup",
   "arm_a": "E3@2",
   "arm_b": "E3@1"
  },
  {
   "diff": -0.1122076921301951,
   "lo": -0.4418432225787224,
   "hi": 0.19463898096527454,
   "a": 64.39410251291879,
   "b": 64.50631020504899,
   "n_suites": 6,
   "name": "H1 E3 K=4 vs K=1",
   "group": "reasoning",
   "arm_a": "E3@4",
   "arm_b": "E3@1",
   "verdict": "not met"
  },
  {
   "diff": 0.10252873563220133,
   "lo": -0.1724137931034475,
   "hi": 0.41241379310344767,
   "a": 75.41425287356323,
   "b": 75.31172413793102,
   "n_suites": 5,
   "name": "H1 E3 K=4 vs K=1",
   "group": "lookup",
   "arm_a": "E3@4",
   "arm_b": "E3@1",
   "verdict": "ok (>= -1.0)"
  },
  {
   "diff": -0.036533705704044905,
   "lo": -0.5793228220323845,
   "hi": 0.590335763159028,
   "a": 64.46977649934495,
   "b": 64.50631020504899,
   "n_suites": 6,
   "name": "H1 E3 K=8 vs K=1",
   "group": "reasoning",
   "arm_a": "E3@8",
   "arm_b": "E3@1",
   "verdict": "not met"
  },
  {
   "diff": 0.03172413793105511,
   "lo": -0.26482758620689717,
   "hi": 0.3416091954022993,
   "a": 75.34344827586207,
   "b": 75.31172413793102,
   "n_suites": 5,
   "name": "H1 E3 K=8 vs K=1",
   "group": "lookup",
   "arm_a": "E3@8",
   "arm_b": "E3@1",
   "verdict": "ok (>= -1.0)"
  },
  {
   "diff": -0.3114396421644905,
   "lo": -1.5090417705336712,
   "hi": 0.9148080381266052,
   "a": 64.6742695097169,
   "b": 64.9857091518814,
   "n_suites": 6,
   "name": "H2 E2(best K) vs E1",
   "group": "reasoning",
   "arm_a": "E2@1",
   "arm_b": "E1@1",
   "verdict": "not supported"
  },
  {
   "diff": -0.5922304752836061,
   "lo": -1.8443107151265083,
   "hi": 0.6046214022024817,
   "a": 64.0820390344333,
   "b": 64.6742695097169,
   "n_suites": 6,
   "name": "H3 E3(best K) vs E2(best K)",
   "group": "reasoning",
   "arm_a": "E3@2",
   "arm_b": "E2@1",
   "verdict": "not supported"
  },
  {
   "diff": 0.9184877332396368,
   "lo": -0.5860637261446555,
   "hi": 2.4683407881110853,
   "a": 64.9857091518814,
   "b": 64.06722141864175,
   "n_suites": 6,
   "name": "E1 vs S1",
   "group": "reasoning",
   "arm_a": "E1@1",
   "arm_b": "S1@1"
  },
  {
   "diff": -0.3114396421644905,
   "lo": -1.5090417705336712,
   "hi": 0.9148080381266052,
   "a": 64.6742695097169,
   "b": 64.9857091518814,
   "n_suites": 6,
   "name": "slot cost: E2@1 vs E1",
   "group": "reasoning",
   "arm_a": "E2@1",
   "arm_b": "E1@1"
  },
  {
   "diff": -0.9036701174480966,
   "lo": -2.2529662270835127,
   "hi": 0.4068849333645518,
   "a": 64.0820390344333,
   "b": 64.9857091518814,
   "n_suites": 6,
   "name": "E3(best) vs E1",
   "group": "reasoning",
   "arm_a": "E3@2",
   "arm_b": "E1@1"
  },
  {
   "diff": 0.14528735632183265,
   "lo": -1.380701149425287,
   "hi": 1.6528850574712646,
   "a": 74.1048275862069,
   "b": 73.95954022988506,
   "n_suites": 5,
   "name": "E1 vs S1",
   "group": "lookup",
   "arm_a": "E1@1",
   "arm_b": "S1@1"
  },
  {
   "diff": 0.33977011494252807,
   "lo": -0.5918045977011505,
   "hi": 1.286896551724137,
   "a": 74.44459770114943,
   "b": 74.1048275862069,
   "n_suites": 5,
   "name": "slot cost: E2@1 vs E1",
   "group": "lookup",
   "arm_a": "E2@1",
   "arm_b": "E1@1"
  },
  {
   "diff": 1.1719540229885084,
   "lo": 0.3121839080459776,
   "hi": 2.044160919540233,
   "a": 75.27678160919541,
   "b": 74.1048275862069,
   "n_suites": 5,
   "name": "E3(best) vs E1",
   "group": "lookup",
   "arm_a": "E3@2",
   "arm_b": "E1@1"
  },
  {
   "diff": 0.015743646394750233,
   "lo": -0.7780318058700959,
   "hi": 0.8197626966598804,
   "a": 66.08475224574127,
   "b": 66.06900859934652,
   "n_suites": 17,
   "name": "E1 vs S1",
   "group": "core17",
   "arm_a": "E1@1",
   "arm_b": "S1@1"
  },
  {
   "diff": -0.24982574538372493,
   "lo": -0.8424906248279496,
   "hi": 0.3572883101773978,
   "a": 65.83492650035755,
   "b": 66.08475224574127,
   "n_suites": 17,
   "name": "slot cost: E2@1 vs E1",
   "group": "core17",
   "arm_a": "E2@1",
   "arm_b": "E1@1"
  },
  {
   "diff": -0.33218103382520736,
   "lo": -0.9379607641375027,
   "hi": 0.282567860615002,
   "a": 65.75257121191606,
   "b": 66.08475224574127,
   "n_suites": 17,
   "name": "E3(best) vs E1",
   "group": "core17",
   "arm_a": "E3@2",
   "arm_b": "E1@1"
  },
  {
   "diff": -0.032508439813705436,
   "lo": -0.3646371725295938,
   "hi": 0.2987397510798284,
   "a": 61.79066179458904,
   "b": 61.823170234402745,
   "n_suites": 5,
   "name": "sensitivity E2 K=8 vs K=1",
   "group": "reasoning_no_mnli",
   "arm_a": "E2@8",
   "arm_b": "E2@1"
  },
  {
   "diff": -0.4406172027044186,
   "lo": -1.8293572393645052,
   "hi": 0.9341674667497024,
   "a": 61.823170234402745,
   "b": 62.26378743710717,
   "n_suites": 5,
   "name": "sensitivity E2(best) vs E1",
   "group": "reasoning_no_mnli",
   "arm_a": "E2@1",
   "arm_b": "E1@1"
  },
  {
   "diff": -0.0438404468448339,
   "lo": -0.6872245477978947,
   "hi": 0.6998517803490172,
   "a": 61.77844751827749,
   "b": 61.82228796512233,
   "n_suites": 5,
   "name": "sensitivity E3 K=8 vs K=1",
   "group": "reasoning_no_mnli",
   "arm_a": "E3@8",
   "arm_b": "E3@1"
  },
  {
   "diff": 0.0356824787358323,
   "lo": -0.15545721817280717,
   "hi": 0.2244861926166043,
   "a": 63.968824237292324,
   "b": 63.93314175855649,
   "n_suites": 14,
   "name": "sensitivity E2 K=8 vs K=1",
   "group": "core14",
   "arm_a": "E2@8",
   "arm_b": "E2@1"
  },
  {
   "diff": -0.21296327371848056,
   "lo": -0.9012121804347,
   "hi": 0.47911302752026863,
   "a": 63.93314175855649,
   "b": 64.14610503227497,
   "n_suites": 14,
   "name": "sensitivity E2(best) vs E1",
   "group": "core14",
   "arm_a": "E2@1",
   "arm_b": "E1@1"
  },
  {
   "diff": -0.00506766297547756,
   "lo": -0.2897248713392722,
   "hi": 0.3125790056706852,
   "a": 64.01324616246264,
   "b": 64.01831382543811,
   "n_suites": 14,
   "name": "sensitivity E3 K=8 vs K=1",
   "group": "core14",
   "arm_a": "E3@8",
   "arm_b": "E3@1"
  }
 ],
 "best_k": {
  "S1": 1,
  "E1": 1,
  "E2": 1,
  "E3": 2
 },
 "dev_acc": {
  "S1": {},
  "E1": {},
  "E2": {
   "1": 0.9147368421052632,
   "2": 0.9136842105263158,
   "3": 0.9136842105263158,
   "4": 0.9136842105263158,
   "5": 0.9136842105263158,
   "6": 0.9147368421052632,
   "7": 0.9136842105263158,
   "8": 0.9136842105263158
  },
  "E3": {
   "1": 0.9136842105263158,
   "2": 0.9147368421052632,
   "3": 0.9147368421052632,
   "4": 0.9147368421052632,
   "5": 0.9136842105263158,
   "6": 0.9147368421052632,
   "7": 0.9136842105263158,
   "8": 0.9147368421052632
  }
 },
 "adaptive": {
  "E2": {
   "0.001": [
    4.515272835112693,
    0.6294483985765125
   ],
   "0.003": [
    3.9443950177935942,
    0.6294483985765125
   ],
   "0.01": [
    2.997479240806643,
    0.6298932384341637
   ],
   "0.03": [
    2.07488137603796,
    0.6297449584816133
   ],
   "fixed": {
    "1": 0.6295966785290629,
    "2": 0.6291518386714117,
    "3": 0.6295966785290629,
    "4": 0.6298932384341637,
    "5": 0.630338078291815,
    "6": 0.6294483985765125,
    "7": 0.6315243179122183,
    "8": 0.6297449584816133
   }
  },
  "E3": {
   "0.001": [
    4.460409252669039,
    0.6356761565836299
   ],
   "0.003": [
    3.9304567022538555,
    0.6358244365361803
   ],
   "0.01": [
    2.9529952550415186,
    0.6361209964412812
   ],
   "0.03": [
    2.0440391459074734,
    0.6353795966785291
   ],
   "fixed": {
    "1": 0.6352313167259787,
    "2": 0.6356761565836299,
    "3": 0.6367141162514828,
    "4": 0.6361209964412812,
    "5": 0.6359727164887308,
    "6": 0.6343416370106761,
    "7": 0.6361209964412812,
    "8": 0.6343416370106761
   }
  }
 }
}

Xet Storage Details

Size:: 12 kB
Xet hash:: f5aad50c127290ca6b374f02492bd5ca1a5d66133bc96eb2ec6227b967276359

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	{
	"table": {
	"S1@1": {
	"reasoning": 0.6406833333333333,
	"lookup": 0.7396,
	"other": 0.61495,
	"core17": 0.6606941176470588,
	"core14": 0.6405142857142856,
	"ece_r": 0.16933333333333334,
	"ece_l": 0.11779999999999999
	},
	"E1@1": {
	"reasoning": 0.6498666666666667,
	"lookup": 0.7410399999999999,
	"other": 0.605,
	"core17": 0.6608470588235293,
	"core14": 0.641457142857143,
	"ece_r": 0.16533333333333336,
	"ece_l": 0.13491999999999998
	},
	"E2@1": {
	"reasoning": 0.6467333333333333,
	"lookup": 0.7444600000000001,
	"other": 0.5982,
	"core17": 0.6583470588235294,
	"core14": 0.6393285714285714,
	"ece_r": 0.1665333333333333,
	"ece_l": 0.13146
	},
	"E2@2": {
	"reasoning": 0.6464666666666667,
	"lookup": 0.74448,
	"other": 0.5975333333333334,
	"core17": 0.6580235294117648,
	"core14": 0.6384142857142857,
	"ece_r": 0.16528333333333334,
	"ece_l": 0.12835999999999997
	},
	"E2@4": {
	"reasoning": 0.6481666666666667,
	"lookup": 0.74434,
	"other": 0.5982,
	"core17": 0.6588176470588236,
	"core14": 0.6396642857142857,
	"ece_r": 0.16638333333333333,
	"ece_l": 0.12762
	},
	"E2@8": {
	"reasoning": 0.6476000000000001,
	"lookup": 0.74354,
	"other": 0.5994,
	"core17": 0.6588058823529411,
	"core14": 0.6396928571428572,
	"ece_r": 0.16634999999999997,
	"ece_l": 0.12882
	},
	"E3@1": {
	"reasoning": 0.6450666666666667,
	"lookup": 0.75312,
	"other": 0.5956333333333333,
	"core17": 0.6594000000000001,
	"core14": 0.6401785714285715,
	"ece_r": 0.16726666666666667,
	"ece_l": 0.12766
	},
	"E3@2": {
	"reasoning": 0.6408166666666667,
	"lookup": 0.7527600000000001,
	"other": 0.5948666666666667,
	"core17": 0.6575235294117647,
	"core14": 0.6376142857142858,
	"ece_r": 0.16968333333333332,
	"ece_l": 0.128
	},
	"E3@4": {
	"reasoning": 0.64395,
	"lookup": 0.7541399999999999,
	"other": 0.59545,
	"core17": 0.6592411764705881,
	"core14": 0.6394142857142857,
	"ece_r": 0.16671666666666665,
	"ece_l": 0.12678
	},
	"E3@8": {
	"reasoning": 0.6447,
	"lookup": 0.75344,
	"other": 0.5963166666666666,
	"core17": 0.6596058823529412,
	"core14": 0.6401428571428571,
	"ece_r": 0.17145,
	"ece_l": 0.1268
	}
	},
	"tests": [
	{
	"diff": -0.02699730914900167,
	"lo": -0.36332414447067657,
	"hi": 0.29900863107664405,
	"a": 64.6472722005679,
	"b": 64.6742695097169,
	"n_suites": 6,
	"name": "H1 E2 K=2 vs K=1",
	"group": "reasoning",
	"arm_a": "E2@2",
	"arm_b": "E2@1"
	},
	{
	"diff": 0.004137931034475795,
	"lo": -0.3291954022988497,
	"hi": 0.33241379310344765,
	"a": 74.44873563218391,
	"b": 74.44459770114943,
	"n_suites": 5,
	"name": "H1 E2 K=2 vs K=1",
	"group": "lookup",
	"arm_a": "E2@2",
	"arm_b": "E2@1"
	},
	{
	"diff": 0.14219617683347563,
	"lo": -0.17713875633987694,
	"hi": 0.47481730443776826,
	"a": 64.81646568655037,
	"b": 64.6742695097169,
	"n_suites": 6,
	"name": "H1 E2 K=4 vs K=1",
	"group": "reasoning",
	"arm_a": "E2@4",
	"arm_b": "E2@1",
	"verdict": "not met"
	},
	{
	"diff": -0.00919540229885607,
	"lo": -0.34505747126437036,
	"hi": 0.3241379310344806,
	"a": 74.43540229885058,
	"b": 74.44459770114943,
	"n_suites": 5,
	"name": "H1 E2 K=4 vs K=1",
	"group": "lookup",
	"arm_a": "E2@4",
	"arm_b": "E2@1",
	"verdict": "ok (>= -1.0)"
	},
	{
	"diff": 0.08439235366695108,
	"lo": -0.2688539974959878,
	"hi": 0.43259550598698837,
	"a": 64.75866186338385,
	"b": 64.6742695097169,
	"n_suites": 6,
	"name": "H1 E2 K=8 vs K=1",
	"group": "reasoning",
	"arm_a": "E2@8",
	"arm_b": "E2@1",
	"verdict": "not met"
	},
	{
	"diff": -0.08919540229884726,
	"lo": -0.38574712643677955,
	"hi": 0.1724137931034475,
	"a": 74.35540229885058,
	"b": 74.44459770114943,
	"n_suites": 5,
	"name": "H1 E2 K=8 vs K=1",
	"group": "lookup",
	"arm_a": "E2@8",
	"arm_b": "E2@1",
	"verdict": "ok (>= -1.0)"
	},
	{
	"diff": -0.42427117061569763,
	"lo": -1.037234941509671,
	"hi": 0.06575557999000761,
	"a": 64.0820390344333,
	"b": 64.50631020504899,
	"n_suites": 6,
	"name": "H1 E3 K=2 vs K=1",
	"group": "reasoning",
	"arm_a": "E3@2",
	"arm_b": "E3@1"
	},
	{
	"diff": -0.03494252873561532,
	"lo": -0.26482758620689717,
	"hi": 0.18252873563218586,
	"a": 75.27678160919541,
	"b": 75.31172413793102,
	"n_suites": 5,
	"name": "H1 E3 K=2 vs K=1",
	"group": "lookup",
	"arm_a": "E3@2",
	"arm_b": "E3@1"
	},
	{
	"diff": -0.1122076921301951,
	"lo": -0.4418432225787224,
	"hi": 0.19463898096527454,
	"a": 64.39410251291879,
	"b": 64.50631020504899,
	"n_suites": 6,
	"name": "H1 E3 K=4 vs K=1",
	"group": "reasoning",
	"arm_a": "E3@4",
	"arm_b": "E3@1",
	"verdict": "not met"
	},
	{
	"diff": 0.10252873563220133,
	"lo": -0.1724137931034475,
	"hi": 0.41241379310344767,
	"a": 75.41425287356323,
	"b": 75.31172413793102,
	"n_suites": 5,
	"name": "H1 E3 K=4 vs K=1",
	"group": "lookup",
	"arm_a": "E3@4",
	"arm_b": "E3@1",
	"verdict": "ok (>= -1.0)"
	},
	{
	"diff": -0.036533705704044905,
	"lo": -0.5793228220323845,
	"hi": 0.590335763159028,
	"a": 64.46977649934495,
	"b": 64.50631020504899,
	"n_suites": 6,
	"name": "H1 E3 K=8 vs K=1",
	"group": "reasoning",
	"arm_a": "E3@8",
	"arm_b": "E3@1",
	"verdict": "not met"
	},
	{
	"diff": 0.03172413793105511,
	"lo": -0.26482758620689717,
	"hi": 0.3416091954022993,
	"a": 75.34344827586207,
	"b": 75.31172413793102,
	"n_suites": 5,
	"name": "H1 E3 K=8 vs K=1",
	"group": "lookup",
	"arm_a": "E3@8",
	"arm_b": "E3@1",
	"verdict": "ok (>= -1.0)"
	},
	{
	"diff": -0.3114396421644905,
	"lo": -1.5090417705336712,
	"hi": 0.9148080381266052,
	"a": 64.6742695097169,
	"b": 64.9857091518814,
	"n_suites": 6,
	"name": "H2 E2(best K) vs E1",
	"group": "reasoning",
	"arm_a": "E2@1",
	"arm_b": "E1@1",
	"verdict": "not supported"
	},
	{
	"diff": -0.5922304752836061,
	"lo": -1.8443107151265083,
	"hi": 0.6046214022024817,
	"a": 64.0820390344333,
	"b": 64.6742695097169,
	"n_suites": 6,
	"name": "H3 E3(best K) vs E2(best K)",
	"group": "reasoning",
	"arm_a": "E3@2",
	"arm_b": "E2@1",
	"verdict": "not supported"
	},
	{
	"diff": 0.9184877332396368,
	"lo": -0.5860637261446555,
	"hi": 2.4683407881110853,
	"a": 64.9857091518814,
	"b": 64.06722141864175,
	"n_suites": 6,
	"name": "E1 vs S1",
	"group": "reasoning",
	"arm_a": "E1@1",
	"arm_b": "S1@1"
	},
	{
	"diff": -0.3114396421644905,
	"lo": -1.5090417705336712,
	"hi": 0.9148080381266052,
	"a": 64.6742695097169,
	"b": 64.9857091518814,
	"n_suites": 6,
	"name": "slot cost: E2@1 vs E1",
	"group": "reasoning",
	"arm_a": "E2@1",
	"arm_b": "E1@1"
	},
	{
	"diff": -0.9036701174480966,
	"lo": -2.2529662270835127,
	"hi": 0.4068849333645518,
	"a": 64.0820390344333,
	"b": 64.9857091518814,
	"n_suites": 6,
	"name": "E3(best) vs E1",
	"group": "reasoning",
	"arm_a": "E3@2",
	"arm_b": "E1@1"
	},
	{
	"diff": 0.14528735632183265,
	"lo": -1.380701149425287,
	"hi": 1.6528850574712646,
	"a": 74.1048275862069,
	"b": 73.95954022988506,
	"n_suites": 5,
	"name": "E1 vs S1",
	"group": "lookup",
	"arm_a": "E1@1",
	"arm_b": "S1@1"
	},
	{
	"diff": 0.33977011494252807,
	"lo": -0.5918045977011505,
	"hi": 1.286896551724137,
	"a": 74.44459770114943,
	"b": 74.1048275862069,
	"n_suites": 5,
	"name": "slot cost: E2@1 vs E1",
	"group": "lookup",
	"arm_a": "E2@1",
	"arm_b": "E1@1"
	},
	{
	"diff": 1.1719540229885084,
	"lo": 0.3121839080459776,
	"hi": 2.044160919540233,
	"a": 75.27678160919541,
	"b": 74.1048275862069,
	"n_suites": 5,
	"name": "E3(best) vs E1",
	"group": "lookup",
	"arm_a": "E3@2",
	"arm_b": "E1@1"
	},
	{
	"diff": 0.015743646394750233,
	"lo": -0.7780318058700959,
	"hi": 0.8197626966598804,
	"a": 66.08475224574127,
	"b": 66.06900859934652,
	"n_suites": 17,
	"name": "E1 vs S1",
	"group": "core17",
	"arm_a": "E1@1",
	"arm_b": "S1@1"
	},
	{
	"diff": -0.24982574538372493,
	"lo": -0.8424906248279496,
	"hi": 0.3572883101773978,
	"a": 65.83492650035755,
	"b": 66.08475224574127,
	"n_suites": 17,
	"name": "slot cost: E2@1 vs E1",
	"group": "core17",
	"arm_a": "E2@1",
	"arm_b": "E1@1"
	},
	{
	"diff": -0.33218103382520736,
	"lo": -0.9379607641375027,
	"hi": 0.282567860615002,
	"a": 65.75257121191606,
	"b": 66.08475224574127,
	"n_suites": 17,
	"name": "E3(best) vs E1",
	"group": "core17",
	"arm_a": "E3@2",
	"arm_b": "E1@1"
	},
	{
	"diff": -0.032508439813705436,
	"lo": -0.3646371725295938,
	"hi": 0.2987397510798284,
	"a": 61.79066179458904,
	"b": 61.823170234402745,
	"n_suites": 5,
	"name": "sensitivity E2 K=8 vs K=1",
	"group": "reasoning_no_mnli",
	"arm_a": "E2@8",
	"arm_b": "E2@1"
	},
	{
	"diff": -0.4406172027044186,
	"lo": -1.8293572393645052,
	"hi": 0.9341674667497024,
	"a": 61.823170234402745,
	"b": 62.26378743710717,
	"n_suites": 5,
	"name": "sensitivity E2(best) vs E1",
	"group": "reasoning_no_mnli",
	"arm_a": "E2@1",
	"arm_b": "E1@1"
	},
	{
	"diff": -0.0438404468448339,
	"lo": -0.6872245477978947,
	"hi": 0.6998517803490172,
	"a": 61.77844751827749,
	"b": 61.82228796512233,
	"n_suites": 5,
	"name": "sensitivity E3 K=8 vs K=1",
	"group": "reasoning_no_mnli",
	"arm_a": "E3@8",
	"arm_b": "E3@1"
	},
	{
	"diff": 0.0356824787358323,
	"lo": -0.15545721817280717,
	"hi": 0.2244861926166043,
	"a": 63.968824237292324,
	"b": 63.93314175855649,
	"n_suites": 14,
	"name": "sensitivity E2 K=8 vs K=1",
	"group": "core14",
	"arm_a": "E2@8",
	"arm_b": "E2@1"
	},
	{
	"diff": -0.21296327371848056,
	"lo": -0.9012121804347,
	"hi": 0.47911302752026863,
	"a": 63.93314175855649,
	"b": 64.14610503227497,
	"n_suites": 14,
	"name": "sensitivity E2(best) vs E1",
	"group": "core14",
	"arm_a": "E2@1",
	"arm_b": "E1@1"
	},
	{
	"diff": -0.00506766297547756,
	"lo": -0.2897248713392722,
	"hi": 0.3125790056706852,
	"a": 64.01324616246264,
	"b": 64.01831382543811,
	"n_suites": 14,
	"name": "sensitivity E3 K=8 vs K=1",
	"group": "core14",
	"arm_a": "E3@8",
	"arm_b": "E3@1"
	}
	],
	"best_k": {
	"S1": 1,
	"E1": 1,
	"E2": 1,
	"E3": 2
	},
	"dev_acc": {
	"S1": {},
	"E1": {},
	"E2": {
	"1": 0.9147368421052632,
	"2": 0.9136842105263158,
	"3": 0.9136842105263158,
	"4": 0.9136842105263158,
	"5": 0.9136842105263158,
	"6": 0.9147368421052632,
	"7": 0.9136842105263158,
	"8": 0.9136842105263158
	},
	"E3": {
	"1": 0.9136842105263158,
	"2": 0.9147368421052632,
	"3": 0.9147368421052632,
	"4": 0.9147368421052632,
	"5": 0.9136842105263158,
	"6": 0.9147368421052632,
	"7": 0.9136842105263158,
	"8": 0.9147368421052632
	}
	},
	"adaptive": {
	"E2": {
	"0.001": [
	4.515272835112693,
	0.6294483985765125
	],
	"0.003": [
	3.9443950177935942,
	0.6294483985765125
	],
	"0.01": [
	2.997479240806643,
	0.6298932384341637
	],
	"0.03": [
	2.07488137603796,
	0.6297449584816133
	],
	"fixed": {
	"1": 0.6295966785290629,
	"2": 0.6291518386714117,
	"3": 0.6295966785290629,
	"4": 0.6298932384341637,
	"5": 0.630338078291815,
	"6": 0.6294483985765125,
	"7": 0.6315243179122183,
	"8": 0.6297449584816133
	}
	},
	"E3": {
	"0.001": [
	4.460409252669039,
	0.6356761565836299
	],
	"0.003": [
	3.9304567022538555,
	0.6358244365361803
	],
	"0.01": [
	2.9529952550415186,
	0.6361209964412812
	],
	"0.03": [
	2.0440391459074734,
	0.6353795966785291
	],
	"fixed": {
	"1": 0.6352313167259787,
	"2": 0.6356761565836299,
	"3": 0.6367141162514828,
	"4": 0.6361209964412812,
	"5": 0.6359727164887308,
	"6": 0.6343416370106761,
	"7": 0.6361209964412812,
	"8": 0.6343416370106761
	}
	}
	}
	}