| [ |
| { |
| "experiment": "BASIC4 T0", |
| "dataset": "HumanEval", |
| "successful_calls": 6113, |
| "errors": 0, |
| "acceptance_length": 4.807433568954425, |
| "acceptance_rate_pct": 47.59291961193031, |
| "position_1_pct": 83.48910457231432, |
| "position_2_pct": 68.2955461615601, |
| "position_3_pct": 56.05040420414842, |
| "position_4_pct": 46.438411701067665, |
| "position_5_pct": 39.082074736092274, |
| "position_6_pct": 33.35858251014397, |
| "position_7_pct": 28.844422877439253, |
| "position_8_pct": 25.184810132676535 |
| }, |
| { |
| "experiment": "BASIC4 T0", |
| "dataset": "GSM8K", |
| "successful_calls": 112565, |
| "errors": 0, |
| "acceptance_length": 5.8452949929301266, |
| "acceptance_rate_pct": 60.566187411626586, |
| "position_1_pct": 89.111124649917, |
| "position_2_pct": 78.43314942568261, |
| "position_3_pct": 69.42534995217603, |
| "position_4_pct": 61.56513870444342, |
| "position_5_pct": 54.91736490150955, |
| "position_6_pct": 48.928548868838995, |
| "position_7_pct": 43.53965352202276, |
| "position_8_pct": 38.60916926842232 |
| }, |
| { |
| "experiment": "BASIC4 T0", |
| "dataset": "MATH500", |
| "successful_calls": 17996, |
| "errors": 0, |
| "acceptance_length": 5.315660395956892, |
| "acceptance_rate_pct": 53.945754949461154, |
| "position_1_pct": 86.4338302641223, |
| "position_2_pct": 73.8404691747072, |
| "position_3_pct": 63.20606152624376, |
| "position_4_pct": 54.37304319945573, |
| "position_5_pct": 47.034425086389675, |
| "position_6_pct": 40.77491824898309, |
| "position_7_pct": 35.334843618094816, |
| "position_8_pct": 30.568448477692662 |
| }, |
| { |
| "experiment": "BASIC4 T0", |
| "dataset": "MT-Bench", |
| "successful_calls": 4875, |
| "errors": 0, |
| "acceptance_length": 3.6381420740851347, |
| "acceptance_rate_pct": 32.976775926064185, |
| "position_1_pct": 74.9640303021086, |
| "position_2_pct": 54.310188295953424, |
| "position_3_pct": 39.48269998092062, |
| "position_4_pct": 29.28122358727341, |
| "position_5_pct": 22.363451100105905, |
| "position_6_pct": 17.565171746666408, |
| "position_7_pct": 14.180632863906885, |
| "position_8_pct": 11.666809531578215 |
| }, |
| { |
| "experiment": "SPEED-Bench T0", |
| "dataset": "coding", |
| "successful_calls": 89, |
| "errors": 0, |
| "acceptance_length": 4.0971, |
| "acceptance_rate_pct": 38.71, |
| "position_1_pct": 78.33, |
| "position_2_pct": 60.15, |
| "position_3_pct": 46.26, |
| "position_4_pct": 36.22, |
| "position_5_pct": 28.99, |
| "position_6_pct": 23.66, |
| "position_7_pct": 19.59, |
| "position_8_pct": 16.51 |
| }, |
| { |
| "experiment": "SPEED-Bench T0", |
| "dataset": "humanities", |
| "successful_calls": 88, |
| "errors": 0, |
| "acceptance_length": 3.8826, |
| "acceptance_rate_pct": 36.03, |
| "position_1_pct": 76.53, |
| "position_2_pct": 57.73, |
| "position_3_pct": 43.58, |
| "position_4_pct": 33.12, |
| "position_5_pct": 25.87, |
| "position_6_pct": 20.46, |
| "position_7_pct": 16.82, |
| "position_8_pct": 14.16 |
| }, |
| { |
| "experiment": "SPEED-Bench T0", |
| "dataset": "math", |
| "successful_calls": 98, |
| "errors": 0, |
| "acceptance_length": 3.9028, |
| "acceptance_rate_pct": 36.28, |
| "position_1_pct": 77.84, |
| "position_2_pct": 59.14, |
| "position_3_pct": 44.81, |
| "position_4_pct": 34.08, |
| "position_5_pct": 26.16, |
| "position_6_pct": 20.21, |
| "position_7_pct": 15.73, |
| "position_8_pct": 12.3 |
| }, |
| { |
| "experiment": "SPEED-Bench T0", |
| "dataset": "qa", |
| "successful_calls": 80, |
| "errors": 0, |
| "acceptance_length": 4.2508, |
| "acceptance_rate_pct": 40.64, |
| "position_1_pct": 80.65, |
| "position_2_pct": 63.71, |
| "position_3_pct": 49.73, |
| "position_4_pct": 38.84, |
| "position_5_pct": 30.85, |
| "position_6_pct": 24.68, |
| "position_7_pct": 20.06, |
| "position_8_pct": 16.56 |
| }, |
| { |
| "experiment": "SPEED-Bench T0", |
| "dataset": "rag", |
| "successful_calls": 105, |
| "errors": 0, |
| "acceptance_length": 4.9694, |
| "acceptance_rate_pct": 49.62, |
| "position_1_pct": 84.57, |
| "position_2_pct": 70.38, |
| "position_3_pct": 58.39, |
| "position_4_pct": 48.39, |
| "position_5_pct": 41.03, |
| "position_6_pct": 35.35, |
| "position_7_pct": 31.23, |
| "position_8_pct": 27.59 |
| }, |
| { |
| "experiment": "SPEED-Bench T0", |
| "dataset": "reasoning", |
| "successful_calls": 190, |
| "errors": 0, |
| "acceptance_length": 4.5232, |
| "acceptance_rate_pct": 44.04, |
| "position_1_pct": 81.24, |
| "position_2_pct": 65.24, |
| "position_3_pct": 51.8, |
| "position_4_pct": 41.39, |
| "position_5_pct": 34.17, |
| "position_6_pct": 29.37, |
| "position_7_pct": 25.84, |
| "position_8_pct": 23.27 |
| }, |
| { |
| "experiment": "SPEED-Bench T0", |
| "dataset": "stem", |
| "successful_calls": 86, |
| "errors": 0, |
| "acceptance_length": 3.8932, |
| "acceptance_rate_pct": 36.17, |
| "position_1_pct": 77.98, |
| "position_2_pct": 59.32, |
| "position_3_pct": 44.71, |
| "position_4_pct": 33.55, |
| "position_5_pct": 25.57, |
| "position_6_pct": 19.78, |
| "position_7_pct": 15.73, |
| "position_8_pct": 12.67 |
| }, |
| { |
| "experiment": "SPEED-Bench T0", |
| "dataset": "writing", |
| "successful_calls": 84, |
| "errors": 0, |
| "acceptance_length": 3.1636, |
| "acceptance_rate_pct": 27.04, |
| "position_1_pct": 70.81, |
| "position_2_pct": 48.53, |
| "position_3_pct": 32.9, |
| "position_4_pct": 22.57, |
| "position_5_pct": 15.76, |
| "position_6_pct": 11.23, |
| "position_7_pct": 8.27, |
| "position_8_pct": 6.27 |
| }, |
| { |
| "experiment": "SPEED-Bench T0", |
| "dataset": "multilingual", |
| "successful_calls": 80, |
| "errors": 0, |
| "acceptance_length": 3.9722, |
| "acceptance_rate_pct": 37.15, |
| "position_1_pct": 75.38, |
| "position_2_pct": 57.57, |
| "position_3_pct": 44.4, |
| "position_4_pct": 35.04, |
| "position_5_pct": 28.12, |
| "position_6_pct": 22.69, |
| "position_7_pct": 18.64, |
| "position_8_pct": 15.38 |
| }, |
| { |
| "experiment": "SPEED-Bench T0", |
| "dataset": "summarization", |
| "successful_calls": 80, |
| "errors": 0, |
| "acceptance_length": 4.629, |
| "acceptance_rate_pct": 45.36, |
| "position_1_pct": 85.01, |
| "position_2_pct": 69.37, |
| "position_3_pct": 55.91, |
| "position_4_pct": 45.3, |
| "position_5_pct": 36.43, |
| "position_6_pct": 28.88, |
| "position_7_pct": 23.32, |
| "position_8_pct": 18.67 |
| }, |
| { |
| "experiment": "SPEED-Bench T0", |
| "dataset": "roleplay", |
| "successful_calls": 158, |
| "errors": 0, |
| "acceptance_length": 2.9572, |
| "acceptance_rate_pct": 24.47, |
| "position_1_pct": 67.65, |
| "position_2_pct": 43.17, |
| "position_3_pct": 27.69, |
| "position_4_pct": 18.57, |
| "position_5_pct": 13.21, |
| "position_6_pct": 10.11, |
| "position_7_pct": 8.3, |
| "position_8_pct": 7.01 |
| }, |
| { |
| "experiment": "BASIC4 T1", |
| "dataset": "humaneval", |
| "successful_calls": 164, |
| "errors": 0, |
| "acceptance_length": 4.294067058567386, |
| "acceptance_rate_pct": 41.17583823209232, |
| "position_1_pct": 78.17929457870673, |
| "position_2_pct": 61.04670150228609, |
| "position_3_pct": 48.516220335292836, |
| "position_4_pct": 39.07794469845417, |
| "position_5_pct": 32.30677117352493, |
| "position_6_pct": 27.15708687132593, |
| "position_7_pct": 23.161332462442846, |
| "position_8_pct": 19.961354234704984 |
| }, |
| { |
| "experiment": "BASIC4 T1", |
| "dataset": "gsm8k", |
| "successful_calls": 1319, |
| "errors": 0, |
| "acceptance_length": 4.662285405179695, |
| "acceptance_rate_pct": 45.77856756474619, |
| "position_1_pct": 79.8866882408207, |
| "position_2_pct": 64.5829576836571, |
| "position_3_pct": 53.301130270562666, |
| "position_4_pct": 44.72777656515426, |
| "position_5_pct": 37.97841951923168, |
| "position_6_pct": 32.70572157953176, |
| "position_7_pct": 28.346919988991488, |
| "position_8_pct": 24.698926670019834 |
| }, |
| { |
| "experiment": "BASIC4 T1", |
| "dataset": "math500", |
| "successful_calls": 500, |
| "errors": 0, |
| "acceptance_length": 4.632882069795427, |
| "acceptance_rate_pct": 45.41102587244284, |
| "position_1_pct": 80.61321700762134, |
| "position_2_pct": 65.35148415563577, |
| "position_3_pct": 53.61311672683514, |
| "position_4_pct": 44.62845968712395, |
| "position_5_pct": 37.59025270758123, |
| "position_6_pct": 31.784997994384277, |
| "position_7_pct": 26.94494584837545, |
| "position_8_pct": 22.76173285198556 |
| }, |
| { |
| "experiment": "BASIC4 T1", |
| "dataset": "mtbench", |
| "successful_calls": 160, |
| "errors": 0, |
| "acceptance_length": 3.095874052030335, |
| "acceptance_rate_pct": 26.19842565037919, |
| "position_1_pct": 67.74426418354612, |
| "position_2_pct": 45.31208601324757, |
| "position_3_pct": 30.91177882307766, |
| "position_4_pct": 21.625803974272824, |
| "position_5_pct": 15.704329461457233, |
| "position_6_pct": 11.828741480272631, |
| "position_7_pct": 9.20840933090141, |
| "position_8_pct": 7.251991936258039 |
| }, |
| { |
| "experiment": "SPEED-Bench T1", |
| "dataset": "coding", |
| "successful_calls": 89, |
| "errors": 0, |
| "acceptance_length": 3.7522630712699994, |
| "acceptance_rate_pct": 34.40328839087499, |
| "position_1_pct": 73.83975732986298, |
| "position_2_pct": 54.35868483503024, |
| "position_3_pct": 40.78217101737732, |
| "position_4_pct": 31.3069356196892, |
| "position_5_pct": 24.800486105794995, |
| "position_6_pct": 19.984402510908673, |
| "position_7_pct": 16.45774324427773, |
| "position_8_pct": 13.696126464058793 |
| }, |
| { |
| "experiment": "SPEED-Bench T1", |
| "dataset": "humanities", |
| "successful_calls": 78, |
| "errors": 10, |
| "acceptance_length": 2.8185892204995118, |
| "acceptance_rate_pct": 22.732365256243895, |
| "position_1_pct": 63.61483820047356, |
| "position_2_pct": 40.73883322408465, |
| "position_3_pct": 26.604506396621762, |
| "position_4_pct": 17.83451277317744, |
| "position_5_pct": 12.423804618766694, |
| "position_6_pct": 8.951034295167641, |
| "position_7_pct": 6.6441034696489325, |
| "position_8_pct": 5.0472890720104875 |
| }, |
| { |
| "experiment": "SPEED-Bench T1", |
| "dataset": "math", |
| "successful_calls": 88, |
| "errors": 10, |
| "acceptance_length": 3.481256654324715, |
| "acceptance_rate_pct": 31.01570817905894, |
| "position_1_pct": 72.07726167585692, |
| "position_2_pct": 51.40952502026093, |
| "position_3_pct": 37.66519940462002, |
| "position_4_pct": 27.85493704545334, |
| "position_5_pct": 21.03031458733917, |
| "position_6_pct": 16.058044250928294, |
| "position_7_pct": 12.416638857548454, |
| "position_8_pct": 9.613744590464385 |
| }, |
| { |
| "experiment": "SPEED-Bench T1", |
| "dataset": "multilingual", |
| "successful_calls": 80, |
| "errors": 0, |
| "acceptance_length": 3.498545566544608, |
| "acceptance_rate_pct": 31.231819581807603, |
| "position_1_pct": 69.5424523459156, |
| "position_2_pct": 50.07699941822662, |
| "position_3_pct": 37.05383114883132, |
| "position_4_pct": 28.233120016426543, |
| "position_5_pct": 21.867834776359466, |
| "position_6_pct": 17.454912562882857, |
| "position_7_pct": 14.140515382772664, |
| "position_8_pct": 11.484891003045755 |
| }, |
| { |
| "experiment": "SPEED-Bench T1", |
| "dataset": "qa", |
| "successful_calls": 79, |
| "errors": 1, |
| "acceptance_length": 3.1040785289644686, |
| "acceptance_rate_pct": 26.300981612055857, |
| "position_1_pct": 69.17185123738422, |
| "position_2_pct": 47.02889534079912, |
| "position_3_pct": 32.15816397068989, |
| "position_4_pct": 22.065532973869765, |
| "position_5_pct": 15.285497027512788, |
| "position_6_pct": 10.996820129959906, |
| "position_7_pct": 7.902668325729296, |
| "position_8_pct": 5.798423890501867 |
| }, |
| { |
| "experiment": "SPEED-Bench T1", |
| "dataset": "rag", |
| "successful_calls": 105, |
| "errors": 0, |
| "acceptance_length": 3.8383407546285446, |
| "acceptance_rate_pct": 35.47925943285681, |
| "position_1_pct": 72.8521209280525, |
| "position_2_pct": 54.408249355519104, |
| "position_3_pct": 42.06233887977502, |
| "position_4_pct": 33.016170611670965, |
| "position_5_pct": 26.833841106163582, |
| "position_6_pct": 21.837356456526834, |
| "position_7_pct": 17.956409655495666, |
| "position_8_pct": 14.867588469650808 |
| }, |
| { |
| "experiment": "SPEED-Bench T1", |
| "dataset": "reasoning", |
| "successful_calls": 189, |
| "errors": 1, |
| "acceptance_length": 2.8580765924236786, |
| "acceptance_rate_pct": 23.225957405295986, |
| "position_1_pct": 63.283560249007685, |
| "position_2_pct": 40.626543341605135, |
| "position_3_pct": 26.90917072635372, |
| "position_4_pct": 18.538390717695766, |
| "position_5_pct": 13.239965612062605, |
| "position_6_pct": 9.87189883605368, |
| "position_7_pct": 7.515349580211083, |
| "position_8_pct": 5.822780179378213 |
| }, |
| { |
| "experiment": "SPEED-Bench T1", |
| "dataset": "roleplay", |
| "successful_calls": 158, |
| "errors": 0, |
| "acceptance_length": 2.4504565641942078, |
| "acceptance_rate_pct": 18.130707052427596, |
| "position_1_pct": 61.07780819025645, |
| "position_2_pct": 35.241779205404875, |
| "position_3_pct": 20.091840658719207, |
| "position_4_pct": 11.655016927937506, |
| "position_5_pct": 7.141402945234917, |
| "position_6_pct": 4.555839572918316, |
| "position_7_pct": 3.0832221141448186, |
| "position_8_pct": 2.198746804804669 |
| }, |
| { |
| "experiment": "SPEED-Bench T1", |
| "dataset": "stem", |
| "successful_calls": 78, |
| "errors": 8, |
| "acceptance_length": 3.002715318869165, |
| "acceptance_rate_pct": 25.033941485864563, |
| "position_1_pct": 66.85820731974579, |
| "position_2_pct": 44.85163269778654, |
| "position_3_pct": 30.34845496383958, |
| "position_4_pct": 20.8531667762437, |
| "position_5_pct": 14.379793995178611, |
| "position_6_pct": 10.174008327854482, |
| "position_7_pct": 7.408941485864562, |
| "position_8_pct": 5.397326320403243 |
| }, |
| { |
| "experiment": "SPEED-Bench T1", |
| "dataset": "summarization", |
| "successful_calls": 80, |
| "errors": 0, |
| "acceptance_length": 4.29741136747327, |
| "acceptance_rate_pct": 41.21764209341587, |
| "position_1_pct": 81.46689176514725, |
| "position_2_pct": 64.72519227161884, |
| "position_3_pct": 51.15362971299944, |
| "position_4_pct": 40.20821609454136, |
| "position_5_pct": 31.523166385293568, |
| "position_6_pct": 24.826486587882197, |
| "position_7_pct": 19.864940911648848, |
| "position_8_pct": 15.97261301819546 |
| }, |
| { |
| "experiment": "SPEED-Bench T1", |
| "dataset": "writing", |
| "successful_calls": 80, |
| "errors": 4, |
| "acceptance_length": 2.772608882265594, |
| "acceptance_rate_pct": 22.15761102831992, |
| "position_1_pct": 64.7225991517087, |
| "position_2_pct": 41.07388025283453, |
| "position_3_pct": 26.214386828291662, |
| "position_4_pct": 16.991685039202146, |
| "position_5_pct": 11.295443700064249, |
| "position_6_pct": 7.687406526671056, |
| "position_7_pct": 5.4181873438778805, |
| "position_8_pct": 3.8572993839091247 |
| }, |
| { |
| "experiment": "MRCR T0", |
| "dataset": "8 needles", |
| "successful_calls": 458, |
| "errors": 342, |
| "acceptance_length": 6.207113290053785, |
| "acceptance_rate_pct": 65.08891612567231, |
| "position_1_pct": 92.19037613698606, |
| "position_2_pct": 84.60747604809238, |
| "position_3_pct": 75.13221812173944, |
| "position_4_pct": 65.37950417081953, |
| "position_5_pct": 57.71399581571173, |
| "position_6_pct": 52.20124091982509, |
| "position_7_pct": 48.23020768795625, |
| "position_8_pct": 45.25631010424804 |
| } |
| ] |
|
|
