Upload evaluation results: comparison

evaluation/comparison_results.json

@@ -1,339 +1,99 @@
 {
-  "h_curated": {
+  "h_extended": {
     "with_idf": {
-      "recall@1": 0.8078758716583252,
-      "recall@5": 0.9498806595802307,
-      "recall@10": 0.964200496673584,
-      "recall@20": 0.9761336445808411,
-      "mrr": 0.871401846408844,
-      "ndcg@10": 0.8934720082918809
+      "recall@1": 0.5011933445930481,
+      "recall@5": 0.715990424156189,
+      "recall@10": 0.7720763683319092,
+      "recall@20": 0.8532219529151917,
+      "mrr": 0.5991758108139038,
+      "ndcg@10": 0.6341397571131019
     },
     "no_idf": {
-      "recall@1": 0.560859203338623,
-      "recall@5": 0.7088305354118347,
-      "recall@10": 0.7482100129127502,
-      "recall@20": 0.7720763683319092,
-      "mrr": 0.6307040452957153,
-      "ndcg@10": 0.6560224477581484
-    },
-    "hybrid_no_idf": {
-      "best_alpha": 0.5,
-      "best_metrics": {
-        "recall@1": 0.577565610408783,
-        "recall@5": 0.6909307837486267,
-        "recall@10": 0.7338902354240417,
-        "recall@20": 0.7780429720878601,
-        "mrr": 0.6334040760993958,
-        "ndcg@10": 0.6522560542504828
-      }
-    },
-    "hybrid_idf": {
-      "best_alpha": 0.5,
-      "best_metrics": {
-        "recall@1": 0.7995226979255676,
-        "recall@5": 0.9439140558242798,
-        "recall@10": 0.9773269891738892,
-        "recall@20": 0.9952267408370972,
-        "mrr": 0.8643569350242615,
-        "ndcg@10": 0.8908862503996242
-      }
-    },
-    "bootstrap_ci": {
-      "hybrid_idf_r10": {
-        "mean": 0.9773269891738892,
-        "ci_lower": 0.9665871262550354,
-        "ci_upper": 0.9868735074996948,
-        "se": 0.005134093975245222
-      },
-      "hybrid_idf_ndcg10": {
-        "mean": 0.8908862503996242,
-        "ci_lower": 0.8750708682822845,
-        "ci_upper": 0.9066549773221649,
-        "se": 0.008092329911784334
-      },
-      "standalone_idf_r10": {
-        "mean": 0.964200496673584,
-        "ci_lower": 0.9510740041732788,
-        "ci_upper": 0.9761336445808411,
-        "se": 0.006408560281645167
-      }
-    }
-  },
-  "h_curated_s2": {
-    "with_idf": {
-      "recall@1": 0.79474937915802,
-      "recall@5": 0.9510740041732788,
-      "recall@10": 0.9725536704063416,
-      "recall@20": 0.9832935333251953,
-      "mrr": 0.8646395206451416,
-      "ndcg@10": 0.8905745725645697
-    },
-    "no_idf": {
-      "recall@1": 0.4618138372898102,
-      "recall@5": 0.6002386808395386,
-      "recall@10": 0.638424813747406,
-      "recall@20": 0.6587111949920654,
-      "mrr": 0.5274040699005127,
-      "ndcg@10": 0.5508962789654486
-    },
-    "hybrid_no_idf": {
-      "best_alpha": 0.5,
-      "best_metrics": {
-        "recall@1": 0.5441527366638184,
-        "recall@5": 0.6527446508407593,
-        "recall@10": 0.7016706466674805,
-        "recall@20": 0.7446300983428955,
-        "mrr": 0.5978065729141235,
-        "ndcg@10": 0.6173909723302361
-      }
-    },
-    "hybrid_idf": {
-      "best_alpha": 0.5,
-      "best_metrics": {
-        "recall@1": 0.8031026124954224,
-        "recall@5": 0.9594271779060364,
-        "recall@10": 0.9832935333251953,
-        "recall@20": 0.9940333962440491,
-        "mrr": 0.8701260685920715,
-        "ndcg@10": 0.8971763733153765
-      }
-    },
-    "bootstrap_ci": {
-      "hybrid_idf_r10": {
-        "mean": 0.9832935333251953,
-        "ci_lower": 0.9737470149993896,
-        "ci_upper": 0.9916467666625977,
-        "se": 0.004433608641789609
-      },
-      "hybrid_idf_ndcg10": {
-        "mean": 0.8971763733153765,
-        "ci_lower": 0.8821949014944158,
-        "ci_upper": 0.9123868220432041,
-        "se": 0.007695017042453908
-      },
-      "standalone_idf_r10": {
-        "mean": 0.9725536704063416,
-        "ci_lower": 0.9606205224990845,
-        "ci_upper": 0.9832935333251953,
-        "se": 0.005762715650875535
-      }
-    }
-  },
-  "h_rand7552": {
-    "with_idf": {
-      "recall@1": 0.45704057812690735,
-      "recall@5": 0.6360381841659546,
-      "recall@10": 0.7315036058425903,
-      "recall@20": 0.819809079170227,
-      "mrr": 0.54668128490448,
-      "ndcg@10": 0.5825007922082269
-    },
-    "no_idf": {
-      "recall@1": 0.2887828052043915,
-      "recall@5": 0.42124104499816895,
-      "recall@10": 0.4630071520805359,
-      "recall@20": 0.4976133704185486,
-      "mrr": 0.35064178705215454,
-      "ndcg@10": 0.3728723791192576
+      "recall@1": 0.3663484454154968,
+      "recall@5": 0.49164676666259766,
+      "recall@10": 0.5453460812568665,
+      "recall@20": 0.5680190920829773,
+      "mrr": 0.4280979037284851,
+      "ndcg@10": 0.45219292759059226
     },
     "hybrid_no_idf": {
       "best_alpha": 0.3,
       "best_metrics": {
-        "recall@1": 0.5107398629188538,
-        "recall@5": 0.6396181583404541,
-        "recall@10": 0.6897374987602234,
-        "recall@20": 0.7279236316680908,
-        "mrr": 0.5710219144821167,
-        "ndcg@10": 0.594996674033728
+        "recall@1": 0.5155131220817566,
+        "recall@5": 0.6455847024917603,
+        "recall@10": 0.6921241283416748,
+        "recall@20": 0.7315036058425903,
+        "mrr": 0.5743579864501953,
+        "ndcg@10": 0.5978281095200982
       }
     },
     "hybrid_idf": {
       "best_alpha": 0.5,
       "best_metrics": {
-        "recall@1": 0.6229116916656494,
-        "recall@5": 0.8042959570884705,
-        "recall@10": 0.861575186252594,
-        "recall@20": 0.9021479487419128,
-        "mrr": 0.7051689624786377,
-        "ndcg@10": 0.739109985922535
-      }
-    },
-    "bootstrap_ci": {
-      "hybrid_idf_r10": {
-        "mean": 0.861575186252594,
-        "ci_lower": 0.8377088308334351,
-        "ci_upper": 0.8842481970787048,
-        "se": 0.011970612565884334
-      },
-      "hybrid_idf_ndcg10": {
-        "mean": 0.739109985922535,
-        "ci_lower": 0.7137405788253148,
-        "ci_upper": 0.7641767890478637,
-        "se": 0.012806377598930505
-      },
-      "standalone_idf_r10": {
-        "mean": 0.7315036058425903,
-        "ci_lower": 0.7016706466674805,
-        "ci_upper": 0.7613365054130554,
-        "se": 0.01526008860083923
-      }
-    }
-  },
-  "h_notier4": {
-    "with_idf": {
-      "recall@1": 0.45942720770835876,
-      "recall@5": 0.6694510579109192,
-      "recall@10": 0.7613365054130554,
-      "recall@20": 0.843675434589386,
-      "mrr": 0.5589096546173096,
-      "ndcg@10": 0.6001309074358909
-    },
-    "no_idf": {
-      "recall@1": 0.2732696831226349,
-      "recall@5": 0.3806682527065277,
-      "recall@10": 0.42124104499816895,
-      "recall@20": 0.4510739743709564,
-      "mrr": 0.3235103487968445,
-      "ndcg@10": 0.34176754966492734
-    },
-    "hybrid_no_idf": {
-      "best_alpha": 0.2,
-      "best_metrics": {
-        "recall@1": 0.5107398629188538,
-        "recall@5": 0.6348448395729065,
-        "recall@10": 0.6885441541671753,
-        "recall@20": 0.7279236316680908,
-        "mrr": 0.5687209367752075,
-        "ndcg@10": 0.5927738891990314
-      }
-    },
-    "hybrid_idf": {
-      "best_alpha": 0.5,
-      "best_metrics": {
-        "recall@1": 0.6420047879219055,
+        "recall@1": 0.6694510579109192,
         "recall@5": 0.836515486240387,
-        "recall@10": 0.8866348266601562,
-        "recall@20": 0.9248210191726685,
-        "mrr": 0.7279645800590515,
-        "ndcg@10": 0.7632779478919515
-      }
-    },
-    "bootstrap_ci": {
-      "hybrid_idf_r10": {
-        "mean": 0.8866348266601562,
-        "ci_lower": 0.8651551604270935,
-        "ci_upper": 0.9069212675094604,
-        "se": 0.010968634185269548
-      },
-      "hybrid_idf_ndcg10": {
-        "mean": 0.7632779478919515,
-        "ci_lower": 0.7392506147251328,
-        "ci_upper": 0.7868044559342753,
-        "se": 0.0121621765042722
-      },
-      "standalone_idf_r10": {
-        "mean": 0.7613365054130554,
-        "ci_lower": 0.7326968908309937,
-        "ci_upper": 0.7911694645881653,
-        "se": 0.014857916253111171
-      }
-    }
-  },
-  "v12_e15": {
-    "with_idf": {
-      "recall@1": 0.8603818416595459,
-      "recall@5": 0.9546539187431335,
-      "recall@10": 0.9701670408248901,
-      "recall@20": 0.9856801629066467,
-      "mrr": 0.9037026762962341,
-      "ndcg@10": 0.9190719076440962
-    },
-    "no_idf": {
-      "recall@1": 0.5477327108383179,
-      "recall@5": 0.70525062084198,
-      "recall@10": 0.7315036058425903,
-      "recall@20": 0.7541766166687012,
-      "mrr": 0.616544246673584,
-      "ndcg@10": 0.6413623750146198
-    },
-    "hybrid_no_idf": {
-      "best_alpha": 0.5,
-      "best_metrics": {
-        "recall@1": 0.5465393662452698,
-        "recall@5": 0.6873508095741272,
-        "recall@10": 0.7315036058425903,
-        "recall@20": 0.7684963941574097,
-        "mrr": 0.6118062138557434,
-        "ndcg@10": 0.6360296473664324
-      }
-    },
-    "hybrid_idf": {
-      "best_alpha": 0.5,
-      "best_metrics": {
-        "recall@1": 0.7959427237510681,
-        "recall@5": 0.9546539187431335,
-        "recall@10": 0.9844868779182434,
-        "recall@20": 0.9940333962440491,
-        "mrr": 0.8643589615821838,
-        "ndcg@10": 0.8930407819276659
+        "recall@10": 0.8818615674972534,
+        "recall@20": 0.9081145524978638,
+        "mrr": 0.7436834573745728,
+        "ndcg@10": 0.7742396474131288
       }
     },
     "bootstrap_ci": {
       "hybrid_idf_r10": {
-        "mean": 0.9844868779182434,
-        "ci_lower": 0.9761038124561312,
-        "ci_upper": 0.9916467666625977,
-        "se": 0.004230828890499356
+        "mean": 0.8818615674972534,
+        "ci_lower": 0.8603818416595459,
+        "ci_upper": 0.9033412933349609,
+        "se": 0.011077928874225752
       },
       "hybrid_idf_ndcg10": {
-        "mean": 0.8930407819276659,
-        "ci_lower": 0.8778117702491244,
-        "ci_upper": 0.9081692573752533,
-        "se": 0.00784255541184393
+        "mean": 0.7742396474131288,
+        "ci_lower": 0.7506954504355015,
+        "ci_upper": 0.7977965140781702,
+        "se": 0.012106710016022492
       },
       "standalone_idf_r10": {
-        "mean": 0.9701670408248901,
-        "ci_lower": 0.9582338929176331,
-        "ci_upper": 0.9809069037437439,
-        "se": 0.005905478481174235
+        "mean": 0.7720763683319092,
+        "ci_lower": 0.7434367537498474,
+        "ci_upper": 0.800715982913971,
+        "se": 0.014512963508524598
       }
     }
   },
   "_significance_tests": {
     "recall@10": {
-      "delta": 0.2959427237510681,
+      "delta": 0.19331742823123932,
       "p_value": 0.0,
       "significant": true,
-      "ci_lower": 0.26491647958755493,
-      "ci_upper": 0.3269689679145813
+      "ci_lower": 0.16348448395729065,
+      "ci_upper": 0.2219570428133011
     },
     "ndcg@10": {
-      "delta": 0.3019232174052843,
+      "delta": 0.1831220828907474,
       "p_value": 0.0,
       "significant": true,
-      "ci_lower": 0.27495923197187516,
-      "ci_upper": 0.3285048309635248
+      "ci_lower": 0.1592187369075159,
+      "ci_upper": 0.20782378544404753
     },
     "mrr": {
-      "delta": 0.29798927903175354,
+      "delta": 0.17731375992298126,
       "p_value": 0.0,
       "significant": true,
-      "ci_lower": 0.2706925220787525,
-      "ci_upper": 0.32507591545581815
+      "ci_lower": 0.15270963460206985,
+      "ci_upper": 0.20239105299115182
     },
     "standalone_recall@10": {
-      "delta": 0.28162291646003723,
+      "delta": 0.08353222161531448,
       "p_value": 0.0,
       "significant": true,
-      "ci_lower": 0.24821002781391144,
-      "ci_upper": 0.31503579020500183
+      "ci_lower": 0.044152744114398956,
+      "ci_upper": 0.12291169166564941
     },
     "standalone_ndcg@10": {
-      "delta": 0.3279543431217147,
-      "p_value": 0.0,
+      "delta": 0.04302219259072037,
+      "p_value": 0.0066,
       "significant": true,
-      "ci_lower": 0.295606120573298,
-      "ci_upper": 0.3597483916727818
+      "ci_lower": 0.009433026997052838,
+      "ci_upper": 0.07666100482019714
     }
   },
   "_bm25": {