| --- |
| license: apache-2.0 |
| language: |
| - en |
| library_name: sentence-transformers |
| pipeline_tag: sentence-similarity |
| base_model: dmis-lab/UltraBERT-150M |
| tags: |
| - sentence-transformers |
| - sentence-similarity |
| - feature-extraction |
| - mteb |
| - ultrabert |
| - long-context |
| datasets: |
| - lightonai/embeddings-pre-training-curated |
| - lightonai/cornstack |
| - lightonai/embeddings-fine-tuning-filtered-en |
| - codefuse-ai/F2LLM |
| - codefuse-ai/F2LLM-v2 |
| - nomic-ai/nomic-embed-supervised-data |
| - jspringer/open-synthetic-embeddings |
| - nvidia/embed-nemotron-dataset-v1 |
| - thesofakillers/jigsaw-toxic-comment-classification-challenge |
| - ucberkeley-dlab/measuring-hate-speech |
| model-index: |
| - name: UltraBERT-Embed-150M |
| results: |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/ARCChallenge |
| name: MTEB ARCChallenge |
| config: default |
| split: test |
| revision: 61b42fe57d9a44e30f47b9b878b664a95472ec80 |
| metrics: |
| - type: ndcg_at_1 |
| value: 6.655 |
| - type: ndcg_at_3 |
| value: 11.92 |
| - type: ndcg_at_5 |
| value: 13.881 |
| - type: ndcg_at_10 |
| value: 16.149 |
| - type: ndcg_at_20 |
| value: 18.048 |
| - type: ndcg_at_100 |
| value: 21.94 |
| - type: ndcg_at_1000 |
| value: 25.838 |
| - type: map_at_1 |
| value: 6.655 |
| - type: map_at_3 |
| value: 10.538 |
| - type: map_at_5 |
| value: 11.621 |
| - type: map_at_10 |
| value: 12.545 |
| - type: map_at_20 |
| value: 13.067 |
| - type: map_at_100 |
| value: 13.569 |
| - type: map_at_1000 |
| value: 13.699 |
| - type: recall_at_1 |
| value: 6.655 |
| - type: recall_at_3 |
| value: 15.956 |
| - type: recall_at_5 |
| value: 20.734 |
| - type: recall_at_10 |
| value: 27.816 |
| - type: recall_at_20 |
| value: 35.324 |
| - type: recall_at_100 |
| value: 56.826 |
| - type: recall_at_1000 |
| value: 88.481 |
| - type: accuracy |
| value: 6.655 |
| - type: precision_at_1 |
| value: 6.655 |
| - type: precision_at_3 |
| value: 5.319 |
| - type: precision_at_5 |
| value: 4.147 |
| - type: precision_at_10 |
| value: 2.782 |
| - type: precision_at_20 |
| value: 1.766 |
| - type: precision_at_100 |
| value: 0.568 |
| - type: precision_at_1000 |
| value: 0.088 |
| - type: mrr_at_1 |
| value: 6.6553 |
| - type: mrr_at_3 |
| value: 10.5375 |
| - type: mrr_at_5 |
| value: 11.6212 |
| - type: mrr_at_10 |
| value: 12.5453 |
| - type: mrr_at_20 |
| value: 13.0665 |
| - type: mrr_at_100 |
| value: 13.5688 |
| - type: mrr_at_1000 |
| value: 13.6991 |
| - type: hit_rate_at_1 |
| value: 6.655 |
| - type: hit_rate_at_3 |
| value: 15.956 |
| - type: hit_rate_at_5 |
| value: 20.734 |
| - type: hit_rate_at_10 |
| value: 27.816 |
| - type: hit_rate_at_20 |
| value: 35.324 |
| - type: hit_rate_at_100 |
| value: 56.826 |
| - type: hit_rate_at_1000 |
| value: 88.481 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/AlphaNLI |
| name: MTEB AlphaNLI |
| config: default |
| split: test |
| revision: f145e68f0f5097f5d160c345eabeb2db8745eb13 |
| metrics: |
| - type: ndcg_at_1 |
| value: 23.433 |
| - type: ndcg_at_3 |
| value: 31.969 |
| - type: ndcg_at_5 |
| value: 33.933 |
| - type: ndcg_at_10 |
| value: 36.088 |
| - type: ndcg_at_20 |
| value: 37.626 |
| - type: ndcg_at_100 |
| value: 39.847 |
| - type: ndcg_at_1000 |
| value: 41.75 |
| - type: map_at_1 |
| value: 23.433 |
| - type: map_at_3 |
| value: 29.906 |
| - type: map_at_5 |
| value: 30.997 |
| - type: map_at_10 |
| value: 31.9 |
| - type: map_at_20 |
| value: 32.323 |
| - type: map_at_100 |
| value: 32.624 |
| - type: map_at_1000 |
| value: 32.685 |
| - type: recall_at_1 |
| value: 23.433 |
| - type: recall_at_3 |
| value: 37.924 |
| - type: recall_at_5 |
| value: 42.689 |
| - type: recall_at_10 |
| value: 49.282 |
| - type: recall_at_20 |
| value: 55.352 |
| - type: recall_at_100 |
| value: 67.428 |
| - type: recall_at_1000 |
| value: 83.029 |
| - type: accuracy |
| value: 23.433 |
| - type: precision_at_1 |
| value: 23.433 |
| - type: precision_at_3 |
| value: 12.641 |
| - type: precision_at_5 |
| value: 8.538 |
| - type: precision_at_10 |
| value: 4.928 |
| - type: precision_at_20 |
| value: 2.768 |
| - type: precision_at_100 |
| value: 0.674 |
| - type: precision_at_1000 |
| value: 0.083 |
| - type: mrr_at_1 |
| value: 23.4334 |
| - type: mrr_at_3 |
| value: 29.9064 |
| - type: mrr_at_5 |
| value: 30.9965 |
| - type: mrr_at_10 |
| value: 31.9001 |
| - type: mrr_at_20 |
| value: 32.3233 |
| - type: mrr_at_100 |
| value: 32.624 |
| - type: mrr_at_1000 |
| value: 32.6845 |
| - type: hit_rate_at_1 |
| value: 23.433 |
| - type: hit_rate_at_3 |
| value: 37.924 |
| - type: hit_rate_at_5 |
| value: 42.689 |
| - type: hit_rate_at_10 |
| value: 49.282 |
| - type: hit_rate_at_20 |
| value: 55.352 |
| - type: hit_rate_at_100 |
| value: 67.428 |
| - type: hit_rate_at_1000 |
| value: 83.029 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/amazon_counterfactual |
| name: MTEB AmazonCounterfactualClassification (en) |
| config: en |
| split: test |
| revision: 1f7e6a9d6fa6e64c53d146e428565640410c0df1 |
| metrics: |
| - type: accuracy |
| value: 84.7761 |
| - type: f1 |
| value: 79.5912 |
| - type: f1_weighted |
| value: 85.8428 |
| - type: precision |
| value: 77.1388 |
| - type: precision_weighted |
| value: 88.8828 |
| - type: recall |
| value: 85.6554 |
| - type: recall_weighted |
| value: 84.7761 |
| - type: ap |
| value: 52.8481 |
| - type: ap_weighted |
| value: 52.8481 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/apps |
| name: MTEB AppsRetrieval |
| config: default |
| split: test |
| revision: f22508f96b7a36c2415181ed8bb76f76e04ae2d5 |
| metrics: |
| - type: ndcg_at_1 |
| value: 56.175 |
| - type: ndcg_at_3 |
| value: 64.306 |
| - type: ndcg_at_5 |
| value: 66.307 |
| - type: ndcg_at_10 |
| value: 68.496 |
| - type: ndcg_at_20 |
| value: 69.739 |
| - type: ndcg_at_100 |
| value: 71.376 |
| - type: ndcg_at_1000 |
| value: 71.94 |
| - type: map_at_1 |
| value: 56.175 |
| - type: map_at_3 |
| value: 62.346 |
| - type: map_at_5 |
| value: 63.45 |
| - type: map_at_10 |
| value: 64.378 |
| - type: map_at_20 |
| value: 64.72 |
| - type: map_at_100 |
| value: 64.95 |
| - type: map_at_1000 |
| value: 64.972 |
| - type: recall_at_1 |
| value: 56.175 |
| - type: recall_at_3 |
| value: 69.96 |
| - type: recall_at_5 |
| value: 74.847 |
| - type: recall_at_10 |
| value: 81.487 |
| - type: recall_at_20 |
| value: 86.401 |
| - type: recall_at_100 |
| value: 95.166 |
| - type: recall_at_1000 |
| value: 99.548 |
| - type: accuracy |
| value: 56.175 |
| - type: precision_at_1 |
| value: 56.175 |
| - type: precision_at_3 |
| value: 23.32 |
| - type: precision_at_5 |
| value: 14.969 |
| - type: precision_at_10 |
| value: 8.149 |
| - type: precision_at_20 |
| value: 4.32 |
| - type: precision_at_100 |
| value: 0.952 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 56.1753 |
| - type: mrr_at_3 |
| value: 62.3462 |
| - type: mrr_at_5 |
| value: 63.4498 |
| - type: mrr_at_10 |
| value: 64.3782 |
| - type: mrr_at_20 |
| value: 64.7196 |
| - type: mrr_at_100 |
| value: 64.9502 |
| - type: mrr_at_1000 |
| value: 64.9721 |
| - type: hit_rate_at_1 |
| value: 56.175 |
| - type: hit_rate_at_3 |
| value: 69.96 |
| - type: hit_rate_at_5 |
| value: 74.847 |
| - type: hit_rate_at_10 |
| value: 81.487 |
| - type: hit_rate_at_20 |
| value: 86.401 |
| - type: hit_rate_at_100 |
| value: 95.166 |
| - type: hit_rate_at_1000 |
| value: 99.548 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/arxiv-clustering-p2p |
| name: MTEB ArXivHierarchicalClusteringP2P |
| config: default |
| split: test |
| revision: 0bbdb47bcbe3a90093699aefeed338a0f28a7ee8 |
| metrics: |
| - type: v_measure |
| value: 62.9501 |
| - type: v_measure_std |
| value: 3.3202 |
| - type: ami |
| value: 62.7076 |
| - type: ami_std |
| value: 3.451 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/arxiv-clustering-s2s |
| name: MTEB ArXivHierarchicalClusteringS2S |
| config: default |
| split: test |
| revision: b73bd54100e5abfa6e3a23dcafb46fe4d2438dc3 |
| metrics: |
| - type: v_measure |
| value: 60.3742 |
| - type: v_measure_std |
| value: 2.1873 |
| - type: ami |
| value: 60.1156 |
| - type: ami_std |
| value: 2.3132 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/arguana |
| name: MTEB ArguAna |
| config: default |
| split: test |
| revision: c22ab2a51041ffd869aaddef7af8d8215647e41a |
| metrics: |
| - type: ndcg_at_1 |
| value: 25.178 |
| - type: ndcg_at_3 |
| value: 39.161 |
| - type: ndcg_at_5 |
| value: 44.833 |
| - type: ndcg_at_10 |
| value: 50.431 |
| - type: ndcg_at_20 |
| value: 53.557 |
| - type: ndcg_at_100 |
| value: 54.842 |
| - type: ndcg_at_1000 |
| value: 54.935 |
| - type: map_at_1 |
| value: 25.178 |
| - type: map_at_3 |
| value: 35.669 |
| - type: map_at_5 |
| value: 38.844 |
| - type: map_at_10 |
| value: 41.146 |
| - type: map_at_20 |
| value: 42.031 |
| - type: map_at_100 |
| value: 42.24 |
| - type: map_at_1000 |
| value: 42.244 |
| - type: recall_at_1 |
| value: 25.178 |
| - type: recall_at_3 |
| value: 49.289 |
| - type: recall_at_5 |
| value: 62.945 |
| - type: recall_at_10 |
| value: 80.299 |
| - type: recall_at_20 |
| value: 92.461 |
| - type: recall_at_100 |
| value: 98.933 |
| - type: recall_at_1000 |
| value: 99.644 |
| - type: accuracy |
| value: 25.178 |
| - type: precision_at_1 |
| value: 25.178 |
| - type: precision_at_3 |
| value: 16.43 |
| - type: precision_at_5 |
| value: 12.589 |
| - type: precision_at_10 |
| value: 8.03 |
| - type: precision_at_20 |
| value: 4.623 |
| - type: precision_at_100 |
| value: 0.989 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 25.1778 |
| - type: mrr_at_3 |
| value: 35.6686 |
| - type: mrr_at_5 |
| value: 38.8442 |
| - type: mrr_at_10 |
| value: 41.1463 |
| - type: mrr_at_20 |
| value: 42.0311 |
| - type: mrr_at_100 |
| value: 42.2405 |
| - type: mrr_at_1000 |
| value: 42.2444 |
| - type: hit_rate_at_1 |
| value: 25.178 |
| - type: hit_rate_at_3 |
| value: 49.289 |
| - type: hit_rate_at_5 |
| value: 62.945 |
| - type: hit_rate_at_10 |
| value: 80.299 |
| - type: hit_rate_at_20 |
| value: 92.461 |
| - type: hit_rate_at_100 |
| value: 98.933 |
| - type: hit_rate_at_1000 |
| value: 99.644 |
| - task: |
| type: Reranking |
| dataset: |
| type: mteb/AskUbuntuDupQuestions |
| name: MTEB AskUbuntuDupQuestions |
| config: default |
| split: test |
| revision: c5691e3c48741d5f83b5cc8e630653d7a8cfc048 |
| metrics: |
| - type: ndcg_at_1 |
| value: 65.097 |
| - type: ndcg_at_3 |
| value: 62.42 |
| - type: ndcg_at_5 |
| value: 62.556 |
| - type: ndcg_at_10 |
| value: 68.073 |
| - type: ndcg_at_20 |
| value: 77.914 |
| - type: ndcg_at_100 |
| value: 77.914 |
| - type: ndcg_at_1000 |
| value: 77.914 |
| - type: map_at_1 |
| value: 17.663 |
| - type: map_at_3 |
| value: 33.406 |
| - type: map_at_5 |
| value: 41.46 |
| - type: map_at_10 |
| value: 52.825 |
| - type: map_at_20 |
| value: 62.807 |
| - type: map_at_100 |
| value: 62.807 |
| - type: map_at_1000 |
| value: 62.807 |
| - type: recall_at_1 |
| value: 17.663 |
| - type: recall_at_3 |
| value: 38.892 |
| - type: recall_at_5 |
| value: 51.841 |
| - type: recall_at_10 |
| value: 74.746 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 17.663 |
| - type: precision_at_1 |
| value: 65.097 |
| - type: precision_at_3 |
| value: 54.663 |
| - type: precision_at_5 |
| value: 47.368 |
| - type: precision_at_10 |
| value: 37.59 |
| - type: precision_at_20 |
| value: 27.355 |
| - type: precision_at_100 |
| value: 5.471 |
| - type: precision_at_1000 |
| value: 0.547 |
| - type: mrr_at_1 |
| value: 65.097 |
| - type: mrr_at_3 |
| value: 74.9307 |
| - type: mrr_at_5 |
| value: 76.1773 |
| - type: mrr_at_10 |
| value: 76.8788 |
| - type: mrr_at_20 |
| value: 77.1017 |
| - type: mrr_at_100 |
| value: 77.1017 |
| - type: mrr_at_1000 |
| value: 77.1017 |
| - type: hit_rate_at_1 |
| value: 65.097 |
| - type: hit_rate_at_3 |
| value: 86.704 |
| - type: hit_rate_at_5 |
| value: 91.967 |
| - type: hit_rate_at_10 |
| value: 96.953 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/biosses-sts |
| name: MTEB BIOSSES |
| config: default |
| split: test |
| revision: d3fb88f8f02e40887cd149695127462bbcf29b4a |
| metrics: |
| - type: pearson |
| value: 91.007 |
| - type: spearman |
| value: 89.3394 |
| - type: cosine_pearson |
| value: 91.007 |
| - type: cosine_spearman |
| value: 89.3394 |
| - type: manhattan_pearson |
| value: 87.748 |
| - type: manhattan_spearman |
| value: 87.3743 |
| - type: euclidean_pearson |
| value: 87.4415 |
| - type: euclidean_spearman |
| value: 86.908 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/banking77 |
| name: MTEB Banking77Classification |
| config: default |
| split: test |
| revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 |
| metrics: |
| - type: accuracy |
| value: 86.2305 |
| - type: f1 |
| value: 86.1753 |
| - type: f1_weighted |
| value: 86.1753 |
| - type: precision |
| value: 86.9383 |
| - type: precision_weighted |
| value: 86.9383 |
| - type: recall |
| value: 86.2305 |
| - type: recall_weighted |
| value: 86.2305 |
| - type: ap |
| value: .nan |
| - type: ap_weighted |
| value: .nan |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/biorxiv-clustering-p2p |
| name: MTEB BiorxivClusteringP2P.v2 |
| config: default |
| split: test |
| revision: f5dbc242e11dd8e24def4c4268607a49e02946dc |
| metrics: |
| - type: v_measure |
| value: 44.7978 |
| - type: v_measure_std |
| value: 1.1772 |
| - type: ami |
| value: 44.4665 |
| - type: ami_std |
| value: 1.1836 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/CodeSearchNet |
| name: MTEB COIRCodeSearchNetRetrieval (python) |
| config: python |
| split: test |
| revision: 4adc7bc41202b5c13543c9c886a25f340634dab3 |
| metrics: |
| - type: ndcg_at_1 |
| value: 90.642 |
| - type: ndcg_at_3 |
| value: 94.072 |
| - type: ndcg_at_5 |
| value: 94.512 |
| - type: ndcg_at_10 |
| value: 94.826 |
| - type: ndcg_at_20 |
| value: 94.956 |
| - type: ndcg_at_100 |
| value: 95.089 |
| - type: ndcg_at_1000 |
| value: 95.134 |
| - type: map_at_1 |
| value: 90.642 |
| - type: map_at_3 |
| value: 93.286 |
| - type: map_at_5 |
| value: 93.532 |
| - type: map_at_10 |
| value: 93.665 |
| - type: map_at_20 |
| value: 93.702 |
| - type: map_at_100 |
| value: 93.721 |
| - type: map_at_1000 |
| value: 93.723 |
| - type: recall_at_1 |
| value: 90.642 |
| - type: recall_at_3 |
| value: 96.327 |
| - type: recall_at_5 |
| value: 97.386 |
| - type: recall_at_10 |
| value: 98.338 |
| - type: recall_at_20 |
| value: 98.84 |
| - type: recall_at_100 |
| value: 99.551 |
| - type: recall_at_1000 |
| value: 99.906 |
| - type: accuracy |
| value: 90.642 |
| - type: precision_at_1 |
| value: 90.642 |
| - type: precision_at_3 |
| value: 32.109 |
| - type: precision_at_5 |
| value: 19.477 |
| - type: precision_at_10 |
| value: 9.834 |
| - type: precision_at_20 |
| value: 4.942 |
| - type: precision_at_100 |
| value: 0.996 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 90.6422 |
| - type: mrr_at_3 |
| value: 93.2855 |
| - type: mrr_at_5 |
| value: 93.5319 |
| - type: mrr_at_10 |
| value: 93.6646 |
| - type: mrr_at_20 |
| value: 93.702 |
| - type: mrr_at_100 |
| value: 93.7208 |
| - type: mrr_at_1000 |
| value: 93.7226 |
| - type: hit_rate_at_1 |
| value: 90.642 |
| - type: hit_rate_at_3 |
| value: 96.327 |
| - type: hit_rate_at_5 |
| value: 97.386 |
| - type: hit_rate_at_10 |
| value: 98.338 |
| - type: hit_rate_at_20 |
| value: 98.84 |
| - type: hit_rate_at_100 |
| value: 99.551 |
| - type: hit_rate_at_1000 |
| value: 99.906 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/CodeSearchNet |
| name: MTEB COIRCodeSearchNetRetrieval (javascript) |
| config: javascript |
| split: test |
| revision: 4adc7bc41202b5c13543c9c886a25f340634dab3 |
| metrics: |
| - type: ndcg_at_1 |
| value: 63.385 |
| - type: ndcg_at_3 |
| value: 70.542 |
| - type: ndcg_at_5 |
| value: 72.039 |
| - type: ndcg_at_10 |
| value: 73.182 |
| - type: ndcg_at_20 |
| value: 73.954 |
| - type: ndcg_at_100 |
| value: 74.914 |
| - type: ndcg_at_1000 |
| value: 75.492 |
| - type: map_at_1 |
| value: 63.385 |
| - type: map_at_3 |
| value: 68.839 |
| - type: map_at_5 |
| value: 69.673 |
| - type: map_at_10 |
| value: 70.142 |
| - type: map_at_20 |
| value: 70.355 |
| - type: map_at_100 |
| value: 70.492 |
| - type: map_at_1000 |
| value: 70.511 |
| - type: recall_at_1 |
| value: 63.385 |
| - type: recall_at_3 |
| value: 75.448 |
| - type: recall_at_5 |
| value: 79.064 |
| - type: recall_at_10 |
| value: 82.619 |
| - type: recall_at_20 |
| value: 85.658 |
| - type: recall_at_100 |
| value: 90.793 |
| - type: recall_at_1000 |
| value: 95.473 |
| - type: accuracy |
| value: 63.385 |
| - type: precision_at_1 |
| value: 63.385 |
| - type: precision_at_3 |
| value: 25.149 |
| - type: precision_at_5 |
| value: 15.813 |
| - type: precision_at_10 |
| value: 8.262 |
| - type: precision_at_20 |
| value: 4.283 |
| - type: precision_at_100 |
| value: 0.908 |
| - type: precision_at_1000 |
| value: 0.095 |
| - type: mrr_at_1 |
| value: 63.385 |
| - type: mrr_at_3 |
| value: 68.8393 |
| - type: mrr_at_5 |
| value: 69.6734 |
| - type: mrr_at_10 |
| value: 70.142 |
| - type: mrr_at_20 |
| value: 70.3555 |
| - type: mrr_at_100 |
| value: 70.4916 |
| - type: mrr_at_1000 |
| value: 70.5112 |
| - type: hit_rate_at_1 |
| value: 63.385 |
| - type: hit_rate_at_3 |
| value: 75.448 |
| - type: hit_rate_at_5 |
| value: 79.064 |
| - type: hit_rate_at_10 |
| value: 82.619 |
| - type: hit_rate_at_20 |
| value: 85.658 |
| - type: hit_rate_at_100 |
| value: 90.793 |
| - type: hit_rate_at_1000 |
| value: 95.473 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/CodeSearchNet |
| name: MTEB COIRCodeSearchNetRetrieval (go) |
| config: go |
| split: test |
| revision: 4adc7bc41202b5c13543c9c886a25f340634dab3 |
| metrics: |
| - type: ndcg_at_1 |
| value: 80.079 |
| - type: ndcg_at_3 |
| value: 85.629 |
| - type: ndcg_at_5 |
| value: 86.678 |
| - type: ndcg_at_10 |
| value: 87.527 |
| - type: ndcg_at_20 |
| value: 87.932 |
| - type: ndcg_at_100 |
| value: 88.363 |
| - type: ndcg_at_1000 |
| value: 88.496 |
| - type: map_at_1 |
| value: 80.079 |
| - type: map_at_3 |
| value: 84.322 |
| - type: map_at_5 |
| value: 84.907 |
| - type: map_at_10 |
| value: 85.265 |
| - type: map_at_20 |
| value: 85.379 |
| - type: map_at_100 |
| value: 85.441 |
| - type: map_at_1000 |
| value: 85.446 |
| - type: recall_at_1 |
| value: 80.079 |
| - type: recall_at_3 |
| value: 89.387 |
| - type: recall_at_5 |
| value: 91.923 |
| - type: recall_at_10 |
| value: 94.509 |
| - type: recall_at_20 |
| value: 96.085 |
| - type: recall_at_100 |
| value: 98.375 |
| - type: recall_at_1000 |
| value: 99.421 |
| - type: accuracy |
| value: 80.079 |
| - type: precision_at_1 |
| value: 80.079 |
| - type: precision_at_3 |
| value: 29.796 |
| - type: precision_at_5 |
| value: 18.385 |
| - type: precision_at_10 |
| value: 9.451 |
| - type: precision_at_20 |
| value: 4.804 |
| - type: precision_at_100 |
| value: 0.984 |
| - type: precision_at_1000 |
| value: 0.099 |
| - type: mrr_at_1 |
| value: 80.0788 |
| - type: mrr_at_3 |
| value: 84.3224 |
| - type: mrr_at_5 |
| value: 84.9072 |
| - type: mrr_at_10 |
| value: 85.2646 |
| - type: mrr_at_20 |
| value: 85.3792 |
| - type: mrr_at_100 |
| value: 85.4413 |
| - type: mrr_at_1000 |
| value: 85.4463 |
| - type: hit_rate_at_1 |
| value: 80.079 |
| - type: hit_rate_at_3 |
| value: 89.387 |
| - type: hit_rate_at_5 |
| value: 91.923 |
| - type: hit_rate_at_10 |
| value: 94.509 |
| - type: hit_rate_at_20 |
| value: 96.085 |
| - type: hit_rate_at_100 |
| value: 98.375 |
| - type: hit_rate_at_1000 |
| value: 99.421 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/CodeSearchNet |
| name: MTEB COIRCodeSearchNetRetrieval (ruby) |
| config: ruby |
| split: test |
| revision: 4adc7bc41202b5c13543c9c886a25f340634dab3 |
| metrics: |
| - type: ndcg_at_1 |
| value: 65.504 |
| - type: ndcg_at_3 |
| value: 73.219 |
| - type: ndcg_at_5 |
| value: 75.021 |
| - type: ndcg_at_10 |
| value: 76.279 |
| - type: ndcg_at_20 |
| value: 77.132 |
| - type: ndcg_at_100 |
| value: 77.953 |
| - type: ndcg_at_1000 |
| value: 78.348 |
| - type: map_at_1 |
| value: 65.504 |
| - type: map_at_3 |
| value: 71.359 |
| - type: map_at_5 |
| value: 72.362 |
| - type: map_at_10 |
| value: 72.882 |
| - type: map_at_20 |
| value: 73.121 |
| - type: map_at_100 |
| value: 73.244 |
| - type: map_at_1000 |
| value: 73.26 |
| - type: recall_at_1 |
| value: 65.504 |
| - type: recall_at_3 |
| value: 78.588 |
| - type: recall_at_5 |
| value: 82.95 |
| - type: recall_at_10 |
| value: 86.836 |
| - type: recall_at_20 |
| value: 90.167 |
| - type: recall_at_100 |
| value: 94.449 |
| - type: recall_at_1000 |
| value: 97.542 |
| - type: accuracy |
| value: 65.504 |
| - type: precision_at_1 |
| value: 65.504 |
| - type: precision_at_3 |
| value: 26.196 |
| - type: precision_at_5 |
| value: 16.59 |
| - type: precision_at_10 |
| value: 8.684 |
| - type: precision_at_20 |
| value: 4.508 |
| - type: precision_at_100 |
| value: 0.944 |
| - type: precision_at_1000 |
| value: 0.098 |
| - type: mrr_at_1 |
| value: 65.5036 |
| - type: mrr_at_3 |
| value: 71.3587 |
| - type: mrr_at_5 |
| value: 72.3619 |
| - type: mrr_at_10 |
| value: 72.8818 |
| - type: mrr_at_20 |
| value: 73.1211 |
| - type: mrr_at_100 |
| value: 73.2444 |
| - type: mrr_at_1000 |
| value: 73.2595 |
| - type: hit_rate_at_1 |
| value: 65.504 |
| - type: hit_rate_at_3 |
| value: 78.588 |
| - type: hit_rate_at_5 |
| value: 82.95 |
| - type: hit_rate_at_10 |
| value: 86.836 |
| - type: hit_rate_at_20 |
| value: 90.167 |
| - type: hit_rate_at_100 |
| value: 94.449 |
| - type: hit_rate_at_1000 |
| value: 97.542 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/CodeSearchNet |
| name: MTEB COIRCodeSearchNetRetrieval (java) |
| config: java |
| split: test |
| revision: 4adc7bc41202b5c13543c9c886a25f340634dab3 |
| metrics: |
| - type: ndcg_at_1 |
| value: 71.584 |
| - type: ndcg_at_3 |
| value: 78.311 |
| - type: ndcg_at_5 |
| value: 79.577 |
| - type: ndcg_at_10 |
| value: 80.516 |
| - type: ndcg_at_20 |
| value: 81.023 |
| - type: ndcg_at_100 |
| value: 81.678 |
| - type: ndcg_at_1000 |
| value: 82.004 |
| - type: map_at_1 |
| value: 71.584 |
| - type: map_at_3 |
| value: 76.724 |
| - type: map_at_5 |
| value: 77.428 |
| - type: map_at_10 |
| value: 77.816 |
| - type: map_at_20 |
| value: 77.958 |
| - type: map_at_100 |
| value: 78.049 |
| - type: map_at_1000 |
| value: 78.06 |
| - type: recall_at_1 |
| value: 71.584 |
| - type: recall_at_3 |
| value: 82.875 |
| - type: recall_at_5 |
| value: 85.942 |
| - type: recall_at_10 |
| value: 88.845 |
| - type: recall_at_20 |
| value: 90.826 |
| - type: recall_at_100 |
| value: 94.35 |
| - type: recall_at_1000 |
| value: 96.988 |
| - type: accuracy |
| value: 71.584 |
| - type: precision_at_1 |
| value: 71.584 |
| - type: precision_at_3 |
| value: 27.625 |
| - type: precision_at_5 |
| value: 17.188 |
| - type: precision_at_10 |
| value: 8.885 |
| - type: precision_at_20 |
| value: 4.541 |
| - type: precision_at_100 |
| value: 0.943 |
| - type: precision_at_1000 |
| value: 0.097 |
| - type: mrr_at_1 |
| value: 71.5838 |
| - type: mrr_at_3 |
| value: 76.7245 |
| - type: mrr_at_5 |
| value: 77.4278 |
| - type: mrr_at_10 |
| value: 77.8159 |
| - type: mrr_at_20 |
| value: 77.9584 |
| - type: mrr_at_100 |
| value: 78.0491 |
| - type: mrr_at_1000 |
| value: 78.0603 |
| - type: hit_rate_at_1 |
| value: 71.584 |
| - type: hit_rate_at_3 |
| value: 82.875 |
| - type: hit_rate_at_5 |
| value: 85.942 |
| - type: hit_rate_at_10 |
| value: 88.845 |
| - type: hit_rate_at_20 |
| value: 90.826 |
| - type: hit_rate_at_100 |
| value: 94.35 |
| - type: hit_rate_at_1000 |
| value: 96.988 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/CodeSearchNet |
| name: MTEB COIRCodeSearchNetRetrieval (php) |
| config: php |
| split: test |
| revision: 4adc7bc41202b5c13543c9c886a25f340634dab3 |
| metrics: |
| - type: ndcg_at_1 |
| value: 71.179 |
| - type: ndcg_at_3 |
| value: 78.52 |
| - type: ndcg_at_5 |
| value: 79.899 |
| - type: ndcg_at_10 |
| value: 81.067 |
| - type: ndcg_at_20 |
| value: 81.669 |
| - type: ndcg_at_100 |
| value: 82.4 |
| - type: ndcg_at_1000 |
| value: 82.697 |
| - type: map_at_1 |
| value: 71.179 |
| - type: map_at_3 |
| value: 76.771 |
| - type: map_at_5 |
| value: 77.538 |
| - type: map_at_10 |
| value: 78.025 |
| - type: map_at_20 |
| value: 78.191 |
| - type: map_at_100 |
| value: 78.296 |
| - type: map_at_1000 |
| value: 78.307 |
| - type: recall_at_1 |
| value: 71.179 |
| - type: recall_at_3 |
| value: 83.559 |
| - type: recall_at_5 |
| value: 86.899 |
| - type: recall_at_10 |
| value: 90.488 |
| - type: recall_at_20 |
| value: 92.857 |
| - type: recall_at_100 |
| value: 96.753 |
| - type: recall_at_1000 |
| value: 99.087 |
| - type: accuracy |
| value: 71.179 |
| - type: precision_at_1 |
| value: 71.179 |
| - type: precision_at_3 |
| value: 27.853 |
| - type: precision_at_5 |
| value: 17.38 |
| - type: precision_at_10 |
| value: 9.049 |
| - type: precision_at_20 |
| value: 4.643 |
| - type: precision_at_100 |
| value: 0.968 |
| - type: precision_at_1000 |
| value: 0.099 |
| - type: mrr_at_1 |
| value: 71.1788 |
| - type: mrr_at_3 |
| value: 76.7708 |
| - type: mrr_at_5 |
| value: 77.5379 |
| - type: mrr_at_10 |
| value: 78.0247 |
| - type: mrr_at_20 |
| value: 78.1911 |
| - type: mrr_at_100 |
| value: 78.2957 |
| - type: mrr_at_1000 |
| value: 78.3069 |
| - type: hit_rate_at_1 |
| value: 71.179 |
| - type: hit_rate_at_3 |
| value: 83.559 |
| - type: hit_rate_at_5 |
| value: 86.899 |
| - type: hit_rate_at_10 |
| value: 90.488 |
| - type: hit_rate_at_20 |
| value: 92.857 |
| - type: hit_rate_at_100 |
| value: 96.753 |
| - type: hit_rate_at_1000 |
| value: 99.087 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/cqadupstack-gaming |
| name: MTEB CQADupstackGamingRetrieval |
| config: default |
| split: test |
| revision: 4885aa143210c98657558c04aaf3dc47cfb54340 |
| metrics: |
| - type: ndcg_at_1 |
| value: 50.533 |
| - type: ndcg_at_3 |
| value: 58.274 |
| - type: ndcg_at_5 |
| value: 60.755 |
| - type: ndcg_at_10 |
| value: 63.417 |
| - type: ndcg_at_20 |
| value: 65.246 |
| - type: ndcg_at_100 |
| value: 67.148 |
| - type: ndcg_at_1000 |
| value: 67.875 |
| - type: map_at_1 |
| value: 44.382 |
| - type: map_at_3 |
| value: 54.382 |
| - type: map_at_5 |
| value: 56.124 |
| - type: map_at_10 |
| value: 57.577 |
| - type: map_at_20 |
| value: 58.254 |
| - type: map_at_100 |
| value: 58.606 |
| - type: map_at_1000 |
| value: 58.646 |
| - type: recall_at_1 |
| value: 44.382 |
| - type: recall_at_3 |
| value: 63.397 |
| - type: recall_at_5 |
| value: 69.54 |
| - type: recall_at_10 |
| value: 77.053 |
| - type: recall_at_20 |
| value: 83.769 |
| - type: recall_at_100 |
| value: 92.838 |
| - type: recall_at_1000 |
| value: 97.865 |
| - type: accuracy |
| value: 44.382 |
| - type: precision_at_1 |
| value: 50.533 |
| - type: precision_at_3 |
| value: 25.893 |
| - type: precision_at_5 |
| value: 17.492 |
| - type: precision_at_10 |
| value: 10.069 |
| - type: precision_at_20 |
| value: 5.608 |
| - type: precision_at_100 |
| value: 1.283 |
| - type: precision_at_1000 |
| value: 0.138 |
| - type: mrr_at_1 |
| value: 50.5329 |
| - type: mrr_at_3 |
| value: 58.6311 |
| - type: mrr_at_5 |
| value: 60.0136 |
| - type: mrr_at_10 |
| value: 60.9248 |
| - type: mrr_at_20 |
| value: 61.3268 |
| - type: mrr_at_100 |
| value: 61.5205 |
| - type: mrr_at_1000 |
| value: 61.5402 |
| - type: hit_rate_at_1 |
| value: 50.533 |
| - type: hit_rate_at_3 |
| value: 68.84 |
| - type: hit_rate_at_5 |
| value: 74.922 |
| - type: hit_rate_at_10 |
| value: 81.567 |
| - type: hit_rate_at_20 |
| value: 87.21 |
| - type: hit_rate_at_100 |
| value: 94.545 |
| - type: hit_rate_at_1000 |
| value: 98.433 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/cqadupstack-unix |
| name: MTEB CQADupstackUnixRetrieval |
| config: default |
| split: test |
| revision: 6c6430d3a6d36f8d2a829195bc5dc94d7e063e53 |
| metrics: |
| - type: ndcg_at_1 |
| value: 35.354 |
| - type: ndcg_at_3 |
| value: 39.915 |
| - type: ndcg_at_5 |
| value: 42.655 |
| - type: ndcg_at_10 |
| value: 45.653 |
| - type: ndcg_at_20 |
| value: 47.584 |
| - type: ndcg_at_100 |
| value: 50.676 |
| - type: ndcg_at_1000 |
| value: 52.698 |
| - type: map_at_1 |
| value: 30.407 |
| - type: map_at_3 |
| value: 36.743 |
| - type: map_at_5 |
| value: 38.58 |
| - type: map_at_10 |
| value: 39.969 |
| - type: map_at_20 |
| value: 40.575 |
| - type: map_at_100 |
| value: 41.104 |
| - type: map_at_1000 |
| value: 41.195 |
| - type: recall_at_1 |
| value: 30.407 |
| - type: recall_at_3 |
| value: 43.145 |
| - type: recall_at_5 |
| value: 49.958 |
| - type: recall_at_10 |
| value: 58.666 |
| - type: recall_at_20 |
| value: 65.66 |
| - type: recall_at_100 |
| value: 80.446 |
| - type: recall_at_1000 |
| value: 94.486 |
| - type: accuracy |
| value: 30.407 |
| - type: precision_at_1 |
| value: 35.354 |
| - type: precision_at_3 |
| value: 17.973 |
| - type: precision_at_5 |
| value: 12.761 |
| - type: precision_at_10 |
| value: 7.761 |
| - type: precision_at_20 |
| value: 4.436 |
| - type: precision_at_100 |
| value: 1.145 |
| - type: precision_at_1000 |
| value: 0.143 |
| - type: mrr_at_1 |
| value: 35.3545 |
| - type: mrr_at_3 |
| value: 41.3557 |
| - type: mrr_at_5 |
| value: 42.9649 |
| - type: mrr_at_10 |
| value: 44.1532 |
| - type: mrr_at_20 |
| value: 44.6317 |
| - type: mrr_at_100 |
| value: 44.9615 |
| - type: mrr_at_1000 |
| value: 45.0061 |
| - type: hit_rate_at_1 |
| value: 35.354 |
| - type: hit_rate_at_3 |
| value: 49.16 |
| - type: hit_rate_at_5 |
| value: 56.157 |
| - type: hit_rate_at_10 |
| value: 65.205 |
| - type: hit_rate_at_20 |
| value: 72.201 |
| - type: hit_rate_at_100 |
| value: 85.261 |
| - type: hit_rate_at_1000 |
| value: 96.549 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/ClimateFEVER_test_top_250_only_w_correct-v2 |
| name: MTEB ClimateFEVERHardNegatives |
| config: default |
| split: test |
| revision: 3a309e201f3c2c4b13bd4a367a8f37eee2ec1d21 |
| metrics: |
| - type: ndcg_at_1 |
| value: 30.9 |
| - type: ndcg_at_3 |
| value: 27.661 |
| - type: ndcg_at_5 |
| value: 30.043 |
| - type: ndcg_at_10 |
| value: 33.752 |
| - type: ndcg_at_20 |
| value: 37.035 |
| - type: ndcg_at_100 |
| value: 41.692 |
| - type: ndcg_at_1000 |
| value: 44.895 |
| - type: map_at_1 |
| value: 14.223 |
| - type: map_at_3 |
| value: 20.529 |
| - type: map_at_5 |
| value: 22.707 |
| - type: map_at_10 |
| value: 24.684 |
| - type: map_at_20 |
| value: 25.907 |
| - type: map_at_100 |
| value: 26.855 |
| - type: map_at_1000 |
| value: 27.054 |
| - type: recall_at_1 |
| value: 14.223 |
| - type: recall_at_3 |
| value: 25.793 |
| - type: recall_at_5 |
| value: 32.022 |
| - type: recall_at_10 |
| value: 40.008 |
| - type: recall_at_20 |
| value: 49.307 |
| - type: recall_at_100 |
| value: 67.038 |
| - type: recall_at_1000 |
| value: 84.555 |
| - type: accuracy |
| value: 14.223 |
| - type: precision_at_1 |
| value: 30.9 |
| - type: precision_at_3 |
| value: 20.567 |
| - type: precision_at_5 |
| value: 16.0 |
| - type: precision_at_10 |
| value: 10.49 |
| - type: precision_at_20 |
| value: 6.645 |
| - type: precision_at_100 |
| value: 1.896 |
| - type: precision_at_1000 |
| value: 0.25 |
| - type: mrr_at_1 |
| value: 30.9 |
| - type: mrr_at_3 |
| value: 40.3167 |
| - type: mrr_at_5 |
| value: 42.4767 |
| - type: mrr_at_10 |
| value: 43.682 |
| - type: mrr_at_20 |
| value: 44.2838 |
| - type: mrr_at_100 |
| value: 44.5973 |
| - type: mrr_at_1000 |
| value: 44.6214 |
| - type: hit_rate_at_1 |
| value: 30.9 |
| - type: hit_rate_at_3 |
| value: 52.4 |
| - type: hit_rate_at_5 |
| value: 61.8 |
| - type: hit_rate_at_10 |
| value: 70.7 |
| - type: hit_rate_at_20 |
| value: 79.0 |
| - type: hit_rate_at_100 |
| value: 90.8 |
| - type: hit_rate_at_1000 |
| value: 95.9 |
| - task: |
| type: Retrieval |
| dataset: |
| type: cassanof/CodeEditSearch |
| name: MTEB CodeEditSearchRetrieval (python) |
| config: python |
| split: train |
| revision: 4e51c66e0939303f6928472f13ad0848b2a3f4c0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 55.0 |
| - type: ndcg_at_3 |
| value: 61.9 |
| - type: ndcg_at_5 |
| value: 63.704 |
| - type: ndcg_at_10 |
| value: 65.002 |
| - type: ndcg_at_20 |
| value: 65.914 |
| - type: ndcg_at_100 |
| value: 67.946 |
| - type: ndcg_at_1000 |
| value: 69.228 |
| - type: map_at_1 |
| value: 55.0 |
| - type: map_at_3 |
| value: 60.267 |
| - type: map_at_5 |
| value: 61.287 |
| - type: map_at_10 |
| value: 61.826 |
| - type: map_at_20 |
| value: 62.077 |
| - type: map_at_100 |
| value: 62.341 |
| - type: map_at_1000 |
| value: 62.386 |
| - type: recall_at_1 |
| value: 55.0 |
| - type: recall_at_3 |
| value: 66.6 |
| - type: recall_at_5 |
| value: 70.9 |
| - type: recall_at_10 |
| value: 74.9 |
| - type: recall_at_20 |
| value: 78.5 |
| - type: recall_at_100 |
| value: 89.7 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 55.0 |
| - type: precision_at_1 |
| value: 55.0 |
| - type: precision_at_3 |
| value: 22.2 |
| - type: precision_at_5 |
| value: 14.18 |
| - type: precision_at_10 |
| value: 7.49 |
| - type: precision_at_20 |
| value: 3.925 |
| - type: precision_at_100 |
| value: 0.897 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 55.0 |
| - type: mrr_at_3 |
| value: 60.2667 |
| - type: mrr_at_5 |
| value: 61.2867 |
| - type: mrr_at_10 |
| value: 61.8262 |
| - type: mrr_at_20 |
| value: 62.0771 |
| - type: mrr_at_100 |
| value: 62.3412 |
| - type: mrr_at_1000 |
| value: 62.3857 |
| - type: hit_rate_at_1 |
| value: 55.0 |
| - type: hit_rate_at_3 |
| value: 66.6 |
| - type: hit_rate_at_5 |
| value: 70.9 |
| - type: hit_rate_at_10 |
| value: 74.9 |
| - type: hit_rate_at_20 |
| value: 78.5 |
| - type: hit_rate_at_100 |
| value: 89.7 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: cassanof/CodeEditSearch |
| name: MTEB CodeEditSearchRetrieval (javascript) |
| config: javascript |
| split: train |
| revision: 4e51c66e0939303f6928472f13ad0848b2a3f4c0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 52.9 |
| - type: ndcg_at_3 |
| value: 60.624 |
| - type: ndcg_at_5 |
| value: 62.34 |
| - type: ndcg_at_10 |
| value: 63.918 |
| - type: ndcg_at_20 |
| value: 65.018 |
| - type: ndcg_at_100 |
| value: 66.693 |
| - type: ndcg_at_1000 |
| value: 68.039 |
| - type: map_at_1 |
| value: 52.9 |
| - type: map_at_3 |
| value: 58.7 |
| - type: map_at_5 |
| value: 59.645 |
| - type: map_at_10 |
| value: 60.292 |
| - type: map_at_20 |
| value: 60.601 |
| - type: map_at_100 |
| value: 60.82 |
| - type: map_at_1000 |
| value: 60.86 |
| - type: recall_at_1 |
| value: 52.9 |
| - type: recall_at_3 |
| value: 66.2 |
| - type: recall_at_5 |
| value: 70.4 |
| - type: recall_at_10 |
| value: 75.3 |
| - type: recall_at_20 |
| value: 79.6 |
| - type: recall_at_100 |
| value: 88.8 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 52.9 |
| - type: precision_at_1 |
| value: 52.9 |
| - type: precision_at_3 |
| value: 22.067 |
| - type: precision_at_5 |
| value: 14.08 |
| - type: precision_at_10 |
| value: 7.53 |
| - type: precision_at_20 |
| value: 3.98 |
| - type: precision_at_100 |
| value: 0.888 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 52.9 |
| - type: mrr_at_3 |
| value: 58.7 |
| - type: mrr_at_5 |
| value: 59.645 |
| - type: mrr_at_10 |
| value: 60.292 |
| - type: mrr_at_20 |
| value: 60.6006 |
| - type: mrr_at_100 |
| value: 60.8196 |
| - type: mrr_at_1000 |
| value: 60.8604 |
| - type: hit_rate_at_1 |
| value: 52.9 |
| - type: hit_rate_at_3 |
| value: 66.2 |
| - type: hit_rate_at_5 |
| value: 70.4 |
| - type: hit_rate_at_10 |
| value: 75.3 |
| - type: hit_rate_at_20 |
| value: 79.6 |
| - type: hit_rate_at_100 |
| value: 88.8 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: cassanof/CodeEditSearch |
| name: MTEB CodeEditSearchRetrieval (typescript) |
| config: typescript |
| split: train |
| revision: 4e51c66e0939303f6928472f13ad0848b2a3f4c0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 59.2 |
| - type: ndcg_at_3 |
| value: 66.147 |
| - type: ndcg_at_5 |
| value: 67.516 |
| - type: ndcg_at_10 |
| value: 68.838 |
| - type: ndcg_at_20 |
| value: 69.851 |
| - type: ndcg_at_100 |
| value: 71.302 |
| - type: ndcg_at_1000 |
| value: 72.467 |
| - type: map_at_1 |
| value: 59.2 |
| - type: map_at_3 |
| value: 64.467 |
| - type: map_at_5 |
| value: 65.232 |
| - type: map_at_10 |
| value: 65.775 |
| - type: map_at_20 |
| value: 66.054 |
| - type: map_at_100 |
| value: 66.243 |
| - type: map_at_1000 |
| value: 66.279 |
| - type: recall_at_1 |
| value: 59.2 |
| - type: recall_at_3 |
| value: 71.0 |
| - type: recall_at_5 |
| value: 74.3 |
| - type: recall_at_10 |
| value: 78.4 |
| - type: recall_at_20 |
| value: 82.4 |
| - type: recall_at_100 |
| value: 90.4 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 59.2 |
| - type: precision_at_1 |
| value: 59.2 |
| - type: precision_at_3 |
| value: 23.667 |
| - type: precision_at_5 |
| value: 14.86 |
| - type: precision_at_10 |
| value: 7.84 |
| - type: precision_at_20 |
| value: 4.12 |
| - type: precision_at_100 |
| value: 0.904 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 59.2 |
| - type: mrr_at_3 |
| value: 64.4667 |
| - type: mrr_at_5 |
| value: 65.2317 |
| - type: mrr_at_10 |
| value: 65.7752 |
| - type: mrr_at_20 |
| value: 66.0537 |
| - type: mrr_at_100 |
| value: 66.2427 |
| - type: mrr_at_1000 |
| value: 66.2794 |
| - type: hit_rate_at_1 |
| value: 59.2 |
| - type: hit_rate_at_3 |
| value: 71.0 |
| - type: hit_rate_at_5 |
| value: 74.3 |
| - type: hit_rate_at_10 |
| value: 78.4 |
| - type: hit_rate_at_20 |
| value: 82.4 |
| - type: hit_rate_at_100 |
| value: 90.4 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: cassanof/CodeEditSearch |
| name: MTEB CodeEditSearchRetrieval (go) |
| config: go |
| split: train |
| revision: 4e51c66e0939303f6928472f13ad0848b2a3f4c0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 55.0 |
| - type: ndcg_at_3 |
| value: 61.206 |
| - type: ndcg_at_5 |
| value: 63.228 |
| - type: ndcg_at_10 |
| value: 64.808 |
| - type: ndcg_at_20 |
| value: 65.687 |
| - type: ndcg_at_100 |
| value: 67.154 |
| - type: ndcg_at_1000 |
| value: 68.735 |
| - type: map_at_1 |
| value: 55.0 |
| - type: map_at_3 |
| value: 59.717 |
| - type: map_at_5 |
| value: 60.842 |
| - type: map_at_10 |
| value: 61.491 |
| - type: map_at_20 |
| value: 61.728 |
| - type: map_at_100 |
| value: 61.917 |
| - type: map_at_1000 |
| value: 61.965 |
| - type: recall_at_1 |
| value: 55.0 |
| - type: recall_at_3 |
| value: 65.5 |
| - type: recall_at_5 |
| value: 70.4 |
| - type: recall_at_10 |
| value: 75.3 |
| - type: recall_at_20 |
| value: 78.8 |
| - type: recall_at_100 |
| value: 86.9 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 55.0 |
| - type: precision_at_1 |
| value: 55.0 |
| - type: precision_at_3 |
| value: 21.833 |
| - type: precision_at_5 |
| value: 14.08 |
| - type: precision_at_10 |
| value: 7.53 |
| - type: precision_at_20 |
| value: 3.94 |
| - type: precision_at_100 |
| value: 0.869 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 55.0 |
| - type: mrr_at_3 |
| value: 59.7167 |
| - type: mrr_at_5 |
| value: 60.8417 |
| - type: mrr_at_10 |
| value: 61.4908 |
| - type: mrr_at_20 |
| value: 61.7284 |
| - type: mrr_at_100 |
| value: 61.9172 |
| - type: mrr_at_1000 |
| value: 61.9649 |
| - type: hit_rate_at_1 |
| value: 55.0 |
| - type: hit_rate_at_3 |
| value: 65.5 |
| - type: hit_rate_at_5 |
| value: 70.4 |
| - type: hit_rate_at_10 |
| value: 75.3 |
| - type: hit_rate_at_20 |
| value: 78.8 |
| - type: hit_rate_at_100 |
| value: 86.9 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: cassanof/CodeEditSearch |
| name: MTEB CodeEditSearchRetrieval (ruby) |
| config: ruby |
| split: train |
| revision: 4e51c66e0939303f6928472f13ad0848b2a3f4c0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 57.0 |
| - type: ndcg_at_3 |
| value: 62.601 |
| - type: ndcg_at_5 |
| value: 64.12 |
| - type: ndcg_at_10 |
| value: 65.704 |
| - type: ndcg_at_20 |
| value: 66.703 |
| - type: ndcg_at_100 |
| value: 68.392 |
| - type: ndcg_at_1000 |
| value: 69.888 |
| - type: map_at_1 |
| value: 57.0 |
| - type: map_at_3 |
| value: 61.25 |
| - type: map_at_5 |
| value: 62.09 |
| - type: map_at_10 |
| value: 62.743 |
| - type: map_at_20 |
| value: 63.024 |
| - type: map_at_100 |
| value: 63.258 |
| - type: map_at_1000 |
| value: 63.312 |
| - type: recall_at_1 |
| value: 57.0 |
| - type: recall_at_3 |
| value: 66.5 |
| - type: recall_at_5 |
| value: 70.2 |
| - type: recall_at_10 |
| value: 75.1 |
| - type: recall_at_20 |
| value: 79.0 |
| - type: recall_at_100 |
| value: 88.1 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 57.0 |
| - type: precision_at_1 |
| value: 57.0 |
| - type: precision_at_3 |
| value: 22.167 |
| - type: precision_at_5 |
| value: 14.04 |
| - type: precision_at_10 |
| value: 7.51 |
| - type: precision_at_20 |
| value: 3.95 |
| - type: precision_at_100 |
| value: 0.881 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 57.0 |
| - type: mrr_at_3 |
| value: 61.25 |
| - type: mrr_at_5 |
| value: 62.09 |
| - type: mrr_at_10 |
| value: 62.7427 |
| - type: mrr_at_20 |
| value: 63.0244 |
| - type: mrr_at_100 |
| value: 63.2583 |
| - type: mrr_at_1000 |
| value: 63.3121 |
| - type: hit_rate_at_1 |
| value: 57.0 |
| - type: hit_rate_at_3 |
| value: 66.5 |
| - type: hit_rate_at_5 |
| value: 70.2 |
| - type: hit_rate_at_10 |
| value: 75.1 |
| - type: hit_rate_at_20 |
| value: 79.0 |
| - type: hit_rate_at_100 |
| value: 88.1 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: cassanof/CodeEditSearch |
| name: MTEB CodeEditSearchRetrieval (java) |
| config: java |
| split: train |
| revision: 4e51c66e0939303f6928472f13ad0848b2a3f4c0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 59.0 |
| - type: ndcg_at_3 |
| value: 64.377 |
| - type: ndcg_at_5 |
| value: 66.155 |
| - type: ndcg_at_10 |
| value: 67.214 |
| - type: ndcg_at_20 |
| value: 68.326 |
| - type: ndcg_at_100 |
| value: 69.854 |
| - type: ndcg_at_1000 |
| value: 71.242 |
| - type: map_at_1 |
| value: 59.0 |
| - type: map_at_3 |
| value: 63.117 |
| - type: map_at_5 |
| value: 64.107 |
| - type: map_at_10 |
| value: 64.54 |
| - type: map_at_20 |
| value: 64.844 |
| - type: map_at_100 |
| value: 65.038 |
| - type: map_at_1000 |
| value: 65.08 |
| - type: recall_at_1 |
| value: 59.0 |
| - type: recall_at_3 |
| value: 68.0 |
| - type: recall_at_5 |
| value: 72.3 |
| - type: recall_at_10 |
| value: 75.6 |
| - type: recall_at_20 |
| value: 80.0 |
| - type: recall_at_100 |
| value: 88.5 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 59.0 |
| - type: precision_at_1 |
| value: 59.0 |
| - type: precision_at_3 |
| value: 22.667 |
| - type: precision_at_5 |
| value: 14.46 |
| - type: precision_at_10 |
| value: 7.56 |
| - type: precision_at_20 |
| value: 4.0 |
| - type: precision_at_100 |
| value: 0.885 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 59.0 |
| - type: mrr_at_3 |
| value: 63.1167 |
| - type: mrr_at_5 |
| value: 64.1067 |
| - type: mrr_at_10 |
| value: 64.5398 |
| - type: mrr_at_20 |
| value: 64.8443 |
| - type: mrr_at_100 |
| value: 65.0381 |
| - type: mrr_at_1000 |
| value: 65.0799 |
| - type: hit_rate_at_1 |
| value: 59.0 |
| - type: hit_rate_at_3 |
| value: 68.0 |
| - type: hit_rate_at_5 |
| value: 72.3 |
| - type: hit_rate_at_10 |
| value: 75.6 |
| - type: hit_rate_at_20 |
| value: 80.0 |
| - type: hit_rate_at_100 |
| value: 88.5 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: cassanof/CodeEditSearch |
| name: MTEB CodeEditSearchRetrieval (php) |
| config: php |
| split: train |
| revision: 4e51c66e0939303f6928472f13ad0848b2a3f4c0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 51.9 |
| - type: ndcg_at_3 |
| value: 58.025 |
| - type: ndcg_at_5 |
| value: 60.469 |
| - type: ndcg_at_10 |
| value: 61.971 |
| - type: ndcg_at_20 |
| value: 63.223 |
| - type: ndcg_at_100 |
| value: 65.082 |
| - type: ndcg_at_1000 |
| value: 66.538 |
| - type: map_at_1 |
| value: 51.9 |
| - type: map_at_3 |
| value: 56.483 |
| - type: map_at_5 |
| value: 57.848 |
| - type: map_at_10 |
| value: 58.476 |
| - type: map_at_20 |
| value: 58.813 |
| - type: map_at_100 |
| value: 59.06 |
| - type: map_at_1000 |
| value: 59.109 |
| - type: recall_at_1 |
| value: 51.9 |
| - type: recall_at_3 |
| value: 62.5 |
| - type: recall_at_5 |
| value: 68.4 |
| - type: recall_at_10 |
| value: 73.0 |
| - type: recall_at_20 |
| value: 78.0 |
| - type: recall_at_100 |
| value: 88.2 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 51.9 |
| - type: precision_at_1 |
| value: 51.9 |
| - type: precision_at_3 |
| value: 20.833 |
| - type: precision_at_5 |
| value: 13.68 |
| - type: precision_at_10 |
| value: 7.3 |
| - type: precision_at_20 |
| value: 3.9 |
| - type: precision_at_100 |
| value: 0.882 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 51.9 |
| - type: mrr_at_3 |
| value: 56.4833 |
| - type: mrr_at_5 |
| value: 57.8483 |
| - type: mrr_at_10 |
| value: 58.476 |
| - type: mrr_at_20 |
| value: 58.8133 |
| - type: mrr_at_100 |
| value: 59.0602 |
| - type: mrr_at_1000 |
| value: 59.1089 |
| - type: hit_rate_at_1 |
| value: 51.9 |
| - type: hit_rate_at_3 |
| value: 62.5 |
| - type: hit_rate_at_5 |
| value: 68.4 |
| - type: hit_rate_at_10 |
| value: 73.0 |
| - type: hit_rate_at_20 |
| value: 78.0 |
| - type: hit_rate_at_100 |
| value: 88.2 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: cassanof/CodeEditSearch |
| name: MTEB CodeEditSearchRetrieval (c) |
| config: c |
| split: train |
| revision: 4e51c66e0939303f6928472f13ad0848b2a3f4c0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 46.8 |
| - type: ndcg_at_3 |
| value: 52.806 |
| - type: ndcg_at_5 |
| value: 54.153 |
| - type: ndcg_at_10 |
| value: 55.811 |
| - type: ndcg_at_20 |
| value: 56.798 |
| - type: ndcg_at_100 |
| value: 58.674 |
| - type: ndcg_at_1000 |
| value: 61.137 |
| - type: map_at_1 |
| value: 46.8 |
| - type: map_at_3 |
| value: 51.383 |
| - type: map_at_5 |
| value: 52.123 |
| - type: map_at_10 |
| value: 52.813 |
| - type: map_at_20 |
| value: 53.085 |
| - type: map_at_100 |
| value: 53.326 |
| - type: map_at_1000 |
| value: 53.402 |
| - type: recall_at_1 |
| value: 46.8 |
| - type: recall_at_3 |
| value: 56.9 |
| - type: recall_at_5 |
| value: 60.2 |
| - type: recall_at_10 |
| value: 65.3 |
| - type: recall_at_20 |
| value: 69.2 |
| - type: recall_at_100 |
| value: 79.6 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 46.8 |
| - type: precision_at_1 |
| value: 46.8 |
| - type: precision_at_3 |
| value: 18.967 |
| - type: precision_at_5 |
| value: 12.04 |
| - type: precision_at_10 |
| value: 6.53 |
| - type: precision_at_20 |
| value: 3.46 |
| - type: precision_at_100 |
| value: 0.796 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 46.8 |
| - type: mrr_at_3 |
| value: 51.3833 |
| - type: mrr_at_5 |
| value: 52.1233 |
| - type: mrr_at_10 |
| value: 52.8133 |
| - type: mrr_at_20 |
| value: 53.0852 |
| - type: mrr_at_100 |
| value: 53.3263 |
| - type: mrr_at_1000 |
| value: 53.4015 |
| - type: hit_rate_at_1 |
| value: 46.8 |
| - type: hit_rate_at_3 |
| value: 56.9 |
| - type: hit_rate_at_5 |
| value: 60.2 |
| - type: hit_rate_at_10 |
| value: 65.3 |
| - type: hit_rate_at_20 |
| value: 69.2 |
| - type: hit_rate_at_100 |
| value: 79.6 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: cassanof/CodeEditSearch |
| name: MTEB CodeEditSearchRetrieval (c++) |
| config: c++ |
| split: train |
| revision: 4e51c66e0939303f6928472f13ad0848b2a3f4c0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 49.2 |
| - type: ndcg_at_3 |
| value: 56.789 |
| - type: ndcg_at_5 |
| value: 58.015 |
| - type: ndcg_at_10 |
| value: 59.559 |
| - type: ndcg_at_20 |
| value: 60.989 |
| - type: ndcg_at_100 |
| value: 62.867 |
| - type: ndcg_at_1000 |
| value: 64.568 |
| - type: map_at_1 |
| value: 49.2 |
| - type: map_at_3 |
| value: 54.95 |
| - type: map_at_5 |
| value: 55.625 |
| - type: map_at_10 |
| value: 56.257 |
| - type: map_at_20 |
| value: 56.644 |
| - type: map_at_100 |
| value: 56.891 |
| - type: map_at_1000 |
| value: 56.944 |
| - type: recall_at_1 |
| value: 49.2 |
| - type: recall_at_3 |
| value: 62.1 |
| - type: recall_at_5 |
| value: 65.1 |
| - type: recall_at_10 |
| value: 69.9 |
| - type: recall_at_20 |
| value: 75.6 |
| - type: recall_at_100 |
| value: 85.9 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 49.2 |
| - type: precision_at_1 |
| value: 49.2 |
| - type: precision_at_3 |
| value: 20.7 |
| - type: precision_at_5 |
| value: 13.02 |
| - type: precision_at_10 |
| value: 6.99 |
| - type: precision_at_20 |
| value: 3.78 |
| - type: precision_at_100 |
| value: 0.859 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 49.2 |
| - type: mrr_at_3 |
| value: 54.95 |
| - type: mrr_at_5 |
| value: 55.625 |
| - type: mrr_at_10 |
| value: 56.2566 |
| - type: mrr_at_20 |
| value: 56.6436 |
| - type: mrr_at_100 |
| value: 56.8913 |
| - type: mrr_at_1000 |
| value: 56.9436 |
| - type: hit_rate_at_1 |
| value: 49.2 |
| - type: hit_rate_at_3 |
| value: 62.1 |
| - type: hit_rate_at_5 |
| value: 65.1 |
| - type: hit_rate_at_10 |
| value: 69.9 |
| - type: hit_rate_at_20 |
| value: 75.6 |
| - type: hit_rate_at_100 |
| value: 85.9 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: cassanof/CodeEditSearch |
| name: MTEB CodeEditSearchRetrieval (rust) |
| config: rust |
| split: train |
| revision: 4e51c66e0939303f6928472f13ad0848b2a3f4c0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 46.0 |
| - type: ndcg_at_3 |
| value: 51.93 |
| - type: ndcg_at_5 |
| value: 53.651 |
| - type: ndcg_at_10 |
| value: 55.111 |
| - type: ndcg_at_20 |
| value: 56.559 |
| - type: ndcg_at_100 |
| value: 58.792 |
| - type: ndcg_at_1000 |
| value: 60.929 |
| - type: map_at_1 |
| value: 46.0 |
| - type: map_at_3 |
| value: 50.517 |
| - type: map_at_5 |
| value: 51.467 |
| - type: map_at_10 |
| value: 52.072 |
| - type: map_at_20 |
| value: 52.46 |
| - type: map_at_100 |
| value: 52.765 |
| - type: map_at_1000 |
| value: 52.836 |
| - type: recall_at_1 |
| value: 46.0 |
| - type: recall_at_3 |
| value: 56.0 |
| - type: recall_at_5 |
| value: 60.2 |
| - type: recall_at_10 |
| value: 64.7 |
| - type: recall_at_20 |
| value: 70.5 |
| - type: recall_at_100 |
| value: 82.6 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 46.0 |
| - type: precision_at_1 |
| value: 46.0 |
| - type: precision_at_3 |
| value: 18.667 |
| - type: precision_at_5 |
| value: 12.04 |
| - type: precision_at_10 |
| value: 6.47 |
| - type: precision_at_20 |
| value: 3.525 |
| - type: precision_at_100 |
| value: 0.826 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 46.0 |
| - type: mrr_at_3 |
| value: 50.5167 |
| - type: mrr_at_5 |
| value: 51.4667 |
| - type: mrr_at_10 |
| value: 52.0723 |
| - type: mrr_at_20 |
| value: 52.4604 |
| - type: mrr_at_100 |
| value: 52.7653 |
| - type: mrr_at_1000 |
| value: 52.836 |
| - type: hit_rate_at_1 |
| value: 46.0 |
| - type: hit_rate_at_3 |
| value: 56.0 |
| - type: hit_rate_at_5 |
| value: 60.2 |
| - type: hit_rate_at_10 |
| value: 64.7 |
| - type: hit_rate_at_20 |
| value: 70.5 |
| - type: hit_rate_at_100 |
| value: 82.6 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: cassanof/CodeEditSearch |
| name: MTEB CodeEditSearchRetrieval (swift) |
| config: swift |
| split: train |
| revision: 4e51c66e0939303f6928472f13ad0848b2a3f4c0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 53.5 |
| - type: ndcg_at_3 |
| value: 60.574 |
| - type: ndcg_at_5 |
| value: 62.351 |
| - type: ndcg_at_10 |
| value: 63.931 |
| - type: ndcg_at_20 |
| value: 65.096 |
| - type: ndcg_at_100 |
| value: 66.518 |
| - type: ndcg_at_1000 |
| value: 68.106 |
| - type: map_at_1 |
| value: 53.5 |
| - type: map_at_3 |
| value: 58.867 |
| - type: map_at_5 |
| value: 59.857 |
| - type: map_at_10 |
| value: 60.526 |
| - type: map_at_20 |
| value: 60.847 |
| - type: map_at_100 |
| value: 61.04 |
| - type: map_at_1000 |
| value: 61.089 |
| - type: recall_at_1 |
| value: 53.5 |
| - type: recall_at_3 |
| value: 65.5 |
| - type: recall_at_5 |
| value: 69.8 |
| - type: recall_at_10 |
| value: 74.6 |
| - type: recall_at_20 |
| value: 79.2 |
| - type: recall_at_100 |
| value: 86.9 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 53.5 |
| - type: precision_at_1 |
| value: 53.5 |
| - type: precision_at_3 |
| value: 21.833 |
| - type: precision_at_5 |
| value: 13.96 |
| - type: precision_at_10 |
| value: 7.46 |
| - type: precision_at_20 |
| value: 3.96 |
| - type: precision_at_100 |
| value: 0.869 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 53.5 |
| - type: mrr_at_3 |
| value: 58.8667 |
| - type: mrr_at_5 |
| value: 59.8567 |
| - type: mrr_at_10 |
| value: 60.5258 |
| - type: mrr_at_20 |
| value: 60.8471 |
| - type: mrr_at_100 |
| value: 61.0404 |
| - type: mrr_at_1000 |
| value: 61.0886 |
| - type: hit_rate_at_1 |
| value: 53.5 |
| - type: hit_rate_at_3 |
| value: 65.5 |
| - type: hit_rate_at_5 |
| value: 69.8 |
| - type: hit_rate_at_10 |
| value: 74.6 |
| - type: hit_rate_at_20 |
| value: 79.2 |
| - type: hit_rate_at_100 |
| value: 86.9 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: cassanof/CodeEditSearch |
| name: MTEB CodeEditSearchRetrieval (scala) |
| config: scala |
| split: train |
| revision: 4e51c66e0939303f6928472f13ad0848b2a3f4c0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 46.0 |
| - type: ndcg_at_3 |
| value: 53.695 |
| - type: ndcg_at_5 |
| value: 56.2 |
| - type: ndcg_at_10 |
| value: 58.235 |
| - type: ndcg_at_20 |
| value: 59.909 |
| - type: ndcg_at_100 |
| value: 61.916 |
| - type: ndcg_at_1000 |
| value: 63.232 |
| - type: map_at_1 |
| value: 46.0 |
| - type: map_at_3 |
| value: 51.733 |
| - type: map_at_5 |
| value: 53.143 |
| - type: map_at_10 |
| value: 54.001 |
| - type: map_at_20 |
| value: 54.45 |
| - type: map_at_100 |
| value: 54.722 |
| - type: map_at_1000 |
| value: 54.765 |
| - type: recall_at_1 |
| value: 46.0 |
| - type: recall_at_3 |
| value: 59.4 |
| - type: recall_at_5 |
| value: 65.4 |
| - type: recall_at_10 |
| value: 71.6 |
| - type: recall_at_20 |
| value: 78.3 |
| - type: recall_at_100 |
| value: 89.2 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 46.0 |
| - type: precision_at_1 |
| value: 46.0 |
| - type: precision_at_3 |
| value: 19.8 |
| - type: precision_at_5 |
| value: 13.08 |
| - type: precision_at_10 |
| value: 7.16 |
| - type: precision_at_20 |
| value: 3.915 |
| - type: precision_at_100 |
| value: 0.892 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 46.0 |
| - type: mrr_at_3 |
| value: 51.7333 |
| - type: mrr_at_5 |
| value: 53.1433 |
| - type: mrr_at_10 |
| value: 54.0009 |
| - type: mrr_at_20 |
| value: 54.4497 |
| - type: mrr_at_100 |
| value: 54.7219 |
| - type: mrr_at_1000 |
| value: 54.7647 |
| - type: hit_rate_at_1 |
| value: 46.0 |
| - type: hit_rate_at_3 |
| value: 59.4 |
| - type: hit_rate_at_5 |
| value: 65.4 |
| - type: hit_rate_at_10 |
| value: 71.6 |
| - type: hit_rate_at_20 |
| value: 78.3 |
| - type: hit_rate_at_100 |
| value: 89.2 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: cassanof/CodeEditSearch |
| name: MTEB CodeEditSearchRetrieval (shell) |
| config: shell |
| split: train |
| revision: 4e51c66e0939303f6928472f13ad0848b2a3f4c0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 47.3 |
| - type: ndcg_at_3 |
| value: 54.487 |
| - type: ndcg_at_5 |
| value: 56.341 |
| - type: ndcg_at_10 |
| value: 58.611 |
| - type: ndcg_at_20 |
| value: 59.769 |
| - type: ndcg_at_100 |
| value: 61.924 |
| - type: ndcg_at_1000 |
| value: 63.473 |
| - type: map_at_1 |
| value: 47.3 |
| - type: map_at_3 |
| value: 52.75 |
| - type: map_at_5 |
| value: 53.78 |
| - type: map_at_10 |
| value: 54.719 |
| - type: map_at_20 |
| value: 55.048 |
| - type: map_at_100 |
| value: 55.333 |
| - type: map_at_1000 |
| value: 55.382 |
| - type: recall_at_1 |
| value: 47.3 |
| - type: recall_at_3 |
| value: 59.5 |
| - type: recall_at_5 |
| value: 64.0 |
| - type: recall_at_10 |
| value: 71.0 |
| - type: recall_at_20 |
| value: 75.5 |
| - type: recall_at_100 |
| value: 87.3 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 47.3 |
| - type: precision_at_1 |
| value: 47.3 |
| - type: precision_at_3 |
| value: 19.833 |
| - type: precision_at_5 |
| value: 12.8 |
| - type: precision_at_10 |
| value: 7.1 |
| - type: precision_at_20 |
| value: 3.775 |
| - type: precision_at_100 |
| value: 0.873 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 47.3 |
| - type: mrr_at_3 |
| value: 52.75 |
| - type: mrr_at_5 |
| value: 53.78 |
| - type: mrr_at_10 |
| value: 54.7192 |
| - type: mrr_at_20 |
| value: 55.0481 |
| - type: mrr_at_100 |
| value: 55.3328 |
| - type: mrr_at_1000 |
| value: 55.3818 |
| - type: hit_rate_at_1 |
| value: 47.3 |
| - type: hit_rate_at_3 |
| value: 59.5 |
| - type: hit_rate_at_5 |
| value: 64.0 |
| - type: hit_rate_at_10 |
| value: 71.0 |
| - type: hit_rate_at_20 |
| value: 75.5 |
| - type: hit_rate_at_100 |
| value: 87.3 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/codefeedback-mt |
| name: MTEB CodeFeedbackMT |
| config: default |
| split: test |
| revision: b0f12fa0c0dd67f59c95a5c33d02aeeb4c398c5f |
| metrics: |
| - type: ndcg_at_1 |
| value: 76.177 |
| - type: ndcg_at_3 |
| value: 82.191 |
| - type: ndcg_at_5 |
| value: 83.3 |
| - type: ndcg_at_10 |
| value: 84.37 |
| - type: ndcg_at_20 |
| value: 84.893 |
| - type: ndcg_at_100 |
| value: 85.524 |
| - type: ndcg_at_1000 |
| value: 85.754 |
| - type: map_at_1 |
| value: 76.177 |
| - type: map_at_3 |
| value: 80.766 |
| - type: map_at_5 |
| value: 81.382 |
| - type: map_at_10 |
| value: 81.83 |
| - type: map_at_20 |
| value: 81.974 |
| - type: map_at_100 |
| value: 82.061 |
| - type: map_at_1000 |
| value: 82.07 |
| - type: recall_at_1 |
| value: 76.177 |
| - type: recall_at_3 |
| value: 86.292 |
| - type: recall_at_5 |
| value: 88.981 |
| - type: recall_at_10 |
| value: 92.257 |
| - type: recall_at_20 |
| value: 94.321 |
| - type: recall_at_100 |
| value: 97.725 |
| - type: recall_at_1000 |
| value: 99.533 |
| - type: accuracy |
| value: 76.177 |
| - type: precision_at_1 |
| value: 76.177 |
| - type: precision_at_3 |
| value: 28.764 |
| - type: precision_at_5 |
| value: 17.796 |
| - type: precision_at_10 |
| value: 9.226 |
| - type: precision_at_20 |
| value: 4.716 |
| - type: precision_at_100 |
| value: 0.977 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 76.1768 |
| - type: mrr_at_3 |
| value: 80.7662 |
| - type: mrr_at_5 |
| value: 81.3823 |
| - type: mrr_at_10 |
| value: 81.83 |
| - type: mrr_at_20 |
| value: 81.9744 |
| - type: mrr_at_100 |
| value: 82.0608 |
| - type: mrr_at_1000 |
| value: 82.0695 |
| - type: hit_rate_at_1 |
| value: 76.177 |
| - type: hit_rate_at_3 |
| value: 86.292 |
| - type: hit_rate_at_5 |
| value: 88.981 |
| - type: hit_rate_at_10 |
| value: 92.257 |
| - type: hit_rate_at_20 |
| value: 94.321 |
| - type: hit_rate_at_100 |
| value: 97.725 |
| - type: hit_rate_at_1000 |
| value: 99.533 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/codefeedback-st |
| name: MTEB CodeFeedbackST |
| config: default |
| split: test |
| revision: d213819e87aab9010628da8b73ab4eb337c89340 |
| metrics: |
| - type: ndcg_at_1 |
| value: 74.034 |
| - type: ndcg_at_3 |
| value: 84.331 |
| - type: ndcg_at_5 |
| value: 85.704 |
| - type: ndcg_at_10 |
| value: 86.404 |
| - type: ndcg_at_20 |
| value: 86.707 |
| - type: ndcg_at_100 |
| value: 86.966 |
| - type: ndcg_at_1000 |
| value: 87.045 |
| - type: map_at_1 |
| value: 74.034 |
| - type: map_at_3 |
| value: 81.926 |
| - type: map_at_5 |
| value: 82.694 |
| - type: map_at_10 |
| value: 82.988 |
| - type: map_at_20 |
| value: 83.072 |
| - type: map_at_100 |
| value: 83.11 |
| - type: map_at_1000 |
| value: 83.113 |
| - type: recall_at_1 |
| value: 74.034 |
| - type: recall_at_3 |
| value: 91.238 |
| - type: recall_at_5 |
| value: 94.547 |
| - type: recall_at_10 |
| value: 96.684 |
| - type: recall_at_20 |
| value: 97.873 |
| - type: recall_at_100 |
| value: 99.249 |
| - type: recall_at_1000 |
| value: 99.85 |
| - type: accuracy |
| value: 74.034 |
| - type: precision_at_1 |
| value: 74.034 |
| - type: precision_at_3 |
| value: 30.413 |
| - type: precision_at_5 |
| value: 18.909 |
| - type: precision_at_10 |
| value: 9.668 |
| - type: precision_at_20 |
| value: 4.894 |
| - type: precision_at_100 |
| value: 0.992 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 74.0337 |
| - type: mrr_at_3 |
| value: 81.9257 |
| - type: mrr_at_5 |
| value: 82.6938 |
| - type: mrr_at_10 |
| value: 82.9883 |
| - type: mrr_at_20 |
| value: 83.0724 |
| - type: mrr_at_100 |
| value: 83.1096 |
| - type: mrr_at_1000 |
| value: 83.113 |
| - type: hit_rate_at_1 |
| value: 74.034 |
| - type: hit_rate_at_3 |
| value: 91.238 |
| - type: hit_rate_at_5 |
| value: 94.547 |
| - type: hit_rate_at_10 |
| value: 96.684 |
| - type: hit_rate_at_20 |
| value: 97.873 |
| - type: hit_rate_at_100 |
| value: 99.249 |
| - type: hit_rate_at_1000 |
| value: 99.85 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/CodeSearchNet-ccr |
| name: MTEB CodeSearchNetCCRetrieval (python) |
| config: python |
| split: test |
| revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 |
| metrics: |
| - type: ndcg_at_1 |
| value: 90.609 |
| - type: ndcg_at_3 |
| value: 94.31 |
| - type: ndcg_at_5 |
| value: 94.675 |
| - type: ndcg_at_10 |
| value: 94.919 |
| - type: ndcg_at_20 |
| value: 95.062 |
| - type: ndcg_at_100 |
| value: 95.191 |
| - type: ndcg_at_1000 |
| value: 95.23 |
| - type: map_at_1 |
| value: 90.609 |
| - type: map_at_3 |
| value: 93.459 |
| - type: map_at_5 |
| value: 93.664 |
| - type: map_at_10 |
| value: 93.764 |
| - type: map_at_20 |
| value: 93.804 |
| - type: map_at_100 |
| value: 93.822 |
| - type: map_at_1000 |
| value: 93.824 |
| - type: recall_at_1 |
| value: 90.609 |
| - type: recall_at_3 |
| value: 96.749 |
| - type: recall_at_5 |
| value: 97.627 |
| - type: recall_at_10 |
| value: 98.378 |
| - type: recall_at_20 |
| value: 98.948 |
| - type: recall_at_100 |
| value: 99.631 |
| - type: recall_at_1000 |
| value: 99.933 |
| - type: accuracy |
| value: 90.609 |
| - type: precision_at_1 |
| value: 90.609 |
| - type: precision_at_3 |
| value: 32.25 |
| - type: precision_at_5 |
| value: 19.525 |
| - type: precision_at_10 |
| value: 9.838 |
| - type: precision_at_20 |
| value: 4.947 |
| - type: precision_at_100 |
| value: 0.996 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 90.6087 |
| - type: mrr_at_3 |
| value: 93.4587 |
| - type: mrr_at_5 |
| value: 93.6638 |
| - type: mrr_at_10 |
| value: 93.7644 |
| - type: mrr_at_20 |
| value: 93.8036 |
| - type: mrr_at_100 |
| value: 93.8222 |
| - type: mrr_at_1000 |
| value: 93.8237 |
| - type: hit_rate_at_1 |
| value: 90.609 |
| - type: hit_rate_at_3 |
| value: 96.749 |
| - type: hit_rate_at_5 |
| value: 97.627 |
| - type: hit_rate_at_10 |
| value: 98.378 |
| - type: hit_rate_at_20 |
| value: 98.948 |
| - type: hit_rate_at_100 |
| value: 99.631 |
| - type: hit_rate_at_1000 |
| value: 99.933 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/CodeSearchNet-ccr |
| name: MTEB CodeSearchNetCCRetrieval (javascript) |
| config: javascript |
| split: test |
| revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 |
| metrics: |
| - type: ndcg_at_1 |
| value: 90.884 |
| - type: ndcg_at_3 |
| value: 94.825 |
| - type: ndcg_at_5 |
| value: 95.146 |
| - type: ndcg_at_10 |
| value: 95.408 |
| - type: ndcg_at_20 |
| value: 95.479 |
| - type: ndcg_at_100 |
| value: 95.584 |
| - type: ndcg_at_1000 |
| value: 95.611 |
| - type: map_at_1 |
| value: 90.884 |
| - type: map_at_3 |
| value: 93.948 |
| - type: map_at_5 |
| value: 94.123 |
| - type: map_at_10 |
| value: 94.235 |
| - type: map_at_20 |
| value: 94.255 |
| - type: map_at_100 |
| value: 94.272 |
| - type: map_at_1000 |
| value: 94.273 |
| - type: recall_at_1 |
| value: 90.884 |
| - type: recall_at_3 |
| value: 97.326 |
| - type: recall_at_5 |
| value: 98.116 |
| - type: recall_at_10 |
| value: 98.906 |
| - type: recall_at_20 |
| value: 99.18 |
| - type: recall_at_100 |
| value: 99.727 |
| - type: recall_at_1000 |
| value: 99.939 |
| - type: accuracy |
| value: 90.884 |
| - type: precision_at_1 |
| value: 90.884 |
| - type: precision_at_3 |
| value: 32.442 |
| - type: precision_at_5 |
| value: 19.623 |
| - type: precision_at_10 |
| value: 9.891 |
| - type: precision_at_20 |
| value: 4.959 |
| - type: precision_at_100 |
| value: 0.997 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 90.8842 |
| - type: mrr_at_3 |
| value: 93.9481 |
| - type: mrr_at_5 |
| value: 94.1229 |
| - type: mrr_at_10 |
| value: 94.2349 |
| - type: mrr_at_20 |
| value: 94.2554 |
| - type: mrr_at_100 |
| value: 94.2716 |
| - type: mrr_at_1000 |
| value: 94.2727 |
| - type: hit_rate_at_1 |
| value: 90.884 |
| - type: hit_rate_at_3 |
| value: 97.326 |
| - type: hit_rate_at_5 |
| value: 98.116 |
| - type: hit_rate_at_10 |
| value: 98.906 |
| - type: hit_rate_at_20 |
| value: 99.18 |
| - type: hit_rate_at_100 |
| value: 99.727 |
| - type: hit_rate_at_1000 |
| value: 99.939 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/CodeSearchNet-ccr |
| name: MTEB CodeSearchNetCCRetrieval (go) |
| config: go |
| split: test |
| revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 |
| metrics: |
| - type: ndcg_at_1 |
| value: 91.0 |
| - type: ndcg_at_3 |
| value: 94.579 |
| - type: ndcg_at_5 |
| value: 94.997 |
| - type: ndcg_at_10 |
| value: 95.292 |
| - type: ndcg_at_20 |
| value: 95.422 |
| - type: ndcg_at_100 |
| value: 95.507 |
| - type: ndcg_at_1000 |
| value: 95.529 |
| - type: map_at_1 |
| value: 91.0 |
| - type: map_at_3 |
| value: 93.749 |
| - type: map_at_5 |
| value: 93.983 |
| - type: map_at_10 |
| value: 94.104 |
| - type: map_at_20 |
| value: 94.141 |
| - type: map_at_100 |
| value: 94.155 |
| - type: map_at_1000 |
| value: 94.156 |
| - type: recall_at_1 |
| value: 91.0 |
| - type: recall_at_3 |
| value: 96.959 |
| - type: recall_at_5 |
| value: 97.968 |
| - type: recall_at_10 |
| value: 98.88 |
| - type: recall_at_20 |
| value: 99.384 |
| - type: recall_at_100 |
| value: 99.815 |
| - type: recall_at_1000 |
| value: 99.988 |
| - type: accuracy |
| value: 91.0 |
| - type: precision_at_1 |
| value: 91.0 |
| - type: precision_at_3 |
| value: 32.32 |
| - type: precision_at_5 |
| value: 19.594 |
| - type: precision_at_10 |
| value: 9.888 |
| - type: precision_at_20 |
| value: 4.969 |
| - type: precision_at_100 |
| value: 0.998 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 90.9998 |
| - type: mrr_at_3 |
| value: 93.7495 |
| - type: mrr_at_5 |
| value: 93.9828 |
| - type: mrr_at_10 |
| value: 94.1045 |
| - type: mrr_at_20 |
| value: 94.1411 |
| - type: mrr_at_100 |
| value: 94.155 |
| - type: mrr_at_1000 |
| value: 94.1559 |
| - type: hit_rate_at_1 |
| value: 91.0 |
| - type: hit_rate_at_3 |
| value: 96.959 |
| - type: hit_rate_at_5 |
| value: 97.968 |
| - type: hit_rate_at_10 |
| value: 98.88 |
| - type: hit_rate_at_20 |
| value: 99.384 |
| - type: hit_rate_at_100 |
| value: 99.815 |
| - type: hit_rate_at_1000 |
| value: 99.988 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/CodeSearchNet-ccr |
| name: MTEB CodeSearchNetCCRetrieval (ruby) |
| config: ruby |
| split: test |
| revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 |
| metrics: |
| - type: ndcg_at_1 |
| value: 84.853 |
| - type: ndcg_at_3 |
| value: 89.856 |
| - type: ndcg_at_5 |
| value: 90.798 |
| - type: ndcg_at_10 |
| value: 91.306 |
| - type: ndcg_at_20 |
| value: 91.568 |
| - type: ndcg_at_100 |
| value: 91.73 |
| - type: ndcg_at_1000 |
| value: 91.833 |
| - type: map_at_1 |
| value: 84.853 |
| - type: map_at_3 |
| value: 88.673 |
| - type: map_at_5 |
| value: 89.192 |
| - type: map_at_10 |
| value: 89.4 |
| - type: map_at_20 |
| value: 89.473 |
| - type: map_at_100 |
| value: 89.495 |
| - type: map_at_1000 |
| value: 89.499 |
| - type: recall_at_1 |
| value: 84.853 |
| - type: recall_at_3 |
| value: 93.259 |
| - type: recall_at_5 |
| value: 95.559 |
| - type: recall_at_10 |
| value: 97.145 |
| - type: recall_at_20 |
| value: 98.176 |
| - type: recall_at_100 |
| value: 99.048 |
| - type: recall_at_1000 |
| value: 99.841 |
| - type: accuracy |
| value: 84.853 |
| - type: precision_at_1 |
| value: 84.853 |
| - type: precision_at_3 |
| value: 31.086 |
| - type: precision_at_5 |
| value: 19.112 |
| - type: precision_at_10 |
| value: 9.715 |
| - type: precision_at_20 |
| value: 4.909 |
| - type: precision_at_100 |
| value: 0.99 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 84.8533 |
| - type: mrr_at_3 |
| value: 88.673 |
| - type: mrr_at_5 |
| value: 89.1924 |
| - type: mrr_at_10 |
| value: 89.3997 |
| - type: mrr_at_20 |
| value: 89.4725 |
| - type: mrr_at_100 |
| value: 89.4949 |
| - type: mrr_at_1000 |
| value: 89.499 |
| - type: hit_rate_at_1 |
| value: 84.853 |
| - type: hit_rate_at_3 |
| value: 93.259 |
| - type: hit_rate_at_5 |
| value: 95.559 |
| - type: hit_rate_at_10 |
| value: 97.145 |
| - type: hit_rate_at_20 |
| value: 98.176 |
| - type: hit_rate_at_100 |
| value: 99.048 |
| - type: hit_rate_at_1000 |
| value: 99.841 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/CodeSearchNet-ccr |
| name: MTEB CodeSearchNetCCRetrieval (java) |
| config: java |
| split: test |
| revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 |
| metrics: |
| - type: ndcg_at_1 |
| value: 90.141 |
| - type: ndcg_at_3 |
| value: 93.512 |
| - type: ndcg_at_5 |
| value: 93.874 |
| - type: ndcg_at_10 |
| value: 94.099 |
| - type: ndcg_at_20 |
| value: 94.228 |
| - type: ndcg_at_100 |
| value: 94.334 |
| - type: ndcg_at_1000 |
| value: 94.406 |
| - type: map_at_1 |
| value: 90.141 |
| - type: map_at_3 |
| value: 92.728 |
| - type: map_at_5 |
| value: 92.929 |
| - type: map_at_10 |
| value: 93.024 |
| - type: map_at_20 |
| value: 93.059 |
| - type: map_at_100 |
| value: 93.075 |
| - type: map_at_1000 |
| value: 93.077 |
| - type: recall_at_1 |
| value: 90.141 |
| - type: recall_at_3 |
| value: 95.764 |
| - type: recall_at_5 |
| value: 96.641 |
| - type: recall_at_10 |
| value: 97.325 |
| - type: recall_at_20 |
| value: 97.837 |
| - type: recall_at_100 |
| value: 98.393 |
| - type: recall_at_1000 |
| value: 98.978 |
| - type: accuracy |
| value: 90.141 |
| - type: precision_at_1 |
| value: 90.141 |
| - type: precision_at_3 |
| value: 31.921 |
| - type: precision_at_5 |
| value: 19.328 |
| - type: precision_at_10 |
| value: 9.733 |
| - type: precision_at_20 |
| value: 4.892 |
| - type: precision_at_100 |
| value: 0.984 |
| - type: precision_at_1000 |
| value: 0.099 |
| - type: mrr_at_1 |
| value: 90.1415 |
| - type: mrr_at_3 |
| value: 92.7278 |
| - type: mrr_at_5 |
| value: 92.9286 |
| - type: mrr_at_10 |
| value: 93.0239 |
| - type: mrr_at_20 |
| value: 93.0593 |
| - type: mrr_at_100 |
| value: 93.0747 |
| - type: mrr_at_1000 |
| value: 93.0772 |
| - type: hit_rate_at_1 |
| value: 90.141 |
| - type: hit_rate_at_3 |
| value: 95.764 |
| - type: hit_rate_at_5 |
| value: 96.641 |
| - type: hit_rate_at_10 |
| value: 97.325 |
| - type: hit_rate_at_20 |
| value: 97.837 |
| - type: hit_rate_at_100 |
| value: 98.393 |
| - type: hit_rate_at_1000 |
| value: 98.978 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/CodeSearchNet-ccr |
| name: MTEB CodeSearchNetCCRetrieval (php) |
| config: php |
| split: test |
| revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 |
| metrics: |
| - type: ndcg_at_1 |
| value: 85.293 |
| - type: ndcg_at_3 |
| value: 90.036 |
| - type: ndcg_at_5 |
| value: 90.775 |
| - type: ndcg_at_10 |
| value: 91.356 |
| - type: ndcg_at_20 |
| value: 91.625 |
| - type: ndcg_at_100 |
| value: 91.872 |
| - type: ndcg_at_1000 |
| value: 91.971 |
| - type: map_at_1 |
| value: 85.293 |
| - type: map_at_3 |
| value: 88.936 |
| - type: map_at_5 |
| value: 89.349 |
| - type: map_at_10 |
| value: 89.59 |
| - type: map_at_20 |
| value: 89.666 |
| - type: map_at_100 |
| value: 89.702 |
| - type: map_at_1000 |
| value: 89.707 |
| - type: recall_at_1 |
| value: 85.293 |
| - type: recall_at_3 |
| value: 93.193 |
| - type: recall_at_5 |
| value: 94.976 |
| - type: recall_at_10 |
| value: 96.76 |
| - type: recall_at_20 |
| value: 97.809 |
| - type: recall_at_100 |
| value: 99.108 |
| - type: recall_at_1000 |
| value: 99.879 |
| - type: accuracy |
| value: 85.293 |
| - type: precision_at_1 |
| value: 85.293 |
| - type: precision_at_3 |
| value: 31.064 |
| - type: precision_at_5 |
| value: 18.995 |
| - type: precision_at_10 |
| value: 9.676 |
| - type: precision_at_20 |
| value: 4.89 |
| - type: precision_at_100 |
| value: 0.991 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 85.2933 |
| - type: mrr_at_3 |
| value: 88.9361 |
| - type: mrr_at_5 |
| value: 89.3489 |
| - type: mrr_at_10 |
| value: 89.5904 |
| - type: mrr_at_20 |
| value: 89.6663 |
| - type: mrr_at_100 |
| value: 89.7025 |
| - type: mrr_at_1000 |
| value: 89.7065 |
| - type: hit_rate_at_1 |
| value: 85.293 |
| - type: hit_rate_at_3 |
| value: 93.193 |
| - type: hit_rate_at_5 |
| value: 94.976 |
| - type: hit_rate_at_10 |
| value: 96.76 |
| - type: hit_rate_at_20 |
| value: 97.809 |
| - type: hit_rate_at_100 |
| value: 99.108 |
| - type: hit_rate_at_1000 |
| value: 99.879 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/CodeSearchNetRetrieval |
| name: MTEB CodeSearchNetRetrieval (python) |
| config: python |
| split: test |
| revision: 68e8f0731a656fa4bd5b7c81936d95ad48a39bfe |
| metrics: |
| - type: ndcg_at_1 |
| value: 90.1 |
| - type: ndcg_at_3 |
| value: 94.231 |
| - type: ndcg_at_5 |
| value: 94.696 |
| - type: ndcg_at_10 |
| value: 94.957 |
| - type: ndcg_at_20 |
| value: 95.028 |
| - type: ndcg_at_100 |
| value: 95.069 |
| - type: ndcg_at_1000 |
| value: 95.143 |
| - type: map_at_1 |
| value: 90.1 |
| - type: map_at_3 |
| value: 93.267 |
| - type: map_at_5 |
| value: 93.532 |
| - type: map_at_10 |
| value: 93.641 |
| - type: map_at_20 |
| value: 93.658 |
| - type: map_at_100 |
| value: 93.665 |
| - type: map_at_1000 |
| value: 93.668 |
| - type: recall_at_1 |
| value: 90.1 |
| - type: recall_at_3 |
| value: 97.0 |
| - type: recall_at_5 |
| value: 98.1 |
| - type: recall_at_10 |
| value: 98.9 |
| - type: recall_at_20 |
| value: 99.2 |
| - type: recall_at_100 |
| value: 99.4 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 90.1 |
| - type: precision_at_1 |
| value: 90.1 |
| - type: precision_at_3 |
| value: 32.333 |
| - type: precision_at_5 |
| value: 19.62 |
| - type: precision_at_10 |
| value: 9.89 |
| - type: precision_at_20 |
| value: 4.96 |
| - type: precision_at_100 |
| value: 0.994 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 90.1 |
| - type: mrr_at_3 |
| value: 93.2667 |
| - type: mrr_at_5 |
| value: 93.5317 |
| - type: mrr_at_10 |
| value: 93.6407 |
| - type: mrr_at_20 |
| value: 93.6582 |
| - type: mrr_at_100 |
| value: 93.6648 |
| - type: mrr_at_1000 |
| value: 93.6675 |
| - type: hit_rate_at_1 |
| value: 90.1 |
| - type: hit_rate_at_3 |
| value: 97.0 |
| - type: hit_rate_at_5 |
| value: 98.1 |
| - type: hit_rate_at_10 |
| value: 98.9 |
| - type: hit_rate_at_20 |
| value: 99.2 |
| - type: hit_rate_at_100 |
| value: 99.4 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/CodeSearchNetRetrieval |
| name: MTEB CodeSearchNetRetrieval (javascript) |
| config: javascript |
| split: test |
| revision: 68e8f0731a656fa4bd5b7c81936d95ad48a39bfe |
| metrics: |
| - type: ndcg_at_1 |
| value: 75.8 |
| - type: ndcg_at_3 |
| value: 80.999 |
| - type: ndcg_at_5 |
| value: 81.61 |
| - type: ndcg_at_10 |
| value: 82.588 |
| - type: ndcg_at_20 |
| value: 83.126 |
| - type: ndcg_at_100 |
| value: 83.73 |
| - type: ndcg_at_1000 |
| value: 84.398 |
| - type: map_at_1 |
| value: 75.8 |
| - type: map_at_3 |
| value: 79.75 |
| - type: map_at_5 |
| value: 80.085 |
| - type: map_at_10 |
| value: 80.494 |
| - type: map_at_20 |
| value: 80.645 |
| - type: map_at_100 |
| value: 80.718 |
| - type: map_at_1000 |
| value: 80.741 |
| - type: recall_at_1 |
| value: 75.8 |
| - type: recall_at_3 |
| value: 84.6 |
| - type: recall_at_5 |
| value: 86.1 |
| - type: recall_at_10 |
| value: 89.1 |
| - type: recall_at_20 |
| value: 91.2 |
| - type: recall_at_100 |
| value: 94.6 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 75.8 |
| - type: precision_at_1 |
| value: 75.8 |
| - type: precision_at_3 |
| value: 28.2 |
| - type: precision_at_5 |
| value: 17.22 |
| - type: precision_at_10 |
| value: 8.91 |
| - type: precision_at_20 |
| value: 4.56 |
| - type: precision_at_100 |
| value: 0.946 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 75.8 |
| - type: mrr_at_3 |
| value: 79.75 |
| - type: mrr_at_5 |
| value: 80.085 |
| - type: mrr_at_10 |
| value: 80.4936 |
| - type: mrr_at_20 |
| value: 80.6448 |
| - type: mrr_at_100 |
| value: 80.7182 |
| - type: mrr_at_1000 |
| value: 80.7413 |
| - type: hit_rate_at_1 |
| value: 75.8 |
| - type: hit_rate_at_3 |
| value: 84.6 |
| - type: hit_rate_at_5 |
| value: 86.1 |
| - type: hit_rate_at_10 |
| value: 89.1 |
| - type: hit_rate_at_20 |
| value: 91.2 |
| - type: hit_rate_at_100 |
| value: 94.6 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/CodeSearchNetRetrieval |
| name: MTEB CodeSearchNetRetrieval (go) |
| config: go |
| split: test |
| revision: 68e8f0731a656fa4bd5b7c81936d95ad48a39bfe |
| metrics: |
| - type: ndcg_at_1 |
| value: 91.8 |
| - type: ndcg_at_3 |
| value: 95.594 |
| - type: ndcg_at_5 |
| value: 95.882 |
| - type: ndcg_at_10 |
| value: 95.977 |
| - type: ndcg_at_20 |
| value: 96.075 |
| - type: ndcg_at_100 |
| value: 96.134 |
| - type: ndcg_at_1000 |
| value: 96.17 |
| - type: map_at_1 |
| value: 91.8 |
| - type: map_at_3 |
| value: 94.75 |
| - type: map_at_5 |
| value: 94.91 |
| - type: map_at_10 |
| value: 94.948 |
| - type: map_at_20 |
| value: 94.973 |
| - type: map_at_100 |
| value: 94.982 |
| - type: map_at_1000 |
| value: 94.983 |
| - type: recall_at_1 |
| value: 91.8 |
| - type: recall_at_3 |
| value: 98.0 |
| - type: recall_at_5 |
| value: 98.7 |
| - type: recall_at_10 |
| value: 99.0 |
| - type: recall_at_20 |
| value: 99.4 |
| - type: recall_at_100 |
| value: 99.7 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 91.8 |
| - type: precision_at_1 |
| value: 91.8 |
| - type: precision_at_3 |
| value: 32.667 |
| - type: precision_at_5 |
| value: 19.74 |
| - type: precision_at_10 |
| value: 9.9 |
| - type: precision_at_20 |
| value: 4.97 |
| - type: precision_at_100 |
| value: 0.997 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 91.8 |
| - type: mrr_at_3 |
| value: 94.75 |
| - type: mrr_at_5 |
| value: 94.91 |
| - type: mrr_at_10 |
| value: 94.9475 |
| - type: mrr_at_20 |
| value: 94.9728 |
| - type: mrr_at_100 |
| value: 94.9821 |
| - type: mrr_at_1000 |
| value: 94.983 |
| - type: hit_rate_at_1 |
| value: 91.8 |
| - type: hit_rate_at_3 |
| value: 98.0 |
| - type: hit_rate_at_5 |
| value: 98.7 |
| - type: hit_rate_at_10 |
| value: 99.0 |
| - type: hit_rate_at_20 |
| value: 99.4 |
| - type: hit_rate_at_100 |
| value: 99.7 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/CodeSearchNetRetrieval |
| name: MTEB CodeSearchNetRetrieval (ruby) |
| config: ruby |
| split: test |
| revision: 68e8f0731a656fa4bd5b7c81936d95ad48a39bfe |
| metrics: |
| - type: ndcg_at_1 |
| value: 79.2 |
| - type: ndcg_at_3 |
| value: 85.958 |
| - type: ndcg_at_5 |
| value: 86.793 |
| - type: ndcg_at_10 |
| value: 86.995 |
| - type: ndcg_at_20 |
| value: 87.271 |
| - type: ndcg_at_100 |
| value: 87.612 |
| - type: ndcg_at_1000 |
| value: 88.048 |
| - type: map_at_1 |
| value: 79.2 |
| - type: map_at_3 |
| value: 84.317 |
| - type: map_at_5 |
| value: 84.787 |
| - type: map_at_10 |
| value: 84.875 |
| - type: map_at_20 |
| value: 84.95 |
| - type: map_at_100 |
| value: 84.999 |
| - type: map_at_1000 |
| value: 85.009 |
| - type: recall_at_1 |
| value: 79.2 |
| - type: recall_at_3 |
| value: 90.7 |
| - type: recall_at_5 |
| value: 92.7 |
| - type: recall_at_10 |
| value: 93.3 |
| - type: recall_at_20 |
| value: 94.4 |
| - type: recall_at_100 |
| value: 96.2 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 79.2 |
| - type: precision_at_1 |
| value: 79.2 |
| - type: precision_at_3 |
| value: 30.233 |
| - type: precision_at_5 |
| value: 18.54 |
| - type: precision_at_10 |
| value: 9.33 |
| - type: precision_at_20 |
| value: 4.72 |
| - type: precision_at_100 |
| value: 0.962 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 79.2 |
| - type: mrr_at_3 |
| value: 84.3167 |
| - type: mrr_at_5 |
| value: 84.7867 |
| - type: mrr_at_10 |
| value: 84.8746 |
| - type: mrr_at_20 |
| value: 84.9495 |
| - type: mrr_at_100 |
| value: 84.9991 |
| - type: mrr_at_1000 |
| value: 85.0091 |
| - type: hit_rate_at_1 |
| value: 79.2 |
| - type: hit_rate_at_3 |
| value: 90.7 |
| - type: hit_rate_at_5 |
| value: 92.7 |
| - type: hit_rate_at_10 |
| value: 93.3 |
| - type: hit_rate_at_20 |
| value: 94.4 |
| - type: hit_rate_at_100 |
| value: 96.2 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/CodeSearchNetRetrieval |
| name: MTEB CodeSearchNetRetrieval (java) |
| config: java |
| split: test |
| revision: 68e8f0731a656fa4bd5b7c81936d95ad48a39bfe |
| metrics: |
| - type: ndcg_at_1 |
| value: 85.0 |
| - type: ndcg_at_3 |
| value: 90.343 |
| - type: ndcg_at_5 |
| value: 91.001 |
| - type: ndcg_at_10 |
| value: 91.302 |
| - type: ndcg_at_20 |
| value: 91.428 |
| - type: ndcg_at_100 |
| value: 91.527 |
| - type: ndcg_at_1000 |
| value: 91.845 |
| - type: map_at_1 |
| value: 85.0 |
| - type: map_at_3 |
| value: 89.133 |
| - type: map_at_5 |
| value: 89.498 |
| - type: map_at_10 |
| value: 89.629 |
| - type: map_at_20 |
| value: 89.663 |
| - type: map_at_100 |
| value: 89.679 |
| - type: map_at_1000 |
| value: 89.688 |
| - type: recall_at_1 |
| value: 85.0 |
| - type: recall_at_3 |
| value: 93.8 |
| - type: recall_at_5 |
| value: 95.4 |
| - type: recall_at_10 |
| value: 96.3 |
| - type: recall_at_20 |
| value: 96.8 |
| - type: recall_at_100 |
| value: 97.3 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 85.0 |
| - type: precision_at_1 |
| value: 85.0 |
| - type: precision_at_3 |
| value: 31.267 |
| - type: precision_at_5 |
| value: 19.08 |
| - type: precision_at_10 |
| value: 9.63 |
| - type: precision_at_20 |
| value: 4.84 |
| - type: precision_at_100 |
| value: 0.973 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 85.0 |
| - type: mrr_at_3 |
| value: 89.1333 |
| - type: mrr_at_5 |
| value: 89.4983 |
| - type: mrr_at_10 |
| value: 89.6286 |
| - type: mrr_at_20 |
| value: 89.6627 |
| - type: mrr_at_100 |
| value: 89.6788 |
| - type: mrr_at_1000 |
| value: 89.688 |
| - type: hit_rate_at_1 |
| value: 85.0 |
| - type: hit_rate_at_3 |
| value: 93.8 |
| - type: hit_rate_at_5 |
| value: 95.4 |
| - type: hit_rate_at_10 |
| value: 96.3 |
| - type: hit_rate_at_20 |
| value: 96.8 |
| - type: hit_rate_at_100 |
| value: 97.3 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/CodeSearchNetRetrieval |
| name: MTEB CodeSearchNetRetrieval (php) |
| config: php |
| split: test |
| revision: 68e8f0731a656fa4bd5b7c81936d95ad48a39bfe |
| metrics: |
| - type: ndcg_at_1 |
| value: 80.2 |
| - type: ndcg_at_3 |
| value: 86.734 |
| - type: ndcg_at_5 |
| value: 87.901 |
| - type: ndcg_at_10 |
| value: 88.363 |
| - type: ndcg_at_20 |
| value: 88.493 |
| - type: ndcg_at_100 |
| value: 88.756 |
| - type: ndcg_at_1000 |
| value: 89.074 |
| - type: map_at_1 |
| value: 80.2 |
| - type: map_at_3 |
| value: 85.183 |
| - type: map_at_5 |
| value: 85.838 |
| - type: map_at_10 |
| value: 86.035 |
| - type: map_at_20 |
| value: 86.072 |
| - type: map_at_100 |
| value: 86.109 |
| - type: map_at_1000 |
| value: 86.118 |
| - type: recall_at_1 |
| value: 80.2 |
| - type: recall_at_3 |
| value: 91.2 |
| - type: recall_at_5 |
| value: 94.0 |
| - type: recall_at_10 |
| value: 95.4 |
| - type: recall_at_20 |
| value: 95.9 |
| - type: recall_at_100 |
| value: 97.3 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 80.2 |
| - type: precision_at_1 |
| value: 80.2 |
| - type: precision_at_3 |
| value: 30.4 |
| - type: precision_at_5 |
| value: 18.8 |
| - type: precision_at_10 |
| value: 9.54 |
| - type: precision_at_20 |
| value: 4.795 |
| - type: precision_at_100 |
| value: 0.973 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 80.2 |
| - type: mrr_at_3 |
| value: 85.1833 |
| - type: mrr_at_5 |
| value: 85.8383 |
| - type: mrr_at_10 |
| value: 86.0352 |
| - type: mrr_at_20 |
| value: 86.0725 |
| - type: mrr_at_100 |
| value: 86.1093 |
| - type: mrr_at_1000 |
| value: 86.1176 |
| - type: hit_rate_at_1 |
| value: 80.2 |
| - type: hit_rate_at_3 |
| value: 91.2 |
| - type: hit_rate_at_5 |
| value: 94.0 |
| - type: hit_rate_at_10 |
| value: 95.4 |
| - type: hit_rate_at_20 |
| value: 95.9 |
| - type: hit_rate_at_100 |
| value: 97.3 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/codetrans-contest |
| name: MTEB CodeTransOceanContest |
| config: default |
| split: test |
| revision: 20da4eb20a4b17300c0986ee148c90867a7f2a4d |
| metrics: |
| - type: ndcg_at_1 |
| value: 81.9 |
| - type: ndcg_at_3 |
| value: 86.35 |
| - type: ndcg_at_5 |
| value: 87.46 |
| - type: ndcg_at_10 |
| value: 88.075 |
| - type: ndcg_at_20 |
| value: 88.304 |
| - type: ndcg_at_100 |
| value: 89.034 |
| - type: ndcg_at_1000 |
| value: 89.215 |
| - type: map_at_1 |
| value: 81.9 |
| - type: map_at_3 |
| value: 85.37 |
| - type: map_at_5 |
| value: 85.98 |
| - type: map_at_10 |
| value: 86.252 |
| - type: map_at_20 |
| value: 86.315 |
| - type: map_at_100 |
| value: 86.405 |
| - type: map_at_1000 |
| value: 86.413 |
| - type: recall_at_1 |
| value: 81.9 |
| - type: recall_at_3 |
| value: 89.14 |
| - type: recall_at_5 |
| value: 91.855 |
| - type: recall_at_10 |
| value: 93.665 |
| - type: recall_at_20 |
| value: 94.57 |
| - type: recall_at_100 |
| value: 98.643 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 81.9 |
| - type: precision_at_1 |
| value: 81.9 |
| - type: precision_at_3 |
| value: 29.713 |
| - type: precision_at_5 |
| value: 18.371 |
| - type: precision_at_10 |
| value: 9.367 |
| - type: precision_at_20 |
| value: 4.729 |
| - type: precision_at_100 |
| value: 0.986 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 81.9005 |
| - type: mrr_at_3 |
| value: 85.3695 |
| - type: mrr_at_5 |
| value: 85.9804 |
| - type: mrr_at_10 |
| value: 86.2524 |
| - type: mrr_at_20 |
| value: 86.3149 |
| - type: mrr_at_100 |
| value: 86.405 |
| - type: mrr_at_1000 |
| value: 86.4131 |
| - type: hit_rate_at_1 |
| value: 81.9 |
| - type: hit_rate_at_3 |
| value: 89.14 |
| - type: hit_rate_at_5 |
| value: 91.855 |
| - type: hit_rate_at_10 |
| value: 93.665 |
| - type: hit_rate_at_20 |
| value: 94.57 |
| - type: hit_rate_at_100 |
| value: 98.643 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/codetrans-dl |
| name: MTEB CodeTransOceanDL |
| config: default |
| split: test |
| revision: 281562cb8a1265ab5c0824bfa6ddcd9b0a15618f |
| metrics: |
| - type: ndcg_at_1 |
| value: 10.556 |
| - type: ndcg_at_3 |
| value: 14.676 |
| - type: ndcg_at_5 |
| value: 21.996 |
| - type: ndcg_at_10 |
| value: 37.234 |
| - type: ndcg_at_20 |
| value: 41.806 |
| - type: ndcg_at_100 |
| value: 41.806 |
| - type: ndcg_at_1000 |
| value: 41.806 |
| - type: map_at_1 |
| value: 10.556 |
| - type: map_at_3 |
| value: 13.611 |
| - type: map_at_5 |
| value: 17.806 |
| - type: map_at_10 |
| value: 23.863 |
| - type: map_at_20 |
| value: 25.311 |
| - type: map_at_100 |
| value: 25.311 |
| - type: map_at_1000 |
| value: 25.311 |
| - type: recall_at_1 |
| value: 10.556 |
| - type: recall_at_3 |
| value: 17.778 |
| - type: recall_at_5 |
| value: 35.0 |
| - type: recall_at_10 |
| value: 83.333 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 10.556 |
| - type: precision_at_1 |
| value: 10.556 |
| - type: precision_at_3 |
| value: 5.926 |
| - type: precision_at_5 |
| value: 7.0 |
| - type: precision_at_10 |
| value: 8.333 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 10.5556 |
| - type: mrr_at_3 |
| value: 13.6111 |
| - type: mrr_at_5 |
| value: 17.8056 |
| - type: mrr_at_10 |
| value: 23.8629 |
| - type: mrr_at_20 |
| value: 25.3107 |
| - type: mrr_at_100 |
| value: 25.3107 |
| - type: mrr_at_1000 |
| value: 25.3107 |
| - type: hit_rate_at_1 |
| value: 10.556 |
| - type: hit_rate_at_3 |
| value: 17.778 |
| - type: hit_rate_at_5 |
| value: 35.0 |
| - type: hit_rate_at_10 |
| value: 83.333 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/cosqa |
| name: MTEB CosQA |
| config: default |
| split: test |
| revision: bc5efb7e9d437246ce393ed19d772e08e4a79535 |
| metrics: |
| - type: ndcg_at_1 |
| value: 15.6 |
| - type: ndcg_at_3 |
| value: 24.243 |
| - type: ndcg_at_5 |
| value: 31.015 |
| - type: ndcg_at_10 |
| value: 36.294 |
| - type: ndcg_at_20 |
| value: 39.259 |
| - type: ndcg_at_100 |
| value: 42.682 |
| - type: ndcg_at_1000 |
| value: 43.45 |
| - type: map_at_1 |
| value: 15.6 |
| - type: map_at_3 |
| value: 21.933 |
| - type: map_at_5 |
| value: 25.653 |
| - type: map_at_10 |
| value: 27.856 |
| - type: map_at_20 |
| value: 28.687 |
| - type: map_at_100 |
| value: 29.18 |
| - type: map_at_1000 |
| value: 29.211 |
| - type: recall_at_1 |
| value: 15.6 |
| - type: recall_at_3 |
| value: 31.0 |
| - type: recall_at_5 |
| value: 47.6 |
| - type: recall_at_10 |
| value: 63.8 |
| - type: recall_at_20 |
| value: 75.4 |
| - type: recall_at_100 |
| value: 93.6 |
| - type: recall_at_1000 |
| value: 99.6 |
| - type: accuracy |
| value: 15.6 |
| - type: precision_at_1 |
| value: 15.6 |
| - type: precision_at_3 |
| value: 10.333 |
| - type: precision_at_5 |
| value: 9.52 |
| - type: precision_at_10 |
| value: 6.38 |
| - type: precision_at_20 |
| value: 3.77 |
| - type: precision_at_100 |
| value: 0.936 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 15.6 |
| - type: mrr_at_3 |
| value: 21.9333 |
| - type: mrr_at_5 |
| value: 25.6533 |
| - type: mrr_at_10 |
| value: 27.8563 |
| - type: mrr_at_20 |
| value: 28.6874 |
| - type: mrr_at_100 |
| value: 29.1803 |
| - type: mrr_at_1000 |
| value: 29.2108 |
| - type: hit_rate_at_1 |
| value: 15.6 |
| - type: hit_rate_at_3 |
| value: 31.0 |
| - type: hit_rate_at_5 |
| value: 47.6 |
| - type: hit_rate_at_10 |
| value: 63.8 |
| - type: hit_rate_at_20 |
| value: 75.4 |
| - type: hit_rate_at_100 |
| value: 93.6 |
| - type: hit_rate_at_1000 |
| value: 99.6 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/FEVER_test_top_250_only_w_correct-v2 |
| name: MTEB FEVERHardNegatives |
| config: default |
| split: test |
| revision: 080c9ed6267b65029207906e815d44a9240bafca |
| metrics: |
| - type: ndcg_at_1 |
| value: 82.2 |
| - type: ndcg_at_3 |
| value: 87.644 |
| - type: ndcg_at_5 |
| value: 88.436 |
| - type: ndcg_at_10 |
| value: 89.08 |
| - type: ndcg_at_20 |
| value: 89.449 |
| - type: ndcg_at_100 |
| value: 89.824 |
| - type: ndcg_at_1000 |
| value: 89.983 |
| - type: map_at_1 |
| value: 77.021 |
| - type: map_at_3 |
| value: 84.585 |
| - type: map_at_5 |
| value: 85.2 |
| - type: map_at_10 |
| value: 85.57 |
| - type: map_at_20 |
| value: 85.722 |
| - type: map_at_100 |
| value: 85.81 |
| - type: map_at_1000 |
| value: 85.823 |
| - type: recall_at_1 |
| value: 77.021 |
| - type: recall_at_3 |
| value: 91.888 |
| - type: recall_at_5 |
| value: 93.961 |
| - type: recall_at_10 |
| value: 95.756 |
| - type: recall_at_20 |
| value: 97.013 |
| - type: recall_at_100 |
| value: 98.498 |
| - type: recall_at_1000 |
| value: 99.411 |
| - type: accuracy |
| value: 77.021 |
| - type: precision_at_1 |
| value: 82.2 |
| - type: precision_at_3 |
| value: 32.967 |
| - type: precision_at_5 |
| value: 20.38 |
| - type: precision_at_10 |
| value: 10.51 |
| - type: precision_at_20 |
| value: 5.38 |
| - type: precision_at_100 |
| value: 1.119 |
| - type: precision_at_1000 |
| value: 0.115 |
| - type: mrr_at_1 |
| value: 82.2 |
| - type: mrr_at_3 |
| value: 89.05 |
| - type: mrr_at_5 |
| value: 89.455 |
| - type: mrr_at_10 |
| value: 89.5987 |
| - type: mrr_at_20 |
| value: 89.6337 |
| - type: mrr_at_100 |
| value: 89.6381 |
| - type: mrr_at_1000 |
| value: 89.6382 |
| - type: hit_rate_at_1 |
| value: 82.2 |
| - type: hit_rate_at_3 |
| value: 96.6 |
| - type: hit_rate_at_5 |
| value: 98.3 |
| - type: hit_rate_at_10 |
| value: 99.3 |
| - type: hit_rate_at_20 |
| value: 99.8 |
| - type: hit_rate_at_100 |
| value: 99.9 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/fiqa |
| name: MTEB FiQA2018 |
| config: default |
| split: test |
| revision: 27a168819829fe9bcd655c2df245fb19452e8e06 |
| metrics: |
| - type: ndcg_at_1 |
| value: 51.08 |
| - type: ndcg_at_3 |
| value: 47.747 |
| - type: ndcg_at_5 |
| value: 49.0 |
| - type: ndcg_at_10 |
| value: 51.825 |
| - type: ndcg_at_20 |
| value: 54.219 |
| - type: ndcg_at_100 |
| value: 57.719 |
| - type: ndcg_at_1000 |
| value: 60.192 |
| - type: map_at_1 |
| value: 26.119 |
| - type: map_at_3 |
| value: 37.835 |
| - type: map_at_5 |
| value: 40.826 |
| - type: map_at_10 |
| value: 43.202 |
| - type: map_at_20 |
| value: 44.276 |
| - type: map_at_100 |
| value: 45.088 |
| - type: map_at_1000 |
| value: 45.257 |
| - type: recall_at_1 |
| value: 26.119 |
| - type: recall_at_3 |
| value: 43.055 |
| - type: recall_at_5 |
| value: 50.219 |
| - type: recall_at_10 |
| value: 59.662 |
| - type: recall_at_20 |
| value: 66.739 |
| - type: recall_at_100 |
| value: 80.684 |
| - type: recall_at_1000 |
| value: 95.557 |
| - type: accuracy |
| value: 26.119 |
| - type: precision_at_1 |
| value: 51.08 |
| - type: precision_at_3 |
| value: 32.047 |
| - type: precision_at_5 |
| value: 23.519 |
| - type: precision_at_10 |
| value: 14.336 |
| - type: precision_at_20 |
| value: 8.287 |
| - type: precision_at_100 |
| value: 2.086 |
| - type: precision_at_1000 |
| value: 0.25 |
| - type: mrr_at_1 |
| value: 51.0802 |
| - type: mrr_at_3 |
| value: 57.4074 |
| - type: mrr_at_5 |
| value: 58.8889 |
| - type: mrr_at_10 |
| value: 59.8483 |
| - type: mrr_at_20 |
| value: 60.1598 |
| - type: mrr_at_100 |
| value: 60.3469 |
| - type: mrr_at_1000 |
| value: 60.3836 |
| - type: hit_rate_at_1 |
| value: 51.08 |
| - type: hit_rate_at_3 |
| value: 65.278 |
| - type: hit_rate_at_5 |
| value: 71.605 |
| - type: hit_rate_at_10 |
| value: 79.012 |
| - type: hit_rate_at_20 |
| value: 83.796 |
| - type: hit_rate_at_100 |
| value: 91.667 |
| - type: hit_rate_at_1000 |
| value: 98.765 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/HellaSwag |
| name: MTEB HellaSwag |
| config: default |
| split: test |
| revision: 944be88c186b87495f4b1f5e4a416a3a7ea5d680 |
| metrics: |
| - type: ndcg_at_1 |
| value: 18.771 |
| - type: ndcg_at_3 |
| value: 28.551 |
| - type: ndcg_at_5 |
| value: 31.509 |
| - type: ndcg_at_10 |
| value: 33.91 |
| - type: ndcg_at_20 |
| value: 35.526 |
| - type: ndcg_at_100 |
| value: 38.2 |
| - type: ndcg_at_1000 |
| value: 40.169 |
| - type: map_at_1 |
| value: 18.771 |
| - type: map_at_3 |
| value: 26.137 |
| - type: map_at_5 |
| value: 27.784 |
| - type: map_at_10 |
| value: 28.775 |
| - type: map_at_20 |
| value: 29.219 |
| - type: map_at_100 |
| value: 29.581 |
| - type: map_at_1000 |
| value: 29.649 |
| - type: recall_at_1 |
| value: 18.771 |
| - type: recall_at_3 |
| value: 35.541 |
| - type: recall_at_5 |
| value: 42.701 |
| - type: recall_at_10 |
| value: 50.119 |
| - type: recall_at_20 |
| value: 56.513 |
| - type: recall_at_100 |
| value: 71.032 |
| - type: recall_at_1000 |
| value: 86.885 |
| - type: accuracy |
| value: 18.771 |
| - type: precision_at_1 |
| value: 18.771 |
| - type: precision_at_3 |
| value: 11.847 |
| - type: precision_at_5 |
| value: 8.54 |
| - type: precision_at_10 |
| value: 5.012 |
| - type: precision_at_20 |
| value: 2.826 |
| - type: precision_at_100 |
| value: 0.71 |
| - type: precision_at_1000 |
| value: 0.087 |
| - type: mrr_at_1 |
| value: 18.7712 |
| - type: mrr_at_3 |
| value: 26.1369 |
| - type: mrr_at_5 |
| value: 27.784 |
| - type: mrr_at_10 |
| value: 28.7753 |
| - type: mrr_at_20 |
| value: 29.2188 |
| - type: mrr_at_100 |
| value: 29.5812 |
| - type: mrr_at_1000 |
| value: 29.6493 |
| - type: hit_rate_at_1 |
| value: 18.771 |
| - type: hit_rate_at_3 |
| value: 35.541 |
| - type: hit_rate_at_5 |
| value: 42.701 |
| - type: hit_rate_at_10 |
| value: 50.119 |
| - type: hit_rate_at_20 |
| value: 56.513 |
| - type: hit_rate_at_100 |
| value: 71.032 |
| - type: hit_rate_at_1000 |
| value: 86.885 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/HotpotQA_test_top_250_only_w_correct-v2 |
| name: MTEB HotpotQAHardNegatives |
| config: default |
| split: test |
| revision: 617612fa63afcb60e3b134bed8b7216a99707c37 |
| metrics: |
| - type: ndcg_at_1 |
| value: 82.9 |
| - type: ndcg_at_3 |
| value: 68.473 |
| - type: ndcg_at_5 |
| value: 71.024 |
| - type: ndcg_at_10 |
| value: 73.177 |
| - type: ndcg_at_20 |
| value: 74.715 |
| - type: ndcg_at_100 |
| value: 76.925 |
| - type: ndcg_at_1000 |
| value: 77.962 |
| - type: map_at_1 |
| value: 41.45 |
| - type: map_at_3 |
| value: 61.292 |
| - type: map_at_5 |
| value: 63.377 |
| - type: map_at_10 |
| value: 64.678 |
| - type: map_at_20 |
| value: 65.306 |
| - type: map_at_100 |
| value: 65.792 |
| - type: map_at_1000 |
| value: 65.853 |
| - type: recall_at_1 |
| value: 41.45 |
| - type: recall_at_3 |
| value: 65.55 |
| - type: recall_at_5 |
| value: 70.6 |
| - type: recall_at_10 |
| value: 76.0 |
| - type: recall_at_20 |
| value: 80.9 |
| - type: recall_at_100 |
| value: 90.6 |
| - type: recall_at_1000 |
| value: 97.3 |
| - type: accuracy |
| value: 41.45 |
| - type: precision_at_1 |
| value: 82.9 |
| - type: precision_at_3 |
| value: 43.7 |
| - type: precision_at_5 |
| value: 28.24 |
| - type: precision_at_10 |
| value: 15.2 |
| - type: precision_at_20 |
| value: 8.09 |
| - type: precision_at_100 |
| value: 1.812 |
| - type: precision_at_1000 |
| value: 0.195 |
| - type: mrr_at_1 |
| value: 82.9 |
| - type: mrr_at_3 |
| value: 87.25 |
| - type: mrr_at_5 |
| value: 87.7 |
| - type: mrr_at_10 |
| value: 88.0158 |
| - type: mrr_at_20 |
| value: 88.1707 |
| - type: mrr_at_100 |
| value: 88.1998 |
| - type: mrr_at_1000 |
| value: 88.2014 |
| - type: hit_rate_at_1 |
| value: 82.9 |
| - type: hit_rate_at_3 |
| value: 92.2 |
| - type: hit_rate_at_5 |
| value: 94.1 |
| - type: hit_rate_at_10 |
| value: 96.3 |
| - type: hit_rate_at_20 |
| value: 98.5 |
| - type: hit_rate_at_100 |
| value: 99.7 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/imdb |
| name: MTEB ImdbClassification |
| config: default |
| split: test |
| revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 |
| metrics: |
| - type: accuracy |
| value: 94.028 |
| - type: f1 |
| value: 94.0244 |
| - type: f1_weighted |
| value: 94.0244 |
| - type: precision |
| value: 94.1259 |
| - type: precision_weighted |
| value: 94.1259 |
| - type: recall |
| value: 94.028 |
| - type: recall_weighted |
| value: 94.028 |
| - type: ap |
| value: 91.3763 |
| - type: ap_weighted |
| value: 91.3763 |
| - task: |
| type: Retrieval |
| dataset: |
| type: dwzhu/LongEmbed |
| name: MTEB LEMBNarrativeQARetrieval |
| config: default |
| split: test |
| revision: 6e346642246bfb4928c560ee08640dc84d074e8c |
| metrics: |
| - type: ndcg_at_1 |
| value: 53.555 |
| - type: ndcg_at_3 |
| value: 61.273 |
| - type: ndcg_at_5 |
| value: 62.897 |
| - type: ndcg_at_10 |
| value: 64.545 |
| - type: ndcg_at_20 |
| value: 65.786 |
| - type: ndcg_at_100 |
| value: 67.857 |
| - type: ndcg_at_1000 |
| value: 68.943 |
| - type: map_at_1 |
| value: 53.555 |
| - type: map_at_3 |
| value: 59.414 |
| - type: map_at_5 |
| value: 60.315 |
| - type: map_at_10 |
| value: 60.998 |
| - type: map_at_20 |
| value: 61.342 |
| - type: map_at_100 |
| value: 61.62 |
| - type: map_at_1000 |
| value: 61.666 |
| - type: recall_at_1 |
| value: 53.555 |
| - type: recall_at_3 |
| value: 66.638 |
| - type: recall_at_5 |
| value: 70.581 |
| - type: recall_at_10 |
| value: 75.663 |
| - type: recall_at_20 |
| value: 80.544 |
| - type: recall_at_100 |
| value: 91.837 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 53.555 |
| - type: precision_at_1 |
| value: 53.555 |
| - type: precision_at_3 |
| value: 22.213 |
| - type: precision_at_5 |
| value: 14.116 |
| - type: precision_at_10 |
| value: 7.566 |
| - type: precision_at_20 |
| value: 4.027 |
| - type: precision_at_100 |
| value: 0.918 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 53.5554 |
| - type: mrr_at_3 |
| value: 59.414 |
| - type: mrr_at_5 |
| value: 60.315 |
| - type: mrr_at_10 |
| value: 60.9979 |
| - type: mrr_at_20 |
| value: 61.3419 |
| - type: mrr_at_100 |
| value: 61.6196 |
| - type: mrr_at_1000 |
| value: 61.6661 |
| - type: hit_rate_at_1 |
| value: 53.555 |
| - type: hit_rate_at_3 |
| value: 66.638 |
| - type: hit_rate_at_5 |
| value: 70.581 |
| - type: hit_rate_at_10 |
| value: 75.663 |
| - type: hit_rate_at_20 |
| value: 80.544 |
| - type: hit_rate_at_100 |
| value: 91.837 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: dwzhu/LongEmbed |
| name: MTEB LEMBNeedleRetrieval |
| config: default |
| split: test_256 |
| revision: 6e346642246bfb4928c560ee08640dc84d074e8c |
| metrics: |
| - type: ndcg_at_1 |
| value: 84.0 |
| - type: ndcg_at_3 |
| value: 93.571 |
| - type: ndcg_at_5 |
| value: 93.571 |
| - type: ndcg_at_10 |
| value: 93.571 |
| - type: ndcg_at_20 |
| value: 93.571 |
| - type: ndcg_at_100 |
| value: 93.571 |
| - type: ndcg_at_1000 |
| value: 93.571 |
| - type: map_at_1 |
| value: 84.0 |
| - type: map_at_3 |
| value: 91.333 |
| - type: map_at_5 |
| value: 91.333 |
| - type: map_at_10 |
| value: 91.333 |
| - type: map_at_20 |
| value: 91.333 |
| - type: map_at_100 |
| value: 91.333 |
| - type: map_at_1000 |
| value: 91.333 |
| - type: recall_at_1 |
| value: 84.0 |
| - type: recall_at_3 |
| value: 100.0 |
| - type: recall_at_5 |
| value: 100.0 |
| - type: recall_at_10 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 84.0 |
| - type: precision_at_1 |
| value: 84.0 |
| - type: precision_at_3 |
| value: 33.333 |
| - type: precision_at_5 |
| value: 20.0 |
| - type: precision_at_10 |
| value: 10.0 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 84.0 |
| - type: mrr_at_3 |
| value: 91.3333 |
| - type: mrr_at_5 |
| value: 91.3333 |
| - type: mrr_at_10 |
| value: 91.3333 |
| - type: mrr_at_20 |
| value: 91.3333 |
| - type: mrr_at_100 |
| value: 91.3333 |
| - type: mrr_at_1000 |
| value: 91.3333 |
| - type: hit_rate_at_1 |
| value: 84.0 |
| - type: hit_rate_at_3 |
| value: 100.0 |
| - type: hit_rate_at_5 |
| value: 100.0 |
| - type: hit_rate_at_10 |
| value: 100.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: dwzhu/LongEmbed |
| name: MTEB LEMBNeedleRetrieval |
| config: default |
| split: test_512 |
| revision: 6e346642246bfb4928c560ee08640dc84d074e8c |
| metrics: |
| - type: ndcg_at_1 |
| value: 78.0 |
| - type: ndcg_at_3 |
| value: 87.571 |
| - type: ndcg_at_5 |
| value: 88.345 |
| - type: ndcg_at_10 |
| value: 89.77 |
| - type: ndcg_at_20 |
| value: 89.77 |
| - type: ndcg_at_100 |
| value: 89.77 |
| - type: ndcg_at_1000 |
| value: 89.77 |
| - type: map_at_1 |
| value: 78.0 |
| - type: map_at_3 |
| value: 85.333 |
| - type: map_at_5 |
| value: 85.733 |
| - type: map_at_10 |
| value: 86.4 |
| - type: map_at_20 |
| value: 86.4 |
| - type: map_at_100 |
| value: 86.4 |
| - type: map_at_1000 |
| value: 86.4 |
| - type: recall_at_1 |
| value: 78.0 |
| - type: recall_at_3 |
| value: 94.0 |
| - type: recall_at_5 |
| value: 96.0 |
| - type: recall_at_10 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 78.0 |
| - type: precision_at_1 |
| value: 78.0 |
| - type: precision_at_3 |
| value: 31.333 |
| - type: precision_at_5 |
| value: 19.2 |
| - type: precision_at_10 |
| value: 10.0 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 78.0 |
| - type: mrr_at_3 |
| value: 85.3333 |
| - type: mrr_at_5 |
| value: 85.7333 |
| - type: mrr_at_10 |
| value: 86.4 |
| - type: mrr_at_20 |
| value: 86.4 |
| - type: mrr_at_100 |
| value: 86.4 |
| - type: mrr_at_1000 |
| value: 86.4 |
| - type: hit_rate_at_1 |
| value: 78.0 |
| - type: hit_rate_at_3 |
| value: 94.0 |
| - type: hit_rate_at_5 |
| value: 96.0 |
| - type: hit_rate_at_10 |
| value: 100.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: dwzhu/LongEmbed |
| name: MTEB LEMBNeedleRetrieval |
| config: default |
| split: test_1024 |
| revision: 6e346642246bfb4928c560ee08640dc84d074e8c |
| metrics: |
| - type: ndcg_at_1 |
| value: 78.0 |
| - type: ndcg_at_3 |
| value: 87.833 |
| - type: ndcg_at_5 |
| value: 89.468 |
| - type: ndcg_at_10 |
| value: 90.099 |
| - type: ndcg_at_20 |
| value: 90.099 |
| - type: ndcg_at_100 |
| value: 90.099 |
| - type: ndcg_at_1000 |
| value: 90.099 |
| - type: map_at_1 |
| value: 78.0 |
| - type: map_at_3 |
| value: 85.667 |
| - type: map_at_5 |
| value: 86.567 |
| - type: map_at_10 |
| value: 86.817 |
| - type: map_at_20 |
| value: 86.817 |
| - type: map_at_100 |
| value: 86.817 |
| - type: map_at_1000 |
| value: 86.817 |
| - type: recall_at_1 |
| value: 78.0 |
| - type: recall_at_3 |
| value: 94.0 |
| - type: recall_at_5 |
| value: 98.0 |
| - type: recall_at_10 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 78.0 |
| - type: precision_at_1 |
| value: 78.0 |
| - type: precision_at_3 |
| value: 31.333 |
| - type: precision_at_5 |
| value: 19.6 |
| - type: precision_at_10 |
| value: 10.0 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 78.0 |
| - type: mrr_at_3 |
| value: 85.6667 |
| - type: mrr_at_5 |
| value: 86.5667 |
| - type: mrr_at_10 |
| value: 86.8167 |
| - type: mrr_at_20 |
| value: 86.8167 |
| - type: mrr_at_100 |
| value: 86.8167 |
| - type: mrr_at_1000 |
| value: 86.8167 |
| - type: hit_rate_at_1 |
| value: 78.0 |
| - type: hit_rate_at_3 |
| value: 94.0 |
| - type: hit_rate_at_5 |
| value: 98.0 |
| - type: hit_rate_at_10 |
| value: 100.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: dwzhu/LongEmbed |
| name: MTEB LEMBNeedleRetrieval |
| config: default |
| split: test_2048 |
| revision: 6e346642246bfb4928c560ee08640dc84d074e8c |
| metrics: |
| - type: ndcg_at_1 |
| value: 68.0 |
| - type: ndcg_at_3 |
| value: 84.357 |
| - type: ndcg_at_5 |
| value: 85.13 |
| - type: ndcg_at_10 |
| value: 85.843 |
| - type: ndcg_at_20 |
| value: 85.843 |
| - type: ndcg_at_100 |
| value: 85.843 |
| - type: ndcg_at_1000 |
| value: 85.843 |
| - type: map_at_1 |
| value: 68.0 |
| - type: map_at_3 |
| value: 80.333 |
| - type: map_at_5 |
| value: 80.733 |
| - type: map_at_10 |
| value: 81.067 |
| - type: map_at_20 |
| value: 81.067 |
| - type: map_at_100 |
| value: 81.067 |
| - type: map_at_1000 |
| value: 81.067 |
| - type: recall_at_1 |
| value: 68.0 |
| - type: recall_at_3 |
| value: 96.0 |
| - type: recall_at_5 |
| value: 98.0 |
| - type: recall_at_10 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 68.0 |
| - type: precision_at_1 |
| value: 68.0 |
| - type: precision_at_3 |
| value: 32.0 |
| - type: precision_at_5 |
| value: 19.6 |
| - type: precision_at_10 |
| value: 10.0 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 68.0 |
| - type: mrr_at_3 |
| value: 80.3333 |
| - type: mrr_at_5 |
| value: 80.7333 |
| - type: mrr_at_10 |
| value: 81.0667 |
| - type: mrr_at_20 |
| value: 81.0667 |
| - type: mrr_at_100 |
| value: 81.0667 |
| - type: mrr_at_1000 |
| value: 81.0667 |
| - type: hit_rate_at_1 |
| value: 68.0 |
| - type: hit_rate_at_3 |
| value: 96.0 |
| - type: hit_rate_at_5 |
| value: 98.0 |
| - type: hit_rate_at_10 |
| value: 100.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: dwzhu/LongEmbed |
| name: MTEB LEMBNeedleRetrieval |
| config: default |
| split: test_4096 |
| revision: 6e346642246bfb4928c560ee08640dc84d074e8c |
| metrics: |
| - type: ndcg_at_1 |
| value: 76.0 |
| - type: ndcg_at_3 |
| value: 85.833 |
| - type: ndcg_at_5 |
| value: 88.242 |
| - type: ndcg_at_10 |
| value: 88.242 |
| - type: ndcg_at_20 |
| value: 88.742 |
| - type: ndcg_at_100 |
| value: 88.742 |
| - type: ndcg_at_1000 |
| value: 88.742 |
| - type: map_at_1 |
| value: 76.0 |
| - type: map_at_3 |
| value: 83.667 |
| - type: map_at_5 |
| value: 84.967 |
| - type: map_at_10 |
| value: 84.967 |
| - type: map_at_20 |
| value: 85.1 |
| - type: map_at_100 |
| value: 85.1 |
| - type: map_at_1000 |
| value: 85.1 |
| - type: recall_at_1 |
| value: 76.0 |
| - type: recall_at_3 |
| value: 92.0 |
| - type: recall_at_5 |
| value: 98.0 |
| - type: recall_at_10 |
| value: 98.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 76.0 |
| - type: precision_at_1 |
| value: 76.0 |
| - type: precision_at_3 |
| value: 30.667 |
| - type: precision_at_5 |
| value: 19.6 |
| - type: precision_at_10 |
| value: 9.8 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 76.0 |
| - type: mrr_at_3 |
| value: 83.6667 |
| - type: mrr_at_5 |
| value: 84.9667 |
| - type: mrr_at_10 |
| value: 84.9667 |
| - type: mrr_at_20 |
| value: 85.1 |
| - type: mrr_at_100 |
| value: 85.1 |
| - type: mrr_at_1000 |
| value: 85.1 |
| - type: hit_rate_at_1 |
| value: 76.0 |
| - type: hit_rate_at_3 |
| value: 92.0 |
| - type: hit_rate_at_5 |
| value: 98.0 |
| - type: hit_rate_at_10 |
| value: 98.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: dwzhu/LongEmbed |
| name: MTEB LEMBNeedleRetrieval |
| config: default |
| split: test_8192 |
| revision: 6e346642246bfb4928c560ee08640dc84d074e8c |
| metrics: |
| - type: ndcg_at_1 |
| value: 62.0 |
| - type: ndcg_at_3 |
| value: 74.357 |
| - type: ndcg_at_5 |
| value: 76.079 |
| - type: ndcg_at_10 |
| value: 80.018 |
| - type: ndcg_at_20 |
| value: 80.497 |
| - type: ndcg_at_100 |
| value: 80.497 |
| - type: ndcg_at_1000 |
| value: 80.497 |
| - type: map_at_1 |
| value: 62.0 |
| - type: map_at_3 |
| value: 71.667 |
| - type: map_at_5 |
| value: 72.667 |
| - type: map_at_10 |
| value: 74.327 |
| - type: map_at_20 |
| value: 74.445 |
| - type: map_at_100 |
| value: 74.445 |
| - type: map_at_1000 |
| value: 74.445 |
| - type: recall_at_1 |
| value: 62.0 |
| - type: recall_at_3 |
| value: 82.0 |
| - type: recall_at_5 |
| value: 86.0 |
| - type: recall_at_10 |
| value: 98.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 62.0 |
| - type: precision_at_1 |
| value: 62.0 |
| - type: precision_at_3 |
| value: 27.333 |
| - type: precision_at_5 |
| value: 17.2 |
| - type: precision_at_10 |
| value: 9.8 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 62.0 |
| - type: mrr_at_3 |
| value: 71.6667 |
| - type: mrr_at_5 |
| value: 72.6667 |
| - type: mrr_at_10 |
| value: 74.327 |
| - type: mrr_at_20 |
| value: 74.4446 |
| - type: mrr_at_100 |
| value: 74.4446 |
| - type: mrr_at_1000 |
| value: 74.4446 |
| - type: hit_rate_at_1 |
| value: 62.0 |
| - type: hit_rate_at_3 |
| value: 82.0 |
| - type: hit_rate_at_5 |
| value: 86.0 |
| - type: hit_rate_at_10 |
| value: 98.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: dwzhu/LongEmbed |
| name: MTEB LEMBNeedleRetrieval |
| config: default |
| split: test_16384 |
| revision: 6e346642246bfb4928c560ee08640dc84d074e8c |
| metrics: |
| - type: ndcg_at_1 |
| value: 54.0 |
| - type: ndcg_at_3 |
| value: 67.095 |
| - type: ndcg_at_5 |
| value: 72.861 |
| - type: ndcg_at_10 |
| value: 74.159 |
| - type: ndcg_at_20 |
| value: 75.722 |
| - type: ndcg_at_100 |
| value: 75.722 |
| - type: ndcg_at_1000 |
| value: 75.722 |
| - type: map_at_1 |
| value: 54.0 |
| - type: map_at_3 |
| value: 64.0 |
| - type: map_at_5 |
| value: 67.2 |
| - type: map_at_10 |
| value: 67.736 |
| - type: map_at_20 |
| value: 68.186 |
| - type: map_at_100 |
| value: 68.186 |
| - type: map_at_1000 |
| value: 68.186 |
| - type: recall_at_1 |
| value: 54.0 |
| - type: recall_at_3 |
| value: 76.0 |
| - type: recall_at_5 |
| value: 90.0 |
| - type: recall_at_10 |
| value: 94.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 54.0 |
| - type: precision_at_1 |
| value: 54.0 |
| - type: precision_at_3 |
| value: 25.333 |
| - type: precision_at_5 |
| value: 18.0 |
| - type: precision_at_10 |
| value: 9.4 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 54.0 |
| - type: mrr_at_3 |
| value: 64.0 |
| - type: mrr_at_5 |
| value: 67.2 |
| - type: mrr_at_10 |
| value: 67.7357 |
| - type: mrr_at_20 |
| value: 68.1863 |
| - type: mrr_at_100 |
| value: 68.1863 |
| - type: mrr_at_1000 |
| value: 68.1863 |
| - type: hit_rate_at_1 |
| value: 54.0 |
| - type: hit_rate_at_3 |
| value: 76.0 |
| - type: hit_rate_at_5 |
| value: 90.0 |
| - type: hit_rate_at_10 |
| value: 94.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: dwzhu/LongEmbed |
| name: MTEB LEMBNeedleRetrieval |
| config: default |
| split: test_32768 |
| revision: 6e346642246bfb4928c560ee08640dc84d074e8c |
| metrics: |
| - type: ndcg_at_1 |
| value: 32.0 |
| - type: ndcg_at_3 |
| value: 53.404 |
| - type: ndcg_at_5 |
| value: 58.222 |
| - type: ndcg_at_10 |
| value: 60.934 |
| - type: ndcg_at_20 |
| value: 63.568 |
| - type: ndcg_at_100 |
| value: 63.949 |
| - type: ndcg_at_1000 |
| value: 63.949 |
| - type: map_at_1 |
| value: 32.0 |
| - type: map_at_3 |
| value: 48.333 |
| - type: map_at_5 |
| value: 50.933 |
| - type: map_at_10 |
| value: 52.124 |
| - type: map_at_20 |
| value: 52.9 |
| - type: map_at_100 |
| value: 52.955 |
| - type: map_at_1000 |
| value: 52.955 |
| - type: recall_at_1 |
| value: 32.0 |
| - type: recall_at_3 |
| value: 68.0 |
| - type: recall_at_5 |
| value: 80.0 |
| - type: recall_at_10 |
| value: 88.0 |
| - type: recall_at_20 |
| value: 98.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 32.0 |
| - type: precision_at_1 |
| value: 32.0 |
| - type: precision_at_3 |
| value: 22.667 |
| - type: precision_at_5 |
| value: 16.0 |
| - type: precision_at_10 |
| value: 8.8 |
| - type: precision_at_20 |
| value: 4.9 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 32.0 |
| - type: mrr_at_3 |
| value: 48.3333 |
| - type: mrr_at_5 |
| value: 50.9333 |
| - type: mrr_at_10 |
| value: 52.1238 |
| - type: mrr_at_20 |
| value: 52.9005 |
| - type: mrr_at_100 |
| value: 52.9546 |
| - type: mrr_at_1000 |
| value: 52.9546 |
| - type: hit_rate_at_1 |
| value: 32.0 |
| - type: hit_rate_at_3 |
| value: 68.0 |
| - type: hit_rate_at_5 |
| value: 80.0 |
| - type: hit_rate_at_10 |
| value: 88.0 |
| - type: hit_rate_at_20 |
| value: 98.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/LEMBPasskeyRetrieval |
| name: MTEB LEMBPasskeyRetrieval |
| config: default |
| split: test_256 |
| revision: 86ac5a6437198a94ecd666440c78b3e78f258274 |
| metrics: |
| - type: ndcg_at_1 |
| value: 100.0 |
| - type: ndcg_at_3 |
| value: 100.0 |
| - type: ndcg_at_5 |
| value: 100.0 |
| - type: ndcg_at_10 |
| value: 100.0 |
| - type: ndcg_at_20 |
| value: 100.0 |
| - type: ndcg_at_100 |
| value: 100.0 |
| - type: ndcg_at_1000 |
| value: 100.0 |
| - type: map_at_1 |
| value: 100.0 |
| - type: map_at_3 |
| value: 100.0 |
| - type: map_at_5 |
| value: 100.0 |
| - type: map_at_10 |
| value: 100.0 |
| - type: map_at_20 |
| value: 100.0 |
| - type: map_at_100 |
| value: 100.0 |
| - type: map_at_1000 |
| value: 100.0 |
| - type: recall_at_1 |
| value: 100.0 |
| - type: recall_at_3 |
| value: 100.0 |
| - type: recall_at_5 |
| value: 100.0 |
| - type: recall_at_10 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 100.0 |
| - type: precision_at_1 |
| value: 100.0 |
| - type: precision_at_3 |
| value: 33.333 |
| - type: precision_at_5 |
| value: 20.0 |
| - type: precision_at_10 |
| value: 10.0 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 100.0 |
| - type: mrr_at_3 |
| value: 100.0 |
| - type: mrr_at_5 |
| value: 100.0 |
| - type: mrr_at_10 |
| value: 100.0 |
| - type: mrr_at_20 |
| value: 100.0 |
| - type: mrr_at_100 |
| value: 100.0 |
| - type: mrr_at_1000 |
| value: 100.0 |
| - type: hit_rate_at_1 |
| value: 100.0 |
| - type: hit_rate_at_3 |
| value: 100.0 |
| - type: hit_rate_at_5 |
| value: 100.0 |
| - type: hit_rate_at_10 |
| value: 100.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/LEMBPasskeyRetrieval |
| name: MTEB LEMBPasskeyRetrieval |
| config: default |
| split: test_512 |
| revision: 86ac5a6437198a94ecd666440c78b3e78f258274 |
| metrics: |
| - type: ndcg_at_1 |
| value: 100.0 |
| - type: ndcg_at_3 |
| value: 100.0 |
| - type: ndcg_at_5 |
| value: 100.0 |
| - type: ndcg_at_10 |
| value: 100.0 |
| - type: ndcg_at_20 |
| value: 100.0 |
| - type: ndcg_at_100 |
| value: 100.0 |
| - type: ndcg_at_1000 |
| value: 100.0 |
| - type: map_at_1 |
| value: 100.0 |
| - type: map_at_3 |
| value: 100.0 |
| - type: map_at_5 |
| value: 100.0 |
| - type: map_at_10 |
| value: 100.0 |
| - type: map_at_20 |
| value: 100.0 |
| - type: map_at_100 |
| value: 100.0 |
| - type: map_at_1000 |
| value: 100.0 |
| - type: recall_at_1 |
| value: 100.0 |
| - type: recall_at_3 |
| value: 100.0 |
| - type: recall_at_5 |
| value: 100.0 |
| - type: recall_at_10 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 100.0 |
| - type: precision_at_1 |
| value: 100.0 |
| - type: precision_at_3 |
| value: 33.333 |
| - type: precision_at_5 |
| value: 20.0 |
| - type: precision_at_10 |
| value: 10.0 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 100.0 |
| - type: mrr_at_3 |
| value: 100.0 |
| - type: mrr_at_5 |
| value: 100.0 |
| - type: mrr_at_10 |
| value: 100.0 |
| - type: mrr_at_20 |
| value: 100.0 |
| - type: mrr_at_100 |
| value: 100.0 |
| - type: mrr_at_1000 |
| value: 100.0 |
| - type: hit_rate_at_1 |
| value: 100.0 |
| - type: hit_rate_at_3 |
| value: 100.0 |
| - type: hit_rate_at_5 |
| value: 100.0 |
| - type: hit_rate_at_10 |
| value: 100.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/LEMBPasskeyRetrieval |
| name: MTEB LEMBPasskeyRetrieval |
| config: default |
| split: test_1024 |
| revision: 86ac5a6437198a94ecd666440c78b3e78f258274 |
| metrics: |
| - type: ndcg_at_1 |
| value: 100.0 |
| - type: ndcg_at_3 |
| value: 100.0 |
| - type: ndcg_at_5 |
| value: 100.0 |
| - type: ndcg_at_10 |
| value: 100.0 |
| - type: ndcg_at_20 |
| value: 100.0 |
| - type: ndcg_at_100 |
| value: 100.0 |
| - type: ndcg_at_1000 |
| value: 100.0 |
| - type: map_at_1 |
| value: 100.0 |
| - type: map_at_3 |
| value: 100.0 |
| - type: map_at_5 |
| value: 100.0 |
| - type: map_at_10 |
| value: 100.0 |
| - type: map_at_20 |
| value: 100.0 |
| - type: map_at_100 |
| value: 100.0 |
| - type: map_at_1000 |
| value: 100.0 |
| - type: recall_at_1 |
| value: 100.0 |
| - type: recall_at_3 |
| value: 100.0 |
| - type: recall_at_5 |
| value: 100.0 |
| - type: recall_at_10 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 100.0 |
| - type: precision_at_1 |
| value: 100.0 |
| - type: precision_at_3 |
| value: 33.333 |
| - type: precision_at_5 |
| value: 20.0 |
| - type: precision_at_10 |
| value: 10.0 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 100.0 |
| - type: mrr_at_3 |
| value: 100.0 |
| - type: mrr_at_5 |
| value: 100.0 |
| - type: mrr_at_10 |
| value: 100.0 |
| - type: mrr_at_20 |
| value: 100.0 |
| - type: mrr_at_100 |
| value: 100.0 |
| - type: mrr_at_1000 |
| value: 100.0 |
| - type: hit_rate_at_1 |
| value: 100.0 |
| - type: hit_rate_at_3 |
| value: 100.0 |
| - type: hit_rate_at_5 |
| value: 100.0 |
| - type: hit_rate_at_10 |
| value: 100.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/LEMBPasskeyRetrieval |
| name: MTEB LEMBPasskeyRetrieval |
| config: default |
| split: test_2048 |
| revision: 86ac5a6437198a94ecd666440c78b3e78f258274 |
| metrics: |
| - type: ndcg_at_1 |
| value: 100.0 |
| - type: ndcg_at_3 |
| value: 100.0 |
| - type: ndcg_at_5 |
| value: 100.0 |
| - type: ndcg_at_10 |
| value: 100.0 |
| - type: ndcg_at_20 |
| value: 100.0 |
| - type: ndcg_at_100 |
| value: 100.0 |
| - type: ndcg_at_1000 |
| value: 100.0 |
| - type: map_at_1 |
| value: 100.0 |
| - type: map_at_3 |
| value: 100.0 |
| - type: map_at_5 |
| value: 100.0 |
| - type: map_at_10 |
| value: 100.0 |
| - type: map_at_20 |
| value: 100.0 |
| - type: map_at_100 |
| value: 100.0 |
| - type: map_at_1000 |
| value: 100.0 |
| - type: recall_at_1 |
| value: 100.0 |
| - type: recall_at_3 |
| value: 100.0 |
| - type: recall_at_5 |
| value: 100.0 |
| - type: recall_at_10 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 100.0 |
| - type: precision_at_1 |
| value: 100.0 |
| - type: precision_at_3 |
| value: 33.333 |
| - type: precision_at_5 |
| value: 20.0 |
| - type: precision_at_10 |
| value: 10.0 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 100.0 |
| - type: mrr_at_3 |
| value: 100.0 |
| - type: mrr_at_5 |
| value: 100.0 |
| - type: mrr_at_10 |
| value: 100.0 |
| - type: mrr_at_20 |
| value: 100.0 |
| - type: mrr_at_100 |
| value: 100.0 |
| - type: mrr_at_1000 |
| value: 100.0 |
| - type: hit_rate_at_1 |
| value: 100.0 |
| - type: hit_rate_at_3 |
| value: 100.0 |
| - type: hit_rate_at_5 |
| value: 100.0 |
| - type: hit_rate_at_10 |
| value: 100.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/LEMBPasskeyRetrieval |
| name: MTEB LEMBPasskeyRetrieval |
| config: default |
| split: test_4096 |
| revision: 86ac5a6437198a94ecd666440c78b3e78f258274 |
| metrics: |
| - type: ndcg_at_1 |
| value: 100.0 |
| - type: ndcg_at_3 |
| value: 100.0 |
| - type: ndcg_at_5 |
| value: 100.0 |
| - type: ndcg_at_10 |
| value: 100.0 |
| - type: ndcg_at_20 |
| value: 100.0 |
| - type: ndcg_at_100 |
| value: 100.0 |
| - type: ndcg_at_1000 |
| value: 100.0 |
| - type: map_at_1 |
| value: 100.0 |
| - type: map_at_3 |
| value: 100.0 |
| - type: map_at_5 |
| value: 100.0 |
| - type: map_at_10 |
| value: 100.0 |
| - type: map_at_20 |
| value: 100.0 |
| - type: map_at_100 |
| value: 100.0 |
| - type: map_at_1000 |
| value: 100.0 |
| - type: recall_at_1 |
| value: 100.0 |
| - type: recall_at_3 |
| value: 100.0 |
| - type: recall_at_5 |
| value: 100.0 |
| - type: recall_at_10 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 100.0 |
| - type: precision_at_1 |
| value: 100.0 |
| - type: precision_at_3 |
| value: 33.333 |
| - type: precision_at_5 |
| value: 20.0 |
| - type: precision_at_10 |
| value: 10.0 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 100.0 |
| - type: mrr_at_3 |
| value: 100.0 |
| - type: mrr_at_5 |
| value: 100.0 |
| - type: mrr_at_10 |
| value: 100.0 |
| - type: mrr_at_20 |
| value: 100.0 |
| - type: mrr_at_100 |
| value: 100.0 |
| - type: mrr_at_1000 |
| value: 100.0 |
| - type: hit_rate_at_1 |
| value: 100.0 |
| - type: hit_rate_at_3 |
| value: 100.0 |
| - type: hit_rate_at_5 |
| value: 100.0 |
| - type: hit_rate_at_10 |
| value: 100.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/LEMBPasskeyRetrieval |
| name: MTEB LEMBPasskeyRetrieval |
| config: default |
| split: test_8192 |
| revision: 86ac5a6437198a94ecd666440c78b3e78f258274 |
| metrics: |
| - type: ndcg_at_1 |
| value: 100.0 |
| - type: ndcg_at_3 |
| value: 100.0 |
| - type: ndcg_at_5 |
| value: 100.0 |
| - type: ndcg_at_10 |
| value: 100.0 |
| - type: ndcg_at_20 |
| value: 100.0 |
| - type: ndcg_at_100 |
| value: 100.0 |
| - type: ndcg_at_1000 |
| value: 100.0 |
| - type: map_at_1 |
| value: 100.0 |
| - type: map_at_3 |
| value: 100.0 |
| - type: map_at_5 |
| value: 100.0 |
| - type: map_at_10 |
| value: 100.0 |
| - type: map_at_20 |
| value: 100.0 |
| - type: map_at_100 |
| value: 100.0 |
| - type: map_at_1000 |
| value: 100.0 |
| - type: recall_at_1 |
| value: 100.0 |
| - type: recall_at_3 |
| value: 100.0 |
| - type: recall_at_5 |
| value: 100.0 |
| - type: recall_at_10 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 100.0 |
| - type: precision_at_1 |
| value: 100.0 |
| - type: precision_at_3 |
| value: 33.333 |
| - type: precision_at_5 |
| value: 20.0 |
| - type: precision_at_10 |
| value: 10.0 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 100.0 |
| - type: mrr_at_3 |
| value: 100.0 |
| - type: mrr_at_5 |
| value: 100.0 |
| - type: mrr_at_10 |
| value: 100.0 |
| - type: mrr_at_20 |
| value: 100.0 |
| - type: mrr_at_100 |
| value: 100.0 |
| - type: mrr_at_1000 |
| value: 100.0 |
| - type: hit_rate_at_1 |
| value: 100.0 |
| - type: hit_rate_at_3 |
| value: 100.0 |
| - type: hit_rate_at_5 |
| value: 100.0 |
| - type: hit_rate_at_10 |
| value: 100.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/LEMBPasskeyRetrieval |
| name: MTEB LEMBPasskeyRetrieval |
| config: default |
| split: test_16384 |
| revision: 86ac5a6437198a94ecd666440c78b3e78f258274 |
| metrics: |
| - type: ndcg_at_1 |
| value: 94.0 |
| - type: ndcg_at_3 |
| value: 97.786 |
| - type: ndcg_at_5 |
| value: 97.786 |
| - type: ndcg_at_10 |
| value: 97.786 |
| - type: ndcg_at_20 |
| value: 97.786 |
| - type: ndcg_at_100 |
| value: 97.786 |
| - type: ndcg_at_1000 |
| value: 97.786 |
| - type: map_at_1 |
| value: 94.0 |
| - type: map_at_3 |
| value: 97.0 |
| - type: map_at_5 |
| value: 97.0 |
| - type: map_at_10 |
| value: 97.0 |
| - type: map_at_20 |
| value: 97.0 |
| - type: map_at_100 |
| value: 97.0 |
| - type: map_at_1000 |
| value: 97.0 |
| - type: recall_at_1 |
| value: 94.0 |
| - type: recall_at_3 |
| value: 100.0 |
| - type: recall_at_5 |
| value: 100.0 |
| - type: recall_at_10 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 94.0 |
| - type: precision_at_1 |
| value: 94.0 |
| - type: precision_at_3 |
| value: 33.333 |
| - type: precision_at_5 |
| value: 20.0 |
| - type: precision_at_10 |
| value: 10.0 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 94.0 |
| - type: mrr_at_3 |
| value: 97.0 |
| - type: mrr_at_5 |
| value: 97.0 |
| - type: mrr_at_10 |
| value: 97.0 |
| - type: mrr_at_20 |
| value: 97.0 |
| - type: mrr_at_100 |
| value: 97.0 |
| - type: mrr_at_1000 |
| value: 97.0 |
| - type: hit_rate_at_1 |
| value: 94.0 |
| - type: hit_rate_at_3 |
| value: 100.0 |
| - type: hit_rate_at_5 |
| value: 100.0 |
| - type: hit_rate_at_10 |
| value: 100.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/LEMBPasskeyRetrieval |
| name: MTEB LEMBPasskeyRetrieval |
| config: default |
| split: test_32768 |
| revision: 86ac5a6437198a94ecd666440c78b3e78f258274 |
| metrics: |
| - type: ndcg_at_1 |
| value: 98.0 |
| - type: ndcg_at_3 |
| value: 99.262 |
| - type: ndcg_at_5 |
| value: 99.262 |
| - type: ndcg_at_10 |
| value: 99.262 |
| - type: ndcg_at_20 |
| value: 99.262 |
| - type: ndcg_at_100 |
| value: 99.262 |
| - type: ndcg_at_1000 |
| value: 99.262 |
| - type: map_at_1 |
| value: 98.0 |
| - type: map_at_3 |
| value: 99.0 |
| - type: map_at_5 |
| value: 99.0 |
| - type: map_at_10 |
| value: 99.0 |
| - type: map_at_20 |
| value: 99.0 |
| - type: map_at_100 |
| value: 99.0 |
| - type: map_at_1000 |
| value: 99.0 |
| - type: recall_at_1 |
| value: 98.0 |
| - type: recall_at_3 |
| value: 100.0 |
| - type: recall_at_5 |
| value: 100.0 |
| - type: recall_at_10 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 100.0 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 98.0 |
| - type: precision_at_1 |
| value: 98.0 |
| - type: precision_at_3 |
| value: 33.333 |
| - type: precision_at_5 |
| value: 20.0 |
| - type: precision_at_10 |
| value: 10.0 |
| - type: precision_at_20 |
| value: 5.0 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 98.0 |
| - type: mrr_at_3 |
| value: 99.0 |
| - type: mrr_at_5 |
| value: 99.0 |
| - type: mrr_at_10 |
| value: 99.0 |
| - type: mrr_at_20 |
| value: 99.0 |
| - type: mrr_at_100 |
| value: 99.0 |
| - type: mrr_at_1000 |
| value: 99.0 |
| - type: hit_rate_at_1 |
| value: 98.0 |
| - type: hit_rate_at_3 |
| value: 100.0 |
| - type: hit_rate_at_5 |
| value: 100.0 |
| - type: hit_rate_at_10 |
| value: 100.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: dwzhu/LongEmbed |
| name: MTEB LEMBQMSumRetrieval |
| config: default |
| split: test |
| revision: 6e346642246bfb4928c560ee08640dc84d074e8c |
| metrics: |
| - type: ndcg_at_1 |
| value: 40.21 |
| - type: ndcg_at_3 |
| value: 49.733 |
| - type: ndcg_at_5 |
| value: 52.689 |
| - type: ndcg_at_10 |
| value: 55.278 |
| - type: ndcg_at_20 |
| value: 57.57 |
| - type: ndcg_at_100 |
| value: 60.835 |
| - type: ndcg_at_1000 |
| value: 61.017 |
| - type: map_at_1 |
| value: 40.21 |
| - type: map_at_3 |
| value: 47.37 |
| - type: map_at_5 |
| value: 49.003 |
| - type: map_at_10 |
| value: 50.062 |
| - type: map_at_20 |
| value: 50.687 |
| - type: map_at_100 |
| value: 51.122 |
| - type: map_at_1000 |
| value: 51.134 |
| - type: recall_at_1 |
| value: 40.21 |
| - type: recall_at_3 |
| value: 56.582 |
| - type: recall_at_5 |
| value: 63.785 |
| - type: recall_at_10 |
| value: 71.84 |
| - type: recall_at_20 |
| value: 80.943 |
| - type: recall_at_100 |
| value: 98.756 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 40.21 |
| - type: precision_at_1 |
| value: 40.21 |
| - type: precision_at_3 |
| value: 18.861 |
| - type: precision_at_5 |
| value: 12.757 |
| - type: precision_at_10 |
| value: 7.184 |
| - type: precision_at_20 |
| value: 4.047 |
| - type: precision_at_100 |
| value: 0.988 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 40.2096 |
| - type: mrr_at_3 |
| value: 47.3696 |
| - type: mrr_at_5 |
| value: 49.0035 |
| - type: mrr_at_10 |
| value: 50.0624 |
| - type: mrr_at_20 |
| value: 50.6865 |
| - type: mrr_at_100 |
| value: 51.1225 |
| - type: mrr_at_1000 |
| value: 51.1336 |
| - type: hit_rate_at_1 |
| value: 40.21 |
| - type: hit_rate_at_3 |
| value: 56.582 |
| - type: hit_rate_at_5 |
| value: 63.785 |
| - type: hit_rate_at_10 |
| value: 71.84 |
| - type: hit_rate_at_20 |
| value: 80.943 |
| - type: hit_rate_at_100 |
| value: 98.756 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: dwzhu/LongEmbed |
| name: MTEB LEMBSummScreenFDRetrieval |
| config: default |
| split: validation |
| revision: 6e346642246bfb4928c560ee08640dc84d074e8c |
| metrics: |
| - type: ndcg_at_1 |
| value: 95.536 |
| - type: ndcg_at_3 |
| value: 97.148 |
| - type: ndcg_at_5 |
| value: 97.519 |
| - type: ndcg_at_10 |
| value: 97.625 |
| - type: ndcg_at_20 |
| value: 97.625 |
| - type: ndcg_at_100 |
| value: 97.737 |
| - type: ndcg_at_1000 |
| value: 97.737 |
| - type: map_at_1 |
| value: 95.536 |
| - type: map_at_3 |
| value: 96.776 |
| - type: map_at_5 |
| value: 96.984 |
| - type: map_at_10 |
| value: 97.034 |
| - type: map_at_20 |
| value: 97.034 |
| - type: map_at_100 |
| value: 97.051 |
| - type: map_at_1000 |
| value: 97.051 |
| - type: recall_at_1 |
| value: 95.536 |
| - type: recall_at_3 |
| value: 98.214 |
| - type: recall_at_5 |
| value: 99.107 |
| - type: recall_at_10 |
| value: 99.405 |
| - type: recall_at_20 |
| value: 99.405 |
| - type: recall_at_100 |
| value: 100.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 95.536 |
| - type: precision_at_1 |
| value: 95.536 |
| - type: precision_at_3 |
| value: 32.738 |
| - type: precision_at_5 |
| value: 19.821 |
| - type: precision_at_10 |
| value: 9.94 |
| - type: precision_at_20 |
| value: 4.97 |
| - type: precision_at_100 |
| value: 1.0 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 95.5357 |
| - type: mrr_at_3 |
| value: 96.7758 |
| - type: mrr_at_5 |
| value: 96.9841 |
| - type: mrr_at_10 |
| value: 97.0337 |
| - type: mrr_at_20 |
| value: 97.0337 |
| - type: mrr_at_100 |
| value: 97.0511 |
| - type: mrr_at_1000 |
| value: 97.0511 |
| - type: hit_rate_at_1 |
| value: 95.536 |
| - type: hit_rate_at_3 |
| value: 98.214 |
| - type: hit_rate_at_5 |
| value: 99.107 |
| - type: hit_rate_at_10 |
| value: 99.405 |
| - type: hit_rate_at_20 |
| value: 99.405 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: dwzhu/LongEmbed |
| name: MTEB LEMBWikimQARetrieval |
| config: default |
| split: test |
| revision: 6e346642246bfb4928c560ee08640dc84d074e8c |
| metrics: |
| - type: ndcg_at_1 |
| value: 89.0 |
| - type: ndcg_at_3 |
| value: 91.559 |
| - type: ndcg_at_5 |
| value: 92.521 |
| - type: ndcg_at_10 |
| value: 92.862 |
| - type: ndcg_at_20 |
| value: 93.209 |
| - type: ndcg_at_100 |
| value: 93.406 |
| - type: ndcg_at_1000 |
| value: 93.547 |
| - type: map_at_1 |
| value: 89.0 |
| - type: map_at_3 |
| value: 90.944 |
| - type: map_at_5 |
| value: 91.478 |
| - type: map_at_10 |
| value: 91.629 |
| - type: map_at_20 |
| value: 91.729 |
| - type: map_at_100 |
| value: 91.761 |
| - type: map_at_1000 |
| value: 91.768 |
| - type: recall_at_1 |
| value: 89.0 |
| - type: recall_at_3 |
| value: 93.333 |
| - type: recall_at_5 |
| value: 95.667 |
| - type: recall_at_10 |
| value: 96.667 |
| - type: recall_at_20 |
| value: 98.0 |
| - type: recall_at_100 |
| value: 99.0 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 89.0 |
| - type: precision_at_1 |
| value: 89.0 |
| - type: precision_at_3 |
| value: 31.111 |
| - type: precision_at_5 |
| value: 19.133 |
| - type: precision_at_10 |
| value: 9.667 |
| - type: precision_at_20 |
| value: 4.9 |
| - type: precision_at_100 |
| value: 0.99 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 89.0 |
| - type: mrr_at_3 |
| value: 90.9444 |
| - type: mrr_at_5 |
| value: 91.4778 |
| - type: mrr_at_10 |
| value: 91.6286 |
| - type: mrr_at_20 |
| value: 91.7292 |
| - type: mrr_at_100 |
| value: 91.7608 |
| - type: mrr_at_1000 |
| value: 91.7683 |
| - type: hit_rate_at_1 |
| value: 89.0 |
| - type: hit_rate_at_3 |
| value: 93.333 |
| - type: hit_rate_at_5 |
| value: 95.667 |
| - type: hit_rate_at_10 |
| value: 96.667 |
| - type: hit_rate_at_20 |
| value: 98.0 |
| - type: hit_rate_at_100 |
| value: 99.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/MTOPDomainClassification |
| name: MTEB MTOPDomainClassification (en) |
| config: en |
| split: test |
| revision: a76d16fae880597b9c73047b50159220a441cb54 |
| metrics: |
| - type: accuracy |
| value: 94.5987 |
| - type: f1 |
| value: 94.2853 |
| - type: f1_weighted |
| value: 94.5968 |
| - type: precision |
| value: 94.1672 |
| - type: precision_weighted |
| value: 94.7782 |
| - type: recall |
| value: 94.5935 |
| - type: recall_weighted |
| value: 94.5987 |
| - type: ap |
| value: .nan |
| - type: ap_weighted |
| value: .nan |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/amazon_massive_intent |
| name: MTEB MassiveIntentClassification (en) |
| config: en |
| split: test |
| revision: 4672e20407010da34463acc759c162ca9734bca6 |
| metrics: |
| - type: accuracy |
| value: 75.6792 |
| - type: f1 |
| value: 73.216 |
| - type: f1_weighted |
| value: 75.5343 |
| - type: precision |
| value: 71.0843 |
| - type: precision_weighted |
| value: 78.6203 |
| - type: recall |
| value: 80.5074 |
| - type: recall_weighted |
| value: 75.6792 |
| - type: ap |
| value: .nan |
| - type: ap_weighted |
| value: .nan |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/amazon_massive_scenario |
| name: MTEB MassiveScenarioClassification (en) |
| config: en |
| split: test |
| revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 |
| metrics: |
| - type: accuracy |
| value: 77.6194 |
| - type: f1 |
| value: 77.3532 |
| - type: f1_weighted |
| value: 77.456 |
| - type: precision |
| value: 75.5415 |
| - type: precision_weighted |
| value: 79.5373 |
| - type: recall |
| value: 81.6365 |
| - type: recall_weighted |
| value: 77.6194 |
| - type: ap |
| value: .nan |
| - type: ap_weighted |
| value: .nan |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/medrxiv-clustering-p2p |
| name: MTEB MedrxivClusteringP2P.v2 |
| config: default |
| split: test |
| revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 |
| metrics: |
| - type: v_measure |
| value: 38.5612 |
| - type: v_measure_std |
| value: 0.8631 |
| - type: ami |
| value: 37.1726 |
| - type: ami_std |
| value: 0.8709 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/medrxiv-clustering-s2s |
| name: MTEB MedrxivClusteringS2S.v2 |
| config: default |
| split: test |
| revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 |
| metrics: |
| - type: v_measure |
| value: 38.4619 |
| - type: v_measure_std |
| value: 0.5992 |
| - type: ami |
| value: 37.0538 |
| - type: ami_std |
| value: 0.6203 |
| - task: |
| type: Reranking |
| dataset: |
| type: mteb/MindSmallReranking |
| name: MTEB MindSmallReranking |
| config: default |
| split: test |
| revision: 227478e3235572039f4f7661840e059f31ef6eb1 |
| metrics: |
| - type: ndcg_at_1 |
| value: 13.291 |
| - type: ndcg_at_3 |
| value: 20.876 |
| - type: ndcg_at_5 |
| value: 25.431 |
| - type: ndcg_at_10 |
| value: 31.525 |
| - type: ndcg_at_20 |
| value: 36.685 |
| - type: ndcg_at_100 |
| value: 43.086 |
| - type: ndcg_at_1000 |
| value: 43.44 |
| - type: map_at_1 |
| value: 10.172 |
| - type: map_at_3 |
| value: 17.181 |
| - type: map_at_5 |
| value: 19.886 |
| - type: map_at_10 |
| value: 22.615 |
| - type: map_at_20 |
| value: 24.3 |
| - type: map_at_100 |
| value: 25.663 |
| - type: map_at_1000 |
| value: 25.707 |
| - type: recall_at_1 |
| value: 10.172 |
| - type: recall_at_3 |
| value: 25.909 |
| - type: recall_at_5 |
| value: 36.784 |
| - type: recall_at_10 |
| value: 54.026 |
| - type: recall_at_20 |
| value: 71.596 |
| - type: recall_at_100 |
| value: 98.286 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 10.172 |
| - type: precision_at_1 |
| value: 13.291 |
| - type: precision_at_3 |
| value: 11.478 |
| - type: precision_at_5 |
| value: 10.023 |
| - type: precision_at_10 |
| value: 7.757 |
| - type: precision_at_20 |
| value: 5.51 |
| - type: precision_at_100 |
| value: 1.764 |
| - type: precision_at_1000 |
| value: 0.183 |
| - type: mrr_at_1 |
| value: 13.2911 |
| - type: mrr_at_3 |
| value: 21.5975 |
| - type: mrr_at_5 |
| value: 24.5174 |
| - type: mrr_at_10 |
| value: 27.0451 |
| - type: mrr_at_20 |
| value: 28.2346 |
| - type: mrr_at_100 |
| value: 28.7805 |
| - type: mrr_at_1000 |
| value: 28.7853 |
| - type: hit_rate_at_1 |
| value: 13.291 |
| - type: hit_rate_at_3 |
| value: 32.627 |
| - type: hit_rate_at_5 |
| value: 45.48 |
| - type: hit_rate_at_10 |
| value: 64.401 |
| - type: hit_rate_at_20 |
| value: 81.328 |
| - type: hit_rate_at_100 |
| value: 99.411 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - type: max_over_subqueries_ndcg_at_1 |
| value: 17.962 |
| - type: max_over_subqueries_ndcg_at_3 |
| value: 28.062 |
| - type: max_over_subqueries_ndcg_at_5 |
| value: 33.29 |
| - type: max_over_subqueries_ndcg_at_10 |
| value: 39.321 |
| - type: max_over_subqueries_ndcg_at_20 |
| value: 43.808 |
| - type: max_over_subqueries_ndcg_at_100 |
| value: 48.128 |
| - type: max_over_subqueries_ndcg_at_1000 |
| value: 48.3 |
| - type: max_over_subqueries_map_at_1 |
| value: 14.947 |
| - type: max_over_subqueries_map_at_3 |
| value: 23.977 |
| - type: max_over_subqueries_map_at_5 |
| value: 27.07 |
| - type: max_over_subqueries_map_at_10 |
| value: 29.789 |
| - type: max_over_subqueries_map_at_20 |
| value: 31.259 |
| - type: max_over_subqueries_map_at_100 |
| value: 32.167 |
| - type: max_over_subqueries_map_at_1000 |
| value: 32.186 |
| - type: max_over_subqueries_recall_at_1 |
| value: 14.947 |
| - type: max_over_subqueries_recall_at_3 |
| value: 35.072 |
| - type: max_over_subqueries_recall_at_5 |
| value: 47.412 |
| - type: max_over_subqueries_recall_at_10 |
| value: 64.611 |
| - type: max_over_subqueries_recall_at_20 |
| value: 80.272 |
| - type: max_over_subqueries_recall_at_100 |
| value: 99.074 |
| - type: max_over_subqueries_recall_at_1000 |
| value: 99.999 |
| - type: max_over_subqueries_accuracy |
| value: 14.947 |
| - type: max_over_subqueries_precision_at_1 |
| value: 17.962 |
| - type: max_over_subqueries_precision_at_3 |
| value: 14.375 |
| - type: max_over_subqueries_precision_at_5 |
| value: 11.943 |
| - type: max_over_subqueries_precision_at_10 |
| value: 8.496 |
| - type: max_over_subqueries_precision_at_20 |
| value: 5.551 |
| - type: max_over_subqueries_precision_at_100 |
| value: 1.496 |
| - type: max_over_subqueries_precision_at_1000 |
| value: 0.152 |
| - type: max_over_subqueries_mrr_at_1 |
| value: 17.9622 |
| - type: max_over_subqueries_mrr_at_3 |
| value: 27.981 |
| - type: max_over_subqueries_mrr_at_5 |
| value: 31.0729 |
| - type: max_over_subqueries_mrr_at_10 |
| value: 33.4506 |
| - type: max_over_subqueries_mrr_at_20 |
| value: 34.4594 |
| - type: max_over_subqueries_mrr_at_100 |
| value: 34.8568 |
| - type: max_over_subqueries_mrr_at_1000 |
| value: 34.8601 |
| - type: max_over_subqueries_hit_rate_at_1 |
| value: 17.962 |
| - type: max_over_subqueries_hit_rate_at_3 |
| value: 41.091 |
| - type: max_over_subqueries_hit_rate_at_5 |
| value: 54.627 |
| - type: max_over_subqueries_hit_rate_at_10 |
| value: 72.307 |
| - type: max_over_subqueries_hit_rate_at_20 |
| value: 86.57 |
| - type: max_over_subqueries_hit_rate_at_100 |
| value: 99.593 |
| - type: max_over_subqueries_hit_rate_at_1000 |
| value: 99.999 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/PIQA |
| name: MTEB PIQA |
| config: default |
| split: test |
| revision: aee23bb32dafbaddddf16cd5033d912693984d9a |
| metrics: |
| - type: ndcg_at_1 |
| value: 19.859 |
| - type: ndcg_at_3 |
| value: 30.315 |
| - type: ndcg_at_5 |
| value: 32.7 |
| - type: ndcg_at_10 |
| value: 35.051 |
| - type: ndcg_at_20 |
| value: 36.548 |
| - type: ndcg_at_100 |
| value: 39.288 |
| - type: ndcg_at_1000 |
| value: 41.229 |
| - type: map_at_1 |
| value: 19.859 |
| - type: map_at_3 |
| value: 27.875 |
| - type: map_at_5 |
| value: 29.191 |
| - type: map_at_10 |
| value: 30.147 |
| - type: map_at_20 |
| value: 30.557 |
| - type: map_at_100 |
| value: 30.94 |
| - type: map_at_1000 |
| value: 31.002 |
| - type: recall_at_1 |
| value: 19.859 |
| - type: recall_at_3 |
| value: 37.323 |
| - type: recall_at_5 |
| value: 43.145 |
| - type: recall_at_10 |
| value: 50.49 |
| - type: recall_at_20 |
| value: 56.42 |
| - type: recall_at_100 |
| value: 71.164 |
| - type: recall_at_1000 |
| value: 87.051 |
| - type: accuracy |
| value: 19.859 |
| - type: precision_at_1 |
| value: 19.859 |
| - type: precision_at_3 |
| value: 12.441 |
| - type: precision_at_5 |
| value: 8.629 |
| - type: precision_at_10 |
| value: 5.049 |
| - type: precision_at_20 |
| value: 2.821 |
| - type: precision_at_100 |
| value: 0.712 |
| - type: precision_at_1000 |
| value: 0.087 |
| - type: mrr_at_1 |
| value: 19.8585 |
| - type: mrr_at_3 |
| value: 27.8745 |
| - type: mrr_at_5 |
| value: 29.1911 |
| - type: mrr_at_10 |
| value: 30.1468 |
| - type: mrr_at_20 |
| value: 30.5569 |
| - type: mrr_at_100 |
| value: 30.9395 |
| - type: mrr_at_1000 |
| value: 31.0023 |
| - type: hit_rate_at_1 |
| value: 19.859 |
| - type: hit_rate_at_3 |
| value: 37.323 |
| - type: hit_rate_at_5 |
| value: 43.145 |
| - type: hit_rate_at_10 |
| value: 50.49 |
| - type: hit_rate_at_20 |
| value: 56.42 |
| - type: hit_rate_at_100 |
| value: 71.164 |
| - type: hit_rate_at_1000 |
| value: 87.051 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/Quail |
| name: MTEB Quail |
| config: default |
| split: test |
| revision: 516ac28de67229da337281f702310a08c9fe0a03 |
| metrics: |
| - type: ndcg_at_1 |
| value: 2.868 |
| - type: ndcg_at_3 |
| value: 5.126 |
| - type: ndcg_at_5 |
| value: 6.409 |
| - type: ndcg_at_10 |
| value: 7.968 |
| - type: ndcg_at_20 |
| value: 9.472 |
| - type: ndcg_at_100 |
| value: 11.997 |
| - type: ndcg_at_1000 |
| value: 15.15 |
| - type: map_at_1 |
| value: 2.868 |
| - type: map_at_3 |
| value: 4.516 |
| - type: map_at_5 |
| value: 5.225 |
| - type: map_at_10 |
| value: 5.87 |
| - type: map_at_20 |
| value: 6.282 |
| - type: map_at_100 |
| value: 6.627 |
| - type: map_at_1000 |
| value: 6.728 |
| - type: recall_at_1 |
| value: 2.868 |
| - type: recall_at_3 |
| value: 6.912 |
| - type: recall_at_5 |
| value: 10.037 |
| - type: recall_at_10 |
| value: 14.853 |
| - type: recall_at_20 |
| value: 20.809 |
| - type: recall_at_100 |
| value: 34.485 |
| - type: recall_at_1000 |
| value: 60.368 |
| - type: accuracy |
| value: 2.868 |
| - type: precision_at_1 |
| value: 2.868 |
| - type: precision_at_3 |
| value: 2.304 |
| - type: precision_at_5 |
| value: 2.007 |
| - type: precision_at_10 |
| value: 1.485 |
| - type: precision_at_20 |
| value: 1.04 |
| - type: precision_at_100 |
| value: 0.345 |
| - type: precision_at_1000 |
| value: 0.06 |
| - type: mrr_at_1 |
| value: 2.8676 |
| - type: mrr_at_3 |
| value: 4.5159 |
| - type: mrr_at_5 |
| value: 5.2255 |
| - type: mrr_at_10 |
| value: 5.8699 |
| - type: mrr_at_20 |
| value: 6.2825 |
| - type: mrr_at_100 |
| value: 6.6274 |
| - type: mrr_at_1000 |
| value: 6.7279 |
| - type: hit_rate_at_1 |
| value: 2.868 |
| - type: hit_rate_at_3 |
| value: 6.912 |
| - type: hit_rate_at_5 |
| value: 10.037 |
| - type: hit_rate_at_10 |
| value: 14.853 |
| - type: hit_rate_at_20 |
| value: 20.809 |
| - type: hit_rate_at_100 |
| value: 34.485 |
| - type: hit_rate_at_1000 |
| value: 60.368 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/RARbCode |
| name: MTEB RARbCode |
| config: default |
| split: test |
| revision: ed0a01c3e4d39ec17badb87405e595322f8f2b86 |
| metrics: |
| - type: ndcg_at_1 |
| value: 51.078 |
| - type: ndcg_at_3 |
| value: 64.478 |
| - type: ndcg_at_5 |
| value: 70.411 |
| - type: ndcg_at_10 |
| value: 72.807 |
| - type: ndcg_at_20 |
| value: 73.268 |
| - type: ndcg_at_100 |
| value: 73.56 |
| - type: ndcg_at_1000 |
| value: 73.618 |
| - type: map_at_1 |
| value: 51.078 |
| - type: map_at_3 |
| value: 61.006 |
| - type: map_at_5 |
| value: 64.295 |
| - type: map_at_10 |
| value: 65.348 |
| - type: map_at_20 |
| value: 65.475 |
| - type: map_at_100 |
| value: 65.522 |
| - type: map_at_1000 |
| value: 65.524 |
| - type: recall_at_1 |
| value: 51.078 |
| - type: recall_at_3 |
| value: 74.596 |
| - type: recall_at_5 |
| value: 89.016 |
| - type: recall_at_10 |
| value: 96.092 |
| - type: recall_at_20 |
| value: 97.911 |
| - type: recall_at_100 |
| value: 99.394 |
| - type: recall_at_1000 |
| value: 99.865 |
| - type: accuracy |
| value: 51.078 |
| - type: precision_at_1 |
| value: 51.078 |
| - type: precision_at_3 |
| value: 24.865 |
| - type: precision_at_5 |
| value: 17.803 |
| - type: precision_at_10 |
| value: 9.609 |
| - type: precision_at_20 |
| value: 4.896 |
| - type: precision_at_100 |
| value: 0.994 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 51.0782 |
| - type: mrr_at_3 |
| value: 61.0063 |
| - type: mrr_at_5 |
| value: 64.2947 |
| - type: mrr_at_10 |
| value: 65.3481 |
| - type: mrr_at_20 |
| value: 65.4749 |
| - type: mrr_at_100 |
| value: 65.5219 |
| - type: mrr_at_1000 |
| value: 65.5237 |
| - type: hit_rate_at_1 |
| value: 51.078 |
| - type: hit_rate_at_3 |
| value: 74.596 |
| - type: hit_rate_at_5 |
| value: 89.016 |
| - type: hit_rate_at_10 |
| value: 96.092 |
| - type: hit_rate_at_20 |
| value: 97.911 |
| - type: hit_rate_at_100 |
| value: 99.394 |
| - type: hit_rate_at_1000 |
| value: 99.865 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/RARbMath |
| name: MTEB RARbMath |
| config: default |
| split: test |
| revision: cc879a9719a10f4a80677ac26798cfd701cf94b7 |
| metrics: |
| - type: ndcg_at_1 |
| value: 81.152 |
| - type: ndcg_at_3 |
| value: 83.927 |
| - type: ndcg_at_5 |
| value: 84.704 |
| - type: ndcg_at_10 |
| value: 85.521 |
| - type: ndcg_at_20 |
| value: 86.079 |
| - type: ndcg_at_100 |
| value: 86.985 |
| - type: ndcg_at_1000 |
| value: 87.279 |
| - type: map_at_1 |
| value: 81.152 |
| - type: map_at_3 |
| value: 83.259 |
| - type: map_at_5 |
| value: 83.692 |
| - type: map_at_10 |
| value: 84.031 |
| - type: map_at_20 |
| value: 84.185 |
| - type: map_at_100 |
| value: 84.31 |
| - type: map_at_1000 |
| value: 84.322 |
| - type: recall_at_1 |
| value: 81.152 |
| - type: recall_at_3 |
| value: 85.852 |
| - type: recall_at_5 |
| value: 87.735 |
| - type: recall_at_10 |
| value: 90.252 |
| - type: recall_at_20 |
| value: 92.451 |
| - type: recall_at_100 |
| value: 97.326 |
| - type: recall_at_1000 |
| value: 99.604 |
| - type: accuracy |
| value: 81.152 |
| - type: precision_at_1 |
| value: 81.152 |
| - type: precision_at_3 |
| value: 28.617 |
| - type: precision_at_5 |
| value: 17.547 |
| - type: precision_at_10 |
| value: 9.025 |
| - type: precision_at_20 |
| value: 4.623 |
| - type: precision_at_100 |
| value: 0.973 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 81.1521 |
| - type: mrr_at_3 |
| value: 83.2595 |
| - type: mrr_at_5 |
| value: 83.6915 |
| - type: mrr_at_10 |
| value: 84.0311 |
| - type: mrr_at_20 |
| value: 84.185 |
| - type: mrr_at_100 |
| value: 84.3103 |
| - type: mrr_at_1000 |
| value: 84.3221 |
| - type: hit_rate_at_1 |
| value: 81.152 |
| - type: hit_rate_at_3 |
| value: 85.852 |
| - type: hit_rate_at_5 |
| value: 87.735 |
| - type: hit_rate_at_10 |
| value: 90.252 |
| - type: hit_rate_at_20 |
| value: 92.451 |
| - type: hit_rate_at_100 |
| value: 97.326 |
| - type: hit_rate_at_1000 |
| value: 99.604 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/scidocs |
| name: MTEB SCIDOCS |
| config: default |
| split: test |
| revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88 |
| metrics: |
| - type: ndcg_at_1 |
| value: 27.8 |
| - type: ndcg_at_3 |
| value: 22.61 |
| - type: ndcg_at_5 |
| value: 19.668 |
| - type: ndcg_at_10 |
| value: 23.84 |
| - type: ndcg_at_20 |
| value: 26.907 |
| - type: ndcg_at_100 |
| value: 32.829 |
| - type: ndcg_at_1000 |
| value: 38.371 |
| - type: map_at_1 |
| value: 5.628 |
| - type: map_at_3 |
| value: 10.207 |
| - type: map_at_5 |
| value: 12.352 |
| - type: map_at_10 |
| value: 14.613 |
| - type: map_at_20 |
| value: 15.8 |
| - type: map_at_100 |
| value: 17.02 |
| - type: map_at_1000 |
| value: 17.38 |
| - type: recall_at_1 |
| value: 5.628 |
| - type: recall_at_3 |
| value: 12.867 |
| - type: recall_at_5 |
| value: 17.447 |
| - type: recall_at_10 |
| value: 25.057 |
| - type: recall_at_20 |
| value: 32.278 |
| - type: recall_at_100 |
| value: 51.632 |
| - type: recall_at_1000 |
| value: 78.328 |
| - type: accuracy |
| value: 5.628 |
| - type: precision_at_1 |
| value: 27.8 |
| - type: precision_at_3 |
| value: 21.133 |
| - type: precision_at_5 |
| value: 17.18 |
| - type: precision_at_10 |
| value: 12.36 |
| - type: precision_at_20 |
| value: 7.96 |
| - type: precision_at_100 |
| value: 2.541 |
| - type: precision_at_1000 |
| value: 0.386 |
| - type: mrr_at_1 |
| value: 27.8 |
| - type: mrr_at_3 |
| value: 36.2667 |
| - type: mrr_at_5 |
| value: 37.9567 |
| - type: mrr_at_10 |
| value: 39.4979 |
| - type: mrr_at_20 |
| value: 40.0757 |
| - type: mrr_at_100 |
| value: 40.505 |
| - type: mrr_at_1000 |
| value: 40.5442 |
| - type: hit_rate_at_1 |
| value: 27.8 |
| - type: hit_rate_at_3 |
| value: 47.5 |
| - type: hit_rate_at_5 |
| value: 54.8 |
| - type: hit_rate_at_10 |
| value: 66.2 |
| - type: hit_rate_at_20 |
| value: 74.7 |
| - type: hit_rate_at_100 |
| value: 91.1 |
| - type: hit_rate_at_1000 |
| value: 98.5 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sickr-sts |
| name: MTEB SICK-R |
| config: default |
| split: test |
| revision: 20a6d6f312dd54037fe07a32d58e5e168867909d |
| metrics: |
| - type: pearson |
| value: 86.1633 |
| - type: spearman |
| value: 81.7585 |
| - type: cosine_pearson |
| value: 86.1633 |
| - type: cosine_spearman |
| value: 81.7585 |
| - type: manhattan_pearson |
| value: 83.1762 |
| - type: manhattan_spearman |
| value: 81.396 |
| - type: euclidean_pearson |
| value: 83.2109 |
| - type: euclidean_spearman |
| value: 81.4284 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/SIQA |
| name: MTEB SIQA |
| config: default |
| split: test |
| revision: 59898732ccfb72afca329ce110d6799676ef0f84 |
| metrics: |
| - type: ndcg_at_1 |
| value: 1.279 |
| - type: ndcg_at_3 |
| value: 1.846 |
| - type: ndcg_at_5 |
| value: 2.053 |
| - type: ndcg_at_10 |
| value: 2.364 |
| - type: ndcg_at_20 |
| value: 2.762 |
| - type: ndcg_at_100 |
| value: 3.473 |
| - type: ndcg_at_1000 |
| value: 6.47 |
| - type: map_at_1 |
| value: 1.279 |
| - type: map_at_3 |
| value: 1.706 |
| - type: map_at_5 |
| value: 1.818 |
| - type: map_at_10 |
| value: 1.945 |
| - type: map_at_20 |
| value: 2.052 |
| - type: map_at_100 |
| value: 2.14 |
| - type: map_at_1000 |
| value: 2.217 |
| - type: recall_at_1 |
| value: 1.279 |
| - type: recall_at_3 |
| value: 2.252 |
| - type: recall_at_5 |
| value: 2.764 |
| - type: recall_at_10 |
| value: 3.736 |
| - type: recall_at_20 |
| value: 5.322 |
| - type: recall_at_100 |
| value: 9.314 |
| - type: recall_at_1000 |
| value: 35.056 |
| - type: accuracy |
| value: 1.279 |
| - type: precision_at_1 |
| value: 1.279 |
| - type: precision_at_3 |
| value: 0.751 |
| - type: precision_at_5 |
| value: 0.553 |
| - type: precision_at_10 |
| value: 0.374 |
| - type: precision_at_20 |
| value: 0.266 |
| - type: precision_at_100 |
| value: 0.093 |
| - type: precision_at_1000 |
| value: 0.035 |
| - type: mrr_at_1 |
| value: 1.2794 |
| - type: mrr_at_3 |
| value: 1.7059 |
| - type: mrr_at_5 |
| value: 1.8185 |
| - type: mrr_at_10 |
| value: 1.9446 |
| - type: mrr_at_20 |
| value: 2.0521 |
| - type: mrr_at_100 |
| value: 2.1402 |
| - type: mrr_at_1000 |
| value: 2.2168 |
| - type: hit_rate_at_1 |
| value: 1.279 |
| - type: hit_rate_at_3 |
| value: 2.252 |
| - type: hit_rate_at_5 |
| value: 2.764 |
| - type: hit_rate_at_10 |
| value: 3.736 |
| - type: hit_rate_at_20 |
| value: 5.322 |
| - type: hit_rate_at_100 |
| value: 9.314 |
| - type: hit_rate_at_1000 |
| value: 35.056 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sts12-sts |
| name: MTEB STS12 |
| config: default |
| split: test |
| revision: a0d554a64d88156834ff5ae9920b964011b16384 |
| metrics: |
| - type: pearson |
| value: 82.0189 |
| - type: spearman |
| value: 74.7194 |
| - type: cosine_pearson |
| value: 82.0189 |
| - type: cosine_spearman |
| value: 74.7192 |
| - type: manhattan_pearson |
| value: 78.5505 |
| - type: manhattan_spearman |
| value: 75.0502 |
| - type: euclidean_pearson |
| value: 78.5956 |
| - type: euclidean_spearman |
| value: 75.0884 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sts13-sts |
| name: MTEB STS13 |
| config: default |
| split: test |
| revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca |
| metrics: |
| - type: pearson |
| value: 86.6221 |
| - type: spearman |
| value: 86.4882 |
| - type: cosine_pearson |
| value: 86.6221 |
| - type: cosine_spearman |
| value: 86.4882 |
| - type: manhattan_pearson |
| value: 86.3955 |
| - type: manhattan_spearman |
| value: 87.6413 |
| - type: euclidean_pearson |
| value: 86.5054 |
| - type: euclidean_spearman |
| value: 87.784 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sts14-sts |
| name: MTEB STS14 |
| config: default |
| split: test |
| revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 |
| metrics: |
| - type: pearson |
| value: 84.025 |
| - type: spearman |
| value: 81.0291 |
| - type: cosine_pearson |
| value: 84.025 |
| - type: cosine_spearman |
| value: 81.0291 |
| - type: manhattan_pearson |
| value: 82.0683 |
| - type: manhattan_spearman |
| value: 81.0983 |
| - type: euclidean_pearson |
| value: 82.1368 |
| - type: euclidean_spearman |
| value: 81.1477 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sts15-sts |
| name: MTEB STS15 |
| config: default |
| split: test |
| revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 |
| metrics: |
| - type: pearson |
| value: 88.4824 |
| - type: spearman |
| value: 88.8167 |
| - type: cosine_pearson |
| value: 88.4824 |
| - type: cosine_spearman |
| value: 88.8167 |
| - type: manhattan_pearson |
| value: 87.3373 |
| - type: manhattan_spearman |
| value: 88.1086 |
| - type: euclidean_pearson |
| value: 87.4178 |
| - type: euclidean_spearman |
| value: 88.1941 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sts17-crosslingual-sts |
| name: MTEB STS17 (en-en) |
| config: en-en |
| split: test |
| revision: faeb762787bd10488a50c8b5be4a3b82e411949c |
| metrics: |
| - type: pearson |
| value: 90.595 |
| - type: spearman |
| value: 90.9423 |
| - type: cosine_pearson |
| value: 90.595 |
| - type: cosine_spearman |
| value: 90.9423 |
| - type: manhattan_pearson |
| value: 89.7504 |
| - type: manhattan_spearman |
| value: 89.806 |
| - type: euclidean_pearson |
| value: 89.8143 |
| - type: euclidean_spearman |
| value: 89.8452 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/sts22-crosslingual-sts |
| name: MTEB STS22.v2 (en) |
| config: en |
| split: test |
| revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd |
| metrics: |
| - type: pearson |
| value: 67.7048 |
| - type: spearman |
| value: 67.4456 |
| - type: cosine_pearson |
| value: 67.7048 |
| - type: cosine_spearman |
| value: 67.4456 |
| - type: manhattan_pearson |
| value: 63.7724 |
| - type: manhattan_spearman |
| value: 63.8649 |
| - type: euclidean_pearson |
| value: 63.5939 |
| - type: euclidean_spearman |
| value: 63.8456 |
| - task: |
| type: STS |
| dataset: |
| type: mteb/stsbenchmark-sts |
| name: MTEB STSBenchmark |
| config: default |
| split: test |
| revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 |
| metrics: |
| - type: pearson |
| value: 86.9609 |
| - type: spearman |
| value: 86.7611 |
| - type: cosine_pearson |
| value: 86.9609 |
| - type: cosine_spearman |
| value: 86.7611 |
| - type: manhattan_pearson |
| value: 86.0919 |
| - type: manhattan_spearman |
| value: 86.6102 |
| - type: euclidean_pearson |
| value: 86.0912 |
| - type: euclidean_spearman |
| value: 86.5851 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/SpartQA |
| name: MTEB SpartQA |
| config: default |
| split: test |
| revision: 1c858df377e57725a014a1b7321ebd79d62016b6 |
| metrics: |
| - type: ndcg_at_1 |
| value: 1.085 |
| - type: ndcg_at_3 |
| value: 2.172 |
| - type: ndcg_at_5 |
| value: 3.616 |
| - type: ndcg_at_10 |
| value: 7.048 |
| - type: ndcg_at_20 |
| value: 9.56 |
| - type: ndcg_at_100 |
| value: 11.078 |
| - type: ndcg_at_1000 |
| value: 15.343 |
| - type: map_at_1 |
| value: 0.788 |
| - type: map_at_3 |
| value: 1.671 |
| - type: map_at_5 |
| value: 2.413 |
| - type: map_at_10 |
| value: 3.748 |
| - type: map_at_20 |
| value: 4.435 |
| - type: map_at_100 |
| value: 4.646 |
| - type: map_at_1000 |
| value: 4.73 |
| - type: recall_at_1 |
| value: 0.788 |
| - type: recall_at_3 |
| value: 2.931 |
| - type: recall_at_5 |
| value: 6.177 |
| - type: recall_at_10 |
| value: 15.925 |
| - type: recall_at_20 |
| value: 24.847 |
| - type: recall_at_100 |
| value: 31.961 |
| - type: recall_at_1000 |
| value: 64.385 |
| - type: accuracy |
| value: 0.788 |
| - type: precision_at_1 |
| value: 1.085 |
| - type: precision_at_3 |
| value: 1.447 |
| - type: precision_at_5 |
| value: 1.825 |
| - type: precision_at_10 |
| value: 2.39 |
| - type: precision_at_20 |
| value: 1.874 |
| - type: precision_at_100 |
| value: 0.493 |
| - type: precision_at_1000 |
| value: 0.11 |
| - type: mrr_at_1 |
| value: 1.0851 |
| - type: mrr_at_3 |
| value: 2.3882 |
| - type: mrr_at_5 |
| value: 3.376 |
| - type: mrr_at_10 |
| value: 5.0346 |
| - type: mrr_at_20 |
| value: 5.8236 |
| - type: mrr_at_100 |
| value: 6.083 |
| - type: mrr_at_1000 |
| value: 6.175 |
| - type: hit_rate_at_1 |
| value: 1.085 |
| - type: hit_rate_at_3 |
| value: 4.257 |
| - type: hit_rate_at_5 |
| value: 8.737 |
| - type: hit_rate_at_10 |
| value: 21.592 |
| - type: hit_rate_at_20 |
| value: 32.693 |
| - type: hit_rate_at_100 |
| value: 42.71 |
| - type: hit_rate_at_1000 |
| value: 80.968 |
| - task: |
| type: PairClassification |
| dataset: |
| type: mteb/sprintduplicatequestions-pairclassification |
| name: MTEB SprintDuplicateQuestions |
| config: default |
| split: test |
| revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 |
| metrics: |
| - type: similarity_accuracy |
| value: 99.7881 |
| - type: similarity_f1 |
| value: 89.2354 |
| - type: similarity_precision |
| value: 89.7773 |
| - type: similarity_recall |
| value: 88.7 |
| - type: similarity_ap |
| value: 94.805 |
| - type: cosine_accuracy |
| value: 99.7881 |
| - type: cosine_f1 |
| value: 89.2354 |
| - type: cosine_precision |
| value: 89.7773 |
| - type: cosine_recall |
| value: 88.7 |
| - type: cosine_ap |
| value: 94.805 |
| - type: manhattan_accuracy |
| value: 99.7911 |
| - type: manhattan_f1 |
| value: 89.4553 |
| - type: manhattan_precision |
| value: 89.4106 |
| - type: manhattan_recall |
| value: 89.5 |
| - type: manhattan_ap |
| value: 94.7666 |
| - type: euclidean_accuracy |
| value: 99.7881 |
| - type: euclidean_f1 |
| value: 89.2857 |
| - type: euclidean_precision |
| value: 88.5827 |
| - type: euclidean_recall |
| value: 90.0 |
| - type: euclidean_ap |
| value: 94.7158 |
| - type: dot_accuracy |
| value: 99.6297 |
| - type: dot_f1 |
| value: 80.9665 |
| - type: dot_precision |
| value: 79.8638 |
| - type: dot_recall |
| value: 82.1 |
| - type: dot_ap |
| value: 89.4884 |
| - type: max_accuracy |
| value: 99.7911 |
| - type: max_f1 |
| value: 89.4553 |
| - type: max_precision |
| value: 89.7773 |
| - type: max_recall |
| value: 90.0 |
| - type: max_ap |
| value: 94.805 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/stackexchange-clustering |
| name: MTEB StackExchangeClustering.v2 |
| config: default |
| split: test |
| revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 |
| metrics: |
| - type: v_measure |
| value: 61.5805 |
| - type: v_measure_std |
| value: 0.4115 |
| - type: ami |
| value: 58.0931 |
| - type: ami_std |
| value: 0.4061 |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/stackexchange-clustering-p2p |
| name: MTEB StackExchangeClusteringP2P.v2 |
| config: default |
| split: test |
| revision: 815ca46b2622cec33ccafc3735d572c266efdb44 |
| metrics: |
| - type: v_measure |
| value: 41.6448 |
| - type: v_measure_std |
| value: 0.3637 |
| - type: ami |
| value: 34.7334 |
| - type: ami_std |
| value: 0.391 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/StackOverflowQA |
| name: MTEB StackOverflowQA |
| config: default |
| split: test |
| revision: 7b46f5ebd2365278c1bb025986dda2fee893bbf5 |
| metrics: |
| - type: ndcg_at_1 |
| value: 88.114 |
| - type: ndcg_at_3 |
| value: 91.416 |
| - type: ndcg_at_5 |
| value: 91.982 |
| - type: ndcg_at_10 |
| value: 92.429 |
| - type: ndcg_at_20 |
| value: 92.698 |
| - type: ndcg_at_100 |
| value: 92.947 |
| - type: ndcg_at_1000 |
| value: 93.091 |
| - type: map_at_1 |
| value: 88.114 |
| - type: map_at_3 |
| value: 90.63 |
| - type: map_at_5 |
| value: 90.949 |
| - type: map_at_10 |
| value: 91.138 |
| - type: map_at_20 |
| value: 91.214 |
| - type: map_at_100 |
| value: 91.248 |
| - type: map_at_1000 |
| value: 91.254 |
| - type: recall_at_1 |
| value: 88.114 |
| - type: recall_at_3 |
| value: 93.681 |
| - type: recall_at_5 |
| value: 95.035 |
| - type: recall_at_10 |
| value: 96.389 |
| - type: recall_at_20 |
| value: 97.442 |
| - type: recall_at_100 |
| value: 98.796 |
| - type: recall_at_1000 |
| value: 99.9 |
| - type: accuracy |
| value: 88.114 |
| - type: precision_at_1 |
| value: 88.114 |
| - type: precision_at_3 |
| value: 31.227 |
| - type: precision_at_5 |
| value: 19.007 |
| - type: precision_at_10 |
| value: 9.639 |
| - type: precision_at_20 |
| value: 4.872 |
| - type: precision_at_100 |
| value: 0.988 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 88.1143 |
| - type: mrr_at_3 |
| value: 90.6302 |
| - type: mrr_at_5 |
| value: 90.9487 |
| - type: mrr_at_10 |
| value: 91.1384 |
| - type: mrr_at_20 |
| value: 91.2139 |
| - type: mrr_at_100 |
| value: 91.2479 |
| - type: mrr_at_1000 |
| value: 91.2538 |
| - type: hit_rate_at_1 |
| value: 88.114 |
| - type: hit_rate_at_3 |
| value: 93.681 |
| - type: hit_rate_at_5 |
| value: 95.035 |
| - type: hit_rate_at_10 |
| value: 96.389 |
| - type: hit_rate_at_20 |
| value: 97.442 |
| - type: hit_rate_at_100 |
| value: 98.796 |
| - type: hit_rate_at_1000 |
| value: 99.9 |
| - task: |
| type: Summarization |
| dataset: |
| type: mteb/summeval |
| name: MTEB SummEvalSummarization.v2 |
| config: default |
| split: test |
| revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c |
| metrics: |
| - type: pearson |
| value: 38.4249 |
| - type: spearman |
| value: 34.8894 |
| - type: cosine_spearman |
| value: 34.8894 |
| - type: cosine_pearson |
| value: 38.4249 |
| - type: dot_pearson |
| value: 27.3722 |
| - type: dot_spearman |
| value: 26.4149 |
| - task: |
| type: Retrieval |
| dataset: |
| type: CoIR-Retrieval/synthetic-text2sql |
| name: MTEB SyntheticText2SQL |
| config: default |
| split: test |
| revision: 686b87296c3a0191b5d9415a00526c62db9fce09 |
| metrics: |
| - type: ndcg_at_1 |
| value: 4.991 |
| - type: ndcg_at_3 |
| value: 54.565 |
| - type: ndcg_at_5 |
| value: 56.746 |
| - type: ndcg_at_10 |
| value: 58.046 |
| - type: ndcg_at_20 |
| value: 58.624 |
| - type: ndcg_at_100 |
| value: 59.115 |
| - type: ndcg_at_1000 |
| value: 59.216 |
| - type: map_at_1 |
| value: 4.991 |
| - type: map_at_3 |
| value: 43.787 |
| - type: map_at_5 |
| value: 45.005 |
| - type: map_at_10 |
| value: 45.555 |
| - type: map_at_20 |
| value: 45.718 |
| - type: map_at_100 |
| value: 45.791 |
| - type: map_at_1000 |
| value: 45.795 |
| - type: recall_at_1 |
| value: 4.991 |
| - type: recall_at_3 |
| value: 85.182 |
| - type: recall_at_5 |
| value: 90.446 |
| - type: recall_at_10 |
| value: 94.394 |
| - type: recall_at_20 |
| value: 96.65 |
| - type: recall_at_100 |
| value: 99.214 |
| - type: recall_at_1000 |
| value: 99.983 |
| - type: accuracy |
| value: 4.991 |
| - type: precision_at_1 |
| value: 4.991 |
| - type: precision_at_3 |
| value: 28.394 |
| - type: precision_at_5 |
| value: 18.089 |
| - type: precision_at_10 |
| value: 9.439 |
| - type: precision_at_20 |
| value: 4.833 |
| - type: precision_at_100 |
| value: 0.992 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 4.9906 |
| - type: mrr_at_3 |
| value: 43.7874 |
| - type: mrr_at_5 |
| value: 45.0051 |
| - type: mrr_at_10 |
| value: 45.5552 |
| - type: mrr_at_20 |
| value: 45.7177 |
| - type: mrr_at_100 |
| value: 45.7912 |
| - type: mrr_at_1000 |
| value: 45.7954 |
| - type: hit_rate_at_1 |
| value: 4.991 |
| - type: hit_rate_at_3 |
| value: 85.182 |
| - type: hit_rate_at_5 |
| value: 90.446 |
| - type: hit_rate_at_10 |
| value: 94.394 |
| - type: hit_rate_at_20 |
| value: 96.65 |
| - type: hit_rate_at_100 |
| value: 99.214 |
| - type: hit_rate_at_1000 |
| value: 99.983 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/trec-covid |
| name: MTEB TRECCOVID |
| config: default |
| split: test |
| revision: bb9466bac8153a0349341eb1b22e06409e78ef4e |
| metrics: |
| - type: ndcg_at_1 |
| value: 80.0 |
| - type: ndcg_at_3 |
| value: 84.961 |
| - type: ndcg_at_5 |
| value: 82.751 |
| - type: ndcg_at_10 |
| value: 80.823 |
| - type: ndcg_at_20 |
| value: 77.563 |
| - type: ndcg_at_100 |
| value: 65.588 |
| - type: ndcg_at_1000 |
| value: 60.628 |
| - type: map_at_1 |
| value: 0.231 |
| - type: map_at_3 |
| value: 0.697 |
| - type: map_at_5 |
| value: 1.105 |
| - type: map_at_10 |
| value: 2.063 |
| - type: map_at_20 |
| value: 3.768 |
| - type: map_at_100 |
| value: 13.49 |
| - type: map_at_1000 |
| value: 34.595 |
| - type: recall_at_1 |
| value: 0.231 |
| - type: recall_at_3 |
| value: 0.731 |
| - type: recall_at_5 |
| value: 1.178 |
| - type: recall_at_10 |
| value: 2.253 |
| - type: recall_at_20 |
| value: 4.227 |
| - type: recall_at_100 |
| value: 16.751 |
| - type: recall_at_1000 |
| value: 57.591 |
| - type: accuracy |
| value: 0.231 |
| - type: precision_at_1 |
| value: 86.0 |
| - type: precision_at_3 |
| value: 90.667 |
| - type: precision_at_5 |
| value: 88.4 |
| - type: precision_at_10 |
| value: 85.6 |
| - type: precision_at_20 |
| value: 81.4 |
| - type: precision_at_100 |
| value: 67.86 |
| - type: precision_at_1000 |
| value: 26.878 |
| - type: mrr_at_1 |
| value: 86.0 |
| - type: mrr_at_3 |
| value: 93.0 |
| - type: mrr_at_5 |
| value: 93.0 |
| - type: mrr_at_10 |
| value: 93.0 |
| - type: mrr_at_20 |
| value: 93.0 |
| - type: mrr_at_100 |
| value: 93.0 |
| - type: mrr_at_1000 |
| value: 93.0 |
| - type: hit_rate_at_1 |
| value: 86.0 |
| - type: hit_rate_at_3 |
| value: 100.0 |
| - type: hit_rate_at_5 |
| value: 100.0 |
| - type: hit_rate_at_10 |
| value: 100.0 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/TempReasonL1 |
| name: MTEB TempReasonL1 |
| config: default |
| split: test |
| revision: 2065ce0bde70750ed98bb91abd96d0d68ab94324 |
| metrics: |
| - type: ndcg_at_1 |
| value: 0.025 |
| - type: ndcg_at_3 |
| value: 0.455 |
| - type: ndcg_at_5 |
| value: 0.757 |
| - type: ndcg_at_10 |
| value: 1.405 |
| - type: ndcg_at_20 |
| value: 2.354 |
| - type: ndcg_at_100 |
| value: 5.877 |
| - type: ndcg_at_1000 |
| value: 13.319 |
| - type: map_at_1 |
| value: 0.025 |
| - type: map_at_3 |
| value: 0.337 |
| - type: map_at_5 |
| value: 0.501 |
| - type: map_at_10 |
| value: 0.76 |
| - type: map_at_20 |
| value: 1.014 |
| - type: map_at_100 |
| value: 1.442 |
| - type: map_at_1000 |
| value: 1.682 |
| - type: recall_at_1 |
| value: 0.025 |
| - type: recall_at_3 |
| value: 0.8 |
| - type: recall_at_5 |
| value: 1.55 |
| - type: recall_at_10 |
| value: 3.6 |
| - type: recall_at_20 |
| value: 7.4 |
| - type: recall_at_100 |
| value: 27.3 |
| - type: recall_at_1000 |
| value: 88.025 |
| - type: accuracy |
| value: 0.025 |
| - type: precision_at_1 |
| value: 0.025 |
| - type: precision_at_3 |
| value: 0.267 |
| - type: precision_at_5 |
| value: 0.31 |
| - type: precision_at_10 |
| value: 0.36 |
| - type: precision_at_20 |
| value: 0.37 |
| - type: precision_at_100 |
| value: 0.273 |
| - type: precision_at_1000 |
| value: 0.088 |
| - type: mrr_at_1 |
| value: 0.025 |
| - type: mrr_at_3 |
| value: 0.3375 |
| - type: mrr_at_5 |
| value: 0.5013 |
| - type: mrr_at_10 |
| value: 0.7595 |
| - type: mrr_at_20 |
| value: 1.0142 |
| - type: mrr_at_100 |
| value: 1.4417 |
| - type: mrr_at_1000 |
| value: 1.6818 |
| - type: hit_rate_at_1 |
| value: 0.025 |
| - type: hit_rate_at_3 |
| value: 0.8 |
| - type: hit_rate_at_5 |
| value: 1.55 |
| - type: hit_rate_at_10 |
| value: 3.6 |
| - type: hit_rate_at_20 |
| value: 7.4 |
| - type: hit_rate_at_100 |
| value: 27.3 |
| - type: hit_rate_at_1000 |
| value: 88.025 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/TempReasonL2Context |
| name: MTEB TempReasonL2Context |
| config: default |
| split: test |
| revision: 4ad8f1a456f63f732a633461ccc0f2c84ebe7ba1 |
| metrics: |
| - type: ndcg_at_1 |
| value: 10.988 |
| - type: ndcg_at_3 |
| value: 19.494 |
| - type: ndcg_at_5 |
| value: 22.662 |
| - type: ndcg_at_10 |
| value: 26.01 |
| - type: ndcg_at_20 |
| value: 28.645 |
| - type: ndcg_at_100 |
| value: 32.132 |
| - type: ndcg_at_1000 |
| value: 34.467 |
| - type: map_at_1 |
| value: 10.988 |
| - type: map_at_3 |
| value: 17.334 |
| - type: map_at_5 |
| value: 19.092 |
| - type: map_at_10 |
| value: 20.487 |
| - type: map_at_20 |
| value: 21.209 |
| - type: map_at_100 |
| value: 21.679 |
| - type: map_at_1000 |
| value: 21.763 |
| - type: recall_at_1 |
| value: 10.988 |
| - type: recall_at_3 |
| value: 25.774 |
| - type: recall_at_5 |
| value: 33.463 |
| - type: recall_at_10 |
| value: 43.747 |
| - type: recall_at_20 |
| value: 54.178 |
| - type: recall_at_100 |
| value: 73.17 |
| - type: recall_at_1000 |
| value: 91.829 |
| - type: accuracy |
| value: 10.988 |
| - type: precision_at_1 |
| value: 10.988 |
| - type: precision_at_3 |
| value: 8.591 |
| - type: precision_at_5 |
| value: 6.693 |
| - type: precision_at_10 |
| value: 4.375 |
| - type: precision_at_20 |
| value: 2.709 |
| - type: precision_at_100 |
| value: 0.732 |
| - type: precision_at_1000 |
| value: 0.092 |
| - type: mrr_at_1 |
| value: 10.9876 |
| - type: mrr_at_3 |
| value: 17.3337 |
| - type: mrr_at_5 |
| value: 19.0921 |
| - type: mrr_at_10 |
| value: 20.4875 |
| - type: mrr_at_20 |
| value: 21.2094 |
| - type: mrr_at_100 |
| value: 21.6794 |
| - type: mrr_at_1000 |
| value: 21.7625 |
| - type: hit_rate_at_1 |
| value: 10.988 |
| - type: hit_rate_at_3 |
| value: 25.774 |
| - type: hit_rate_at_5 |
| value: 33.463 |
| - type: hit_rate_at_10 |
| value: 43.747 |
| - type: hit_rate_at_20 |
| value: 54.178 |
| - type: hit_rate_at_100 |
| value: 73.17 |
| - type: hit_rate_at_1000 |
| value: 91.829 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/TempReasonL2Fact |
| name: MTEB TempReasonL2Fact |
| config: default |
| split: test |
| revision: 59b6048796e2a6ae40e843e9fa5675f047ae39b4 |
| metrics: |
| - type: ndcg_at_1 |
| value: 15.805 |
| - type: ndcg_at_3 |
| value: 29.202 |
| - type: ndcg_at_5 |
| value: 34.163 |
| - type: ndcg_at_10 |
| value: 38.365 |
| - type: ndcg_at_20 |
| value: 41.046 |
| - type: ndcg_at_100 |
| value: 43.755 |
| - type: ndcg_at_1000 |
| value: 44.875 |
| - type: map_at_1 |
| value: 15.805 |
| - type: map_at_3 |
| value: 25.792 |
| - type: map_at_5 |
| value: 28.555 |
| - type: map_at_10 |
| value: 30.308 |
| - type: map_at_20 |
| value: 31.051 |
| - type: map_at_100 |
| value: 31.431 |
| - type: map_at_1000 |
| value: 31.471 |
| - type: recall_at_1 |
| value: 15.805 |
| - type: recall_at_3 |
| value: 39.114 |
| - type: recall_at_5 |
| value: 51.121 |
| - type: recall_at_10 |
| value: 64.017 |
| - type: recall_at_20 |
| value: 74.56 |
| - type: recall_at_100 |
| value: 89.087 |
| - type: recall_at_1000 |
| value: 98.036 |
| - type: accuracy |
| value: 15.805 |
| - type: precision_at_1 |
| value: 15.805 |
| - type: precision_at_3 |
| value: 13.038 |
| - type: precision_at_5 |
| value: 10.224 |
| - type: precision_at_10 |
| value: 6.402 |
| - type: precision_at_20 |
| value: 3.728 |
| - type: precision_at_100 |
| value: 0.891 |
| - type: precision_at_1000 |
| value: 0.098 |
| - type: mrr_at_1 |
| value: 15.8051 |
| - type: mrr_at_3 |
| value: 25.7921 |
| - type: mrr_at_5 |
| value: 28.5548 |
| - type: mrr_at_10 |
| value: 30.3079 |
| - type: mrr_at_20 |
| value: 31.0514 |
| - type: mrr_at_100 |
| value: 31.4312 |
| - type: mrr_at_1000 |
| value: 31.4708 |
| - type: hit_rate_at_1 |
| value: 15.805 |
| - type: hit_rate_at_3 |
| value: 39.114 |
| - type: hit_rate_at_5 |
| value: 51.121 |
| - type: hit_rate_at_10 |
| value: 64.017 |
| - type: hit_rate_at_20 |
| value: 74.56 |
| - type: hit_rate_at_100 |
| value: 89.087 |
| - type: hit_rate_at_1000 |
| value: 98.036 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/TempReasonL2Pure |
| name: MTEB TempReasonL2Pure |
| config: default |
| split: test |
| revision: c8b90818b291f2421f8ce59645d7362b6043404e |
| metrics: |
| - type: ndcg_at_1 |
| value: 0.259 |
| - type: ndcg_at_3 |
| value: 0.553 |
| - type: ndcg_at_5 |
| value: 0.862 |
| - type: ndcg_at_10 |
| value: 1.142 |
| - type: ndcg_at_20 |
| value: 1.553 |
| - type: ndcg_at_100 |
| value: 3.488 |
| - type: ndcg_at_1000 |
| value: 8.024 |
| - type: map_at_1 |
| value: 0.259 |
| - type: map_at_3 |
| value: 0.476 |
| - type: map_at_5 |
| value: 0.645 |
| - type: map_at_10 |
| value: 0.759 |
| - type: map_at_20 |
| value: 0.874 |
| - type: map_at_100 |
| value: 1.101 |
| - type: map_at_1000 |
| value: 1.233 |
| - type: recall_at_1 |
| value: 0.259 |
| - type: recall_at_3 |
| value: 0.778 |
| - type: recall_at_5 |
| value: 1.538 |
| - type: recall_at_10 |
| value: 2.409 |
| - type: recall_at_20 |
| value: 4.021 |
| - type: recall_at_100 |
| value: 15.082 |
| - type: recall_at_1000 |
| value: 53.067 |
| - type: accuracy |
| value: 0.259 |
| - type: precision_at_1 |
| value: 0.259 |
| - type: precision_at_3 |
| value: 0.259 |
| - type: precision_at_5 |
| value: 0.308 |
| - type: precision_at_10 |
| value: 0.241 |
| - type: precision_at_20 |
| value: 0.201 |
| - type: precision_at_100 |
| value: 0.151 |
| - type: precision_at_1000 |
| value: 0.053 |
| - type: mrr_at_1 |
| value: 0.2594 |
| - type: mrr_at_3 |
| value: 0.4756 |
| - type: mrr_at_5 |
| value: 0.6451 |
| - type: mrr_at_10 |
| value: 0.7592 |
| - type: mrr_at_20 |
| value: 0.874 |
| - type: mrr_at_100 |
| value: 1.1009 |
| - type: mrr_at_1000 |
| value: 1.2329 |
| - type: hit_rate_at_1 |
| value: 0.259 |
| - type: hit_rate_at_3 |
| value: 0.778 |
| - type: hit_rate_at_5 |
| value: 1.538 |
| - type: hit_rate_at_10 |
| value: 2.409 |
| - type: hit_rate_at_20 |
| value: 4.021 |
| - type: hit_rate_at_100 |
| value: 15.082 |
| - type: hit_rate_at_1000 |
| value: 53.067 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/TempReasonL3Context |
| name: MTEB TempReasonL3Context |
| config: default |
| split: test |
| revision: 2c341c955bf2dc4f229c54b5b2cb9c5f1409ffd7 |
| metrics: |
| - type: ndcg_at_1 |
| value: 7.411 |
| - type: ndcg_at_3 |
| value: 15.711 |
| - type: ndcg_at_5 |
| value: 18.867 |
| - type: ndcg_at_10 |
| value: 21.916 |
| - type: ndcg_at_20 |
| value: 24.429 |
| - type: ndcg_at_100 |
| value: 28.138 |
| - type: ndcg_at_1000 |
| value: 30.745 |
| - type: map_at_1 |
| value: 7.411 |
| - type: map_at_3 |
| value: 13.579 |
| - type: map_at_5 |
| value: 15.331 |
| - type: map_at_10 |
| value: 16.596 |
| - type: map_at_20 |
| value: 17.291 |
| - type: map_at_100 |
| value: 17.792 |
| - type: map_at_1000 |
| value: 17.882 |
| - type: recall_at_1 |
| value: 7.411 |
| - type: recall_at_3 |
| value: 21.916 |
| - type: recall_at_5 |
| value: 29.575 |
| - type: recall_at_10 |
| value: 38.974 |
| - type: recall_at_20 |
| value: 48.87 |
| - type: recall_at_100 |
| value: 69.047 |
| - type: recall_at_1000 |
| value: 90.081 |
| - type: accuracy |
| value: 7.411 |
| - type: precision_at_1 |
| value: 7.411 |
| - type: precision_at_3 |
| value: 7.305 |
| - type: precision_at_5 |
| value: 5.915 |
| - type: precision_at_10 |
| value: 3.897 |
| - type: precision_at_20 |
| value: 2.444 |
| - type: precision_at_100 |
| value: 0.69 |
| - type: precision_at_1000 |
| value: 0.09 |
| - type: mrr_at_1 |
| value: 7.4108 |
| - type: mrr_at_3 |
| value: 13.5789 |
| - type: mrr_at_5 |
| value: 15.331 |
| - type: mrr_at_10 |
| value: 16.5958 |
| - type: mrr_at_20 |
| value: 17.2911 |
| - type: mrr_at_100 |
| value: 17.7916 |
| - type: mrr_at_1000 |
| value: 17.8818 |
| - type: hit_rate_at_1 |
| value: 7.411 |
| - type: hit_rate_at_3 |
| value: 21.916 |
| - type: hit_rate_at_5 |
| value: 29.575 |
| - type: hit_rate_at_10 |
| value: 38.974 |
| - type: hit_rate_at_20 |
| value: 48.87 |
| - type: hit_rate_at_100 |
| value: 69.047 |
| - type: hit_rate_at_1000 |
| value: 90.081 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/TempReasonL3Fact |
| name: MTEB TempReasonL3Fact |
| config: default |
| split: test |
| revision: ce6502fe436f00f05a73fe24ed0317c57fe4d6b0 |
| metrics: |
| - type: ndcg_at_1 |
| value: 12.878 |
| - type: ndcg_at_3 |
| value: 26.532 |
| - type: ndcg_at_5 |
| value: 31.533 |
| - type: ndcg_at_10 |
| value: 35.672 |
| - type: ndcg_at_20 |
| value: 38.278 |
| - type: ndcg_at_100 |
| value: 41.011 |
| - type: ndcg_at_1000 |
| value: 42.356 |
| - type: map_at_1 |
| value: 12.878 |
| - type: map_at_3 |
| value: 23.053 |
| - type: map_at_5 |
| value: 25.825 |
| - type: map_at_10 |
| value: 27.561 |
| - type: map_at_20 |
| value: 28.28 |
| - type: map_at_100 |
| value: 28.662 |
| - type: map_at_1000 |
| value: 28.709 |
| - type: recall_at_1 |
| value: 12.878 |
| - type: recall_at_3 |
| value: 36.647 |
| - type: recall_at_5 |
| value: 48.803 |
| - type: recall_at_10 |
| value: 61.455 |
| - type: recall_at_20 |
| value: 71.735 |
| - type: recall_at_100 |
| value: 86.421 |
| - type: recall_at_1000 |
| value: 97.176 |
| - type: accuracy |
| value: 12.878 |
| - type: precision_at_1 |
| value: 12.878 |
| - type: precision_at_3 |
| value: 12.216 |
| - type: precision_at_5 |
| value: 9.761 |
| - type: precision_at_10 |
| value: 6.146 |
| - type: precision_at_20 |
| value: 3.587 |
| - type: precision_at_100 |
| value: 0.864 |
| - type: precision_at_1000 |
| value: 0.097 |
| - type: mrr_at_1 |
| value: 12.8784 |
| - type: mrr_at_3 |
| value: 23.0532 |
| - type: mrr_at_5 |
| value: 25.8254 |
| - type: mrr_at_10 |
| value: 27.5609 |
| - type: mrr_at_20 |
| value: 28.2799 |
| - type: mrr_at_100 |
| value: 28.6617 |
| - type: mrr_at_1000 |
| value: 28.709 |
| - type: hit_rate_at_1 |
| value: 12.878 |
| - type: hit_rate_at_3 |
| value: 36.647 |
| - type: hit_rate_at_5 |
| value: 48.803 |
| - type: hit_rate_at_10 |
| value: 61.455 |
| - type: hit_rate_at_20 |
| value: 71.735 |
| - type: hit_rate_at_100 |
| value: 86.421 |
| - type: hit_rate_at_1000 |
| value: 97.176 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/TempReasonL3Pure |
| name: MTEB TempReasonL3Pure |
| config: default |
| split: test |
| revision: b2f4c5b7a21b4efd26ffdcea7a2e13a72dae4395 |
| metrics: |
| - type: ndcg_at_1 |
| value: 0.136 |
| - type: ndcg_at_3 |
| value: 3.906 |
| - type: ndcg_at_5 |
| value: 5.684 |
| - type: ndcg_at_10 |
| value: 7.491 |
| - type: ndcg_at_20 |
| value: 8.815 |
| - type: ndcg_at_100 |
| value: 11.635 |
| - type: ndcg_at_1000 |
| value: 15.893 |
| - type: map_at_1 |
| value: 0.136 |
| - type: map_at_3 |
| value: 2.956 |
| - type: map_at_5 |
| value: 3.938 |
| - type: map_at_10 |
| value: 4.68 |
| - type: map_at_20 |
| value: 5.043 |
| - type: map_at_100 |
| value: 5.4 |
| - type: map_at_1000 |
| value: 5.535 |
| - type: recall_at_1 |
| value: 0.136 |
| - type: recall_at_3 |
| value: 6.665 |
| - type: recall_at_5 |
| value: 11.003 |
| - type: recall_at_10 |
| value: 16.606 |
| - type: recall_at_20 |
| value: 21.848 |
| - type: recall_at_100 |
| value: 37.551 |
| - type: recall_at_1000 |
| value: 72.436 |
| - type: accuracy |
| value: 0.136 |
| - type: precision_at_1 |
| value: 0.136 |
| - type: precision_at_3 |
| value: 2.222 |
| - type: precision_at_5 |
| value: 2.201 |
| - type: precision_at_10 |
| value: 1.661 |
| - type: precision_at_20 |
| value: 1.092 |
| - type: precision_at_100 |
| value: 0.376 |
| - type: precision_at_1000 |
| value: 0.072 |
| - type: mrr_at_1 |
| value: 0.1356 |
| - type: mrr_at_3 |
| value: 2.956 |
| - type: mrr_at_5 |
| value: 3.9377 |
| - type: mrr_at_10 |
| value: 4.6803 |
| - type: mrr_at_20 |
| value: 5.043 |
| - type: mrr_at_100 |
| value: 5.3999 |
| - type: mrr_at_1000 |
| value: 5.5354 |
| - type: hit_rate_at_1 |
| value: 0.136 |
| - type: hit_rate_at_3 |
| value: 6.665 |
| - type: hit_rate_at_5 |
| value: 11.003 |
| - type: hit_rate_at_10 |
| value: 16.606 |
| - type: hit_rate_at_20 |
| value: 21.848 |
| - type: hit_rate_at_100 |
| value: 37.551 |
| - type: hit_rate_at_1000 |
| value: 72.436 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/webis-touche2020-v3 |
| name: MTEB Touche2020Retrieval.v3 |
| config: default |
| split: test |
| revision: 431886eaecc48f067a3975b70d0949ea2862463c |
| metrics: |
| - type: ndcg_at_1 |
| value: 70.408 |
| - type: ndcg_at_3 |
| value: 68.962 |
| - type: ndcg_at_5 |
| value: 65.062 |
| - type: ndcg_at_10 |
| value: 59.961 |
| - type: ndcg_at_20 |
| value: 54.968 |
| - type: ndcg_at_100 |
| value: 62.116 |
| - type: ndcg_at_1000 |
| value: 73.049 |
| - type: map_at_1 |
| value: 2.733 |
| - type: map_at_3 |
| value: 7.537 |
| - type: map_at_5 |
| value: 11.048 |
| - type: map_at_10 |
| value: 18.582 |
| - type: map_at_20 |
| value: 27.746 |
| - type: map_at_100 |
| value: 39.335 |
| - type: map_at_1000 |
| value: 42.661 |
| - type: recall_at_1 |
| value: 2.733 |
| - type: recall_at_3 |
| value: 8.056 |
| - type: recall_at_5 |
| value: 12.089 |
| - type: recall_at_10 |
| value: 21.85 |
| - type: recall_at_20 |
| value: 35.98 |
| - type: recall_at_100 |
| value: 65.002 |
| - type: recall_at_1000 |
| value: 92.102 |
| - type: accuracy |
| value: 2.733 |
| - type: precision_at_1 |
| value: 77.551 |
| - type: precision_at_3 |
| value: 76.19 |
| - type: precision_at_5 |
| value: 70.612 |
| - type: precision_at_10 |
| value: 63.673 |
| - type: precision_at_20 |
| value: 52.755 |
| - type: precision_at_100 |
| value: 20.776 |
| - type: precision_at_1000 |
| value: 3.2 |
| - type: mrr_at_1 |
| value: 77.551 |
| - type: mrr_at_3 |
| value: 85.034 |
| - type: mrr_at_5 |
| value: 85.5442 |
| - type: mrr_at_10 |
| value: 85.8844 |
| - type: mrr_at_20 |
| value: 86.0119 |
| - type: mrr_at_100 |
| value: 86.0119 |
| - type: mrr_at_1000 |
| value: 86.0119 |
| - type: hit_rate_at_1 |
| value: 77.551 |
| - type: hit_rate_at_3 |
| value: 93.878 |
| - type: hit_rate_at_5 |
| value: 95.918 |
| - type: hit_rate_at_10 |
| value: 97.959 |
| - type: hit_rate_at_20 |
| value: 100.0 |
| - type: hit_rate_at_100 |
| value: 100.0 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/toxic_conversations_50k |
| name: MTEB ToxicConversationsClassification |
| config: default |
| split: test |
| revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de |
| metrics: |
| - type: accuracy |
| value: 79.7656 |
| - type: f1 |
| value: 62.4086 |
| - type: f1_weighted |
| value: 83.823 |
| - type: precision |
| value: 61.1657 |
| - type: precision_weighted |
| value: 91.4603 |
| - type: recall |
| value: 76.4025 |
| - type: recall_weighted |
| value: 79.7656 |
| - type: ap |
| value: 20.3049 |
| - type: ap_weighted |
| value: 20.3049 |
| - task: |
| type: Classification |
| dataset: |
| type: mteb/tweet_sentiment_extraction |
| name: MTEB TweetSentimentExtractionClassification |
| config: default |
| split: test |
| revision: d604517c81ca91fe16a244d1248fc021f9ecee7a |
| metrics: |
| - type: accuracy |
| value: 61.2564 |
| - type: f1 |
| value: 61.4831 |
| - type: f1_weighted |
| value: 60.4063 |
| - type: precision |
| value: 61.2034 |
| - type: precision_weighted |
| value: 61.0357 |
| - type: recall |
| value: 63.1658 |
| - type: recall_weighted |
| value: 61.2564 |
| - type: ap |
| value: .nan |
| - type: ap_weighted |
| value: .nan |
| - task: |
| type: Clustering |
| dataset: |
| type: mteb/twentynewsgroups-clustering |
| name: MTEB TwentyNewsgroupsClustering.v2 |
| config: default |
| split: test |
| revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 |
| metrics: |
| - type: v_measure |
| value: 49.2585 |
| - type: v_measure_std |
| value: 1.3347 |
| - type: ami |
| value: 49.0605 |
| - type: ami_std |
| value: 1.3403 |
| - task: |
| type: PairClassification |
| dataset: |
| type: mteb/twittersemeval2015-pairclassification |
| name: MTEB TwitterSemEval2015 |
| config: default |
| split: test |
| revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 |
| metrics: |
| - type: similarity_accuracy |
| value: 85.951 |
| - type: similarity_f1 |
| value: 68.5638 |
| - type: similarity_precision |
| value: 63.1357 |
| - type: similarity_recall |
| value: 75.0132 |
| - type: similarity_ap |
| value: 75.0962 |
| - type: cosine_accuracy |
| value: 85.951 |
| - type: cosine_f1 |
| value: 68.5638 |
| - type: cosine_precision |
| value: 63.1357 |
| - type: cosine_recall |
| value: 75.0132 |
| - type: cosine_ap |
| value: 75.0961 |
| - type: manhattan_accuracy |
| value: 85.8258 |
| - type: manhattan_f1 |
| value: 67.0888 |
| - type: manhattan_precision |
| value: 60.3714 |
| - type: manhattan_recall |
| value: 75.4881 |
| - type: manhattan_ap |
| value: 73.8734 |
| - type: euclidean_accuracy |
| value: 85.8497 |
| - type: euclidean_f1 |
| value: 67.416 |
| - type: euclidean_precision |
| value: 60.5978 |
| - type: euclidean_recall |
| value: 75.9631 |
| - type: euclidean_ap |
| value: 74.0609 |
| - type: dot_accuracy |
| value: 84.8304 |
| - type: dot_f1 |
| value: 66.055 |
| - type: dot_precision |
| value: 62.3012 |
| - type: dot_recall |
| value: 70.2902 |
| - type: dot_ap |
| value: 70.9049 |
| - type: max_accuracy |
| value: 85.951 |
| - type: max_f1 |
| value: 68.5638 |
| - type: max_precision |
| value: 63.1357 |
| - type: max_recall |
| value: 75.9631 |
| - type: max_ap |
| value: 75.0962 |
| - task: |
| type: PairClassification |
| dataset: |
| type: mteb/twitterurlcorpus-pairclassification |
| name: MTEB TwitterURLCorpus |
| config: default |
| split: test |
| revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf |
| metrics: |
| - type: similarity_accuracy |
| value: 89.1586 |
| - type: similarity_f1 |
| value: 78.5685 |
| - type: similarity_precision |
| value: 76.535 |
| - type: similarity_recall |
| value: 80.713 |
| - type: similarity_ap |
| value: 86.0935 |
| - type: cosine_accuracy |
| value: 89.1586 |
| - type: cosine_f1 |
| value: 78.5685 |
| - type: cosine_precision |
| value: 76.535 |
| - type: cosine_recall |
| value: 80.713 |
| - type: cosine_ap |
| value: 86.0935 |
| - type: manhattan_accuracy |
| value: 88.9296 |
| - type: manhattan_f1 |
| value: 78.1758 |
| - type: manhattan_precision |
| value: 74.3454 |
| - type: manhattan_recall |
| value: 82.4222 |
| - type: manhattan_ap |
| value: 85.423 |
| - type: euclidean_accuracy |
| value: 88.9626 |
| - type: euclidean_f1 |
| value: 78.3338 |
| - type: euclidean_precision |
| value: 75.327 |
| - type: euclidean_recall |
| value: 81.5907 |
| - type: euclidean_ap |
| value: 85.4693 |
| - type: dot_accuracy |
| value: 88.4542 |
| - type: dot_f1 |
| value: 77.3602 |
| - type: dot_precision |
| value: 73.9198 |
| - type: dot_recall |
| value: 81.1364 |
| - type: dot_ap |
| value: 84.7308 |
| - type: max_accuracy |
| value: 89.1586 |
| - type: max_f1 |
| value: 78.5685 |
| - type: max_precision |
| value: 76.535 |
| - type: max_recall |
| value: 82.4222 |
| - type: max_ap |
| value: 86.0935 |
| - task: |
| type: Retrieval |
| dataset: |
| type: mteb/WinoGrande |
| name: MTEB WinoGrande |
| config: default |
| split: test |
| revision: 4dec9c5666e9f84702ac614363db6d96a68bc6de |
| metrics: |
| - type: ndcg_at_1 |
| value: 34.57 |
| - type: ndcg_at_3 |
| value: 53.414 |
| - type: ndcg_at_5 |
| value: 56.981 |
| - type: ndcg_at_10 |
| value: 59.825 |
| - type: ndcg_at_20 |
| value: 61.516 |
| - type: ndcg_at_100 |
| value: 62.997 |
| - type: ndcg_at_1000 |
| value: 63.283 |
| - type: map_at_1 |
| value: 34.57 |
| - type: map_at_3 |
| value: 48.987 |
| - type: map_at_5 |
| value: 50.98 |
| - type: map_at_10 |
| value: 52.159 |
| - type: map_at_20 |
| value: 52.632 |
| - type: map_at_100 |
| value: 52.86 |
| - type: map_at_1000 |
| value: 52.871 |
| - type: recall_at_1 |
| value: 34.57 |
| - type: recall_at_3 |
| value: 66.14 |
| - type: recall_at_5 |
| value: 74.743 |
| - type: recall_at_10 |
| value: 83.504 |
| - type: recall_at_20 |
| value: 90.134 |
| - type: recall_at_100 |
| value: 97.79 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: accuracy |
| value: 34.57 |
| - type: precision_at_1 |
| value: 34.57 |
| - type: precision_at_3 |
| value: 22.047 |
| - type: precision_at_5 |
| value: 14.949 |
| - type: precision_at_10 |
| value: 8.35 |
| - type: precision_at_20 |
| value: 4.507 |
| - type: precision_at_100 |
| value: 0.978 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: mrr_at_1 |
| value: 34.5699 |
| - type: mrr_at_3 |
| value: 48.9871 |
| - type: mrr_at_5 |
| value: 50.98 |
| - type: mrr_at_10 |
| value: 52.1595 |
| - type: mrr_at_20 |
| value: 52.6322 |
| - type: mrr_at_100 |
| value: 52.8598 |
| - type: mrr_at_1000 |
| value: 52.8713 |
| - type: hit_rate_at_1 |
| value: 34.57 |
| - type: hit_rate_at_3 |
| value: 66.14 |
| - type: hit_rate_at_5 |
| value: 74.743 |
| - type: hit_rate_at_10 |
| value: 83.504 |
| - type: hit_rate_at_20 |
| value: 90.134 |
| - type: hit_rate_at_100 |
| value: 97.79 |
| - type: hit_rate_at_1000 |
| value: 100.0 |
| --- |
| |
| # UltraBERT-Embed-150M |
|
|
| <p align="center"> |
| <img src="img/ultrabert_embed_banner.jpg" alt="UltraBERT-Embed" width="100%" style="border-radius: 12px;"> |
| </p> |
|
|
| UltraBERT-Embed-150M is an English text embedding model built on [UltraBERT-150M](https://huggingface.co/dmis-lab/UltraBERT-150M). It maps text to a 768-dimensional vector, takes inputs up to 32K tokens, follows task instructions on the query side, and embeds code as well as text. On MTEB(eng, v2) it scores 67.24 Mean(Task), the highest of any dense model under 200M parameters, without distillation from a larger embedding model. |
|
|
| | Model | Parameters | Hidden | Context | Description | |
| |---|---|---|---|---| |
| | **[UltraBERT-150M](https://huggingface.co/dmis-lab/UltraBERT-150M)** | 150M | 768 | 32K | Pre-trained encoder | |
| | **[UltraBERT-380M](https://huggingface.co/dmis-lab/UltraBERT-380M)** | 380M | 1,024 | 32K | Pre-trained encoder | |
| | **UltraBERT-Embed-150M** *(this model)* | 150M | 768 | 32K | Text embedding model built on UltraBERT-150M | |
|
|
| For more information about UltraBERT, please check out [our paper](). |
|
|
| ## ✨ Highlights |
|
|
| As of October 11, 2026, UltraBERT-Embed-150M is the state of the art among dense embedding models under 200M parameters on both MTEB(eng, v2) and MTEB(Code, v1). |
|
|
| - **MTEB(eng, v2): 67.24.** Mean over the 41 tasks, ahead of Tarka-Embedding-150M-V1 (66.39) and gte-modernbert-base (66.26). The models closest behind are distilled from larger embedding models; this one is trained with contrastive losses on public pairs only. |
| - **MTEB(Code, v1): 73.30.** Mean over the 12 tasks, with CoIR 72.73, from code pairs in both training stages. |
| - **LongEmbed: 79.30.** Highest of the models we compared, including 99.0 on passkey retrieval, with documents up to 32K tokens embedded whole rather than truncated. |
| - **RAR-b: 28.75.** Highest of the encoders we compared on reasoning-style retrieval. |
| - **Instructions.** Queries carry an E5-mistral style instruction (`Instruct: {task}\nQuery: `) and documents are embedded raw, so one model serves retrieval, classification, clustering and similarity. |
|
|
| ## 🚀 Usage |
|
|
| Requires `sentence-transformers >= 6.0` and `transformers >= 5`, with `trust_remote_code=True`. FlashAttention 2 is recommended: with it, Sentence Transformers packs the batch without padding, which matters for long inputs. SDPA also works (`"attn_implementation": "sdpa"`). |
|
|
| ```python |
| from sentence_transformers import SentenceTransformer |
| |
| model = SentenceTransformer( |
| "dmis-lab/UltraBERT-Embed-150M", |
| trust_remote_code=True, |
| model_kwargs={"attn_implementation": "flash_attention_2", "dtype": "bfloat16"}, |
| ) |
| |
| queries = ["Win big, click now!"] |
| labels = ["spam", "promotional", "announcement"] |
| |
| query_embeddings = model.encode(queries, prompt="Instruct: Labeling spam emails\nQuery: ") |
| label_embeddings = model.encode(labels, prompt="Label: ") |
| print(model.similarity(query_embeddings, label_embeddings)) |
| ``` |
|
|
| `prompt_name="query"` applies the default retrieval instruction. Documents never take an instruction, except label phrases, which take the prefix `Label: `. For symmetric tasks (STS, paraphrase, duplicate detection) apply the same instruction to both sides, e.g. `Instruct: Retrieve semantically similar text.\nQuery: `. For classification and clustering, name the categories, e.g. `Instruct: Classify the sentiment expressed in the given movie review text\nQuery: `. |
|
|
|
|
| ## 📊 Evaluation |
|
|
| ### MTEB(eng, v2) |
|
|
| | Model | Params | Average (41) | Class. (8) | Clust. (8) | PairClass. (3) | Rerank. (2) | Retr. (10) | STS (9) | Summ. (1) | |
| |:--|--:|--:|--:|--:|--:|--:|--:|--:|--:| |
| | **UltraBERT-Embed-150M** | 151M | **67.24** | 81.74 | 49.70 | 85.33 | 47.50 | <u>57.20</u> | <u>83.03</u> | **34.89** | |
| | Tarka-Embedding-150M-V1 (distilled) | 156M | <u>66.39</u> | <u>86.28</u> | <u>52.02</u> | 81.65 | 45.66 | 51.52 | 81.48 | 30.83 | |
| | gte-modernbert-base | 149M | 66.26 | 80.70 | 48.06 | 85.94 | 47.61 | 56.91 | 81.40 | 31.81 | |
| | F2LLM-v2-160M (distilled) | 159M | 65.93 | **87.13** | **52.55** | 82.08 | 45.26 | 49.33 | 80.76 | 28.63 | |
| | gte-base-en-v1.5 | 137M | 65.8 | 81.0 | 47.9 | 85.4 | 46.6 | 55.5 | 81.7 | 29.0 | |
| | GIST-Embedding-v0 | 109M | 65.50 | 78.16 | 48.50 | 86.33 | 47.52 | 53.59 | **83.35** | 32.32 | |
| | bge-base-en-v1.5 | 109M | 65.14 | 77.69 | 47.42 | **86.56** | 46.66 | 54.75 | 82.12 | 30.19 | |
| | stella-base-en-v2 | 109M | 64.90 | 79.37 | 47.50 | <u>86.51</u> | 47.31 | 51.56 | 82.73 | 31.78 | |
| | nomic-embed-text-v1 | 137M | 63.60 | 76.77 | 46.64 | 85.15 | 45.71 | 50.72 | 81.78 | 30.40 | |
| | granite-embedding-english-r2 | 149M | 62.84 | 70.71 | 47.20 | 79.34 | **49.10** | 56.43 | 78.12 | 29.31 | |
| | e5-base-v2 | 109M | 62.67 | 75.48 | 45.20 | 85.63 | 45.13 | 49.67 | 80.64 | <u>34.26</u> | |
| | DenseOn | 149M | 62.47 | 72.61 | 46.11 | 79.57 | <u>47.70</u> | **59.09** | 73.08 | 28.84 | |
| | nomic-embed-text-v1.5 | 137M | 62.20 | 75.71 | 47.55 | 85.16 | 46.01 | 47.97 | 78.70 | 28.56 | |
| | modernbert-embed-base | 149M | 62.1 | 75.1 | 46.3 | 82.4 | 46.3 | 48.7 | 79.3 | 34.1 | |
|
|
| ### MTEB(Code, v1) and CoIR |
|
|
| | Model | Params | Average (12) | CoIR | Apps | COIR-CSN | CodeEdit | CFB-MT | CFB-ST | CSN-CC | CSN | CT-Contest | CT-DL | CosQA | SO-QA | T2SQL | |
| |:--|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:| |
| | **UltraBERT-Embed-150M** | 151M | **73.30** | **72.73** | **68.50** | 82.23 | 62.21 | 84.37 | **86.40** | **93.73** | **90.03** | **88.08** | <u>37.23</u> | 36.29 | <u>92.43</u> | 58.05 | |
| | gte-modernbert-base | 149M | <u>71.66</u> | <u>71.53</u> | 57.72 | <u>83.10</u> | 55.83 | <u>86.15</u> | 86.00 | <u>93.61</u> | 88.76 | 72.35 | **37.27** | **43.36** | 91.14 | <u>64.61</u> | |
| | NightOwl-CodeEmbedding | 151M | 70.91 | 68.68 | 39.18 | **84.26** | **74.81** | 76.69 | 85.21 | 91.80 | <u>89.24</u> | 75.95 | 36.06 | <u>42.81</u> | 86.61 | **68.27** | |
| | F2LLM-v2-160M (distilled) | 159M | 70.38 | 70.52 | <u>58.95</u> | 82.18 | 55.81 | **90.00** | <u>86.22</u> | 85.14 | 83.49 | 80.22 | 34.55 | 36.52 | 87.36 | 64.09 | |
| | Tarka-Embedding-150M-V1 (distilled) | 156M | 57.71 | 55.94 | 38.96 | 62.39 | 51.60 | 49.15 | 78.34 | 54.55 | 81.48 | <u>80.74</u> | 31.79 | 36.03 | 75.97 | 51.47 | |
| | granite-embedding-english-r2 | 149M | 57.22 | 54.65 | 13.96 | 64.65 | 59.35 | 52.54 | 77.18 | 47.67 | 80.79 | 77.07 | 35.03 | 37.01 | 91.80 | 49.55 | |
| | DenseOn | 149M | 55.96 | 52.75 | 12.81 | 55.24 | <u>62.83</u> | 48.97 | 78.63 | 55.43 | 81.11 | 69.66 | 31.84 | 36.07 | **92.60** | 46.29 | |
| | e5-base-v2 | 109M | 53.51 | 50.39 | 11.52 | 62.71 | 55.89 | 41.56 | 74.52 | 56.87 | 82.31 | 62.50 | 21.87 | 32.59 | 87.85 | 51.88 | |
| | modernbert-embed-base | 149M | 53.28 | 49.51 | 6.54 | 71.35 | 57.24 | 40.66 | 70.28 | 56.84 | 86.99 | 56.73 | 26.88 | 35.44 | 76.96 | 53.42 | |
| | bge-base-en-v1.5 | 109M | 50.35 | 46.61 | 6.45 | 72.02 | 53.07 | 33.65 | 70.00 | 50.89 | 85.11 | 45.65 | 23.49 | 33.71 | 80.23 | 49.98 | |
| | nomic-embed-text-v1.5 | 137M | 47.36 | 43.62 | 4.88 | 64.91 | 49.50 | 39.73 | 64.89 | 48.67 | 82.64 | 45.63 | 21.93 | 26.49 | 71.06 | 48.05 | |
|
|
| ### LongEmbed |
|
|
| | Model | Params | Max. Len. | Average (6) | NQA | Needle | Passkey | QMSum | SFD | 2WmQA | |
| |:--|--:|--:|--:|--:|--:|--:|--:|--:|--:| |
| | **UltraBERT-Embed-150M** | 151M | 32768 | **79.30** | **64.55** | 66.50 | **99.00** | **55.28** | **97.62** | **92.86** | |
| | bekko-embedding-v1-a25m | 123M | 8192 | <u>73.19</u> | 48.13 | <u>74.75</u> | 85.00 | 47.08 | <u>96.92</u> | <u>87.24</u> | |
| | BidirLM-270M-Embedding | 268M | 32768 | 71.82 | <u>54.21</u> | 66.75 | 90.50 | 47.21 | 96.19 | 76.04 | |
| | granite-embedding-311m-multilingual-r2 | 312M | 8192 | 71.72 | 51.11 | 55.50 | <u>95.50</u> | <u>47.25</u> | 95.71 | 85.23 | |
| | bekko-embedding-v1-a8m | 106M | 8192 | 70.60 | 44.50 | **76.50** | 85.00 | 42.54 | 96.40 | 78.68 | |
| | granite-embedding-english-r2 | 149M | 8192 | 67.8 | 49.7 | 49.8 | 83.3 | 42.8 | 94.2 | 86.9 | |
| | DenseOn | 149M | 8192 | 66.50 | 49.95 | 60.25 | 64.50 | 44.05 | 94.83 | 85.45 | |
| | F2LLM-v2-330M (distilled) | 334M | 8192 | 66.26 | 48.16 | 52.25 | 76.25 | 40.15 | 96.56 | 84.18 | |
| | jina-embeddings-v5-text-nano (distilled) | 212M | 8192 | 63.65 | 52.17 | 59.75 | 81.50 | 31.83 | 81.80 | 74.87 | |
| | gte-multilingual-base | 305M | 8192 | 62.12 | 52.36 | 42.25 | 55.50 | 43.03 | 95.50 | 84.08 | |
| | gte-base-en-v1.5 | 137M | 8192 | 59.4 | 47.3 | 31.8 | 58.0 | 41.7 | 92.1 | 85.6 | |
| | F2LLM-v2-160M (distilled) | 159M | 8192 | 57.64 | 29.18 | 54.75 | 69.00 | 32.60 | 91.06 | 69.22 | |
| | nomic-embed-text-v1 | 137M | 8192 | 57.5 | 41.2 | 39.5 | 60.7 | 36.7 | 93.0 | 73.8 | |
| | gte-modernbert-base | 149M | 8192 | 57.0 | 30.2 | 24.5 | 66.8 | 40.7 | 93.1 | 87.0 | |
| | modernbert-embed-base | 149M | 8192 | 56.3 | 41.1 | 35.8 | 62.5 | 36.7 | 94.5 | 67.5 | |
|
|
|
|
| ### RAR-b |
|
|
| | Model | Params | Average (17) | ARC-C | αNLI | HellaSwag | WinoGrande | PIQA | SIQA | Quail | SpartQA | TR | Code | Math | |
| |:--|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:| |
| | **UltraBERT-Embed-150M** | 151M | **28.75** | **16.15** | **36.09** | **33.91** | 59.82 | **35.05** | 2.36 | **7.97** | 7.05 | 18.86 | **72.81** | 85.52 | |
| | granite-embedding-311m-multilingual-r2 | 312M | <u>27.95</u> | 9.89 | 30.55 | 26.36 | **66.61** | 27.45 | 2.07 | 5.07 | **64.63** | 15.50 | 58.35 | 75.65 | |
| | e5-large-v2 | 335M | 27.33 | 11.29 | 15.16 | 27.86 | 51.52 | 29.55 | <u>4.07</u> | 5.28 | 10.37 | **25.04** | 58.67 | 75.51 | |
| | KaLM-embedding-multilingual-mini-v1 | 494M | 25.79 | 10.36 | 26.73 | 29.37 | 29.91 | 30.57 | 2.93 | 5.34 | 10.57 | <u>19.72</u> | <u>70.64</u> | 83.96 | |
| | DenseOn | 149M | 24.12 | <u>13.81</u> | 34.33 | <u>32.24</u> | 25.64 | <u>34.26</u> | 1.72 | 5.69 | 9.44 | 14.95 | 65.48 | 82.78 | |
| | e5-base-v2 | 109M | 23.83 | 10.01 | 21.73 | 25.48 | 46.99 | 27.71 | **4.37** | 4.94 | 7.84 | 19.12 | 54.47 | 67.76 | |
| | gte-modernbert-base | 149M | 23.54 | 13.28 | 28.64 | 29.45 | 53.00 | 30.52 | 2.56 | 5.76 | 7.82 | 12.25 | 54.00 | <u>89.41</u> | |
| | arctic-embed-m-v2.0 | 305M | 23.33 | 10.78 | 30.42 | 28.25 | <u>66.33</u> | 30.83 | 1.67 | 5.12 | 3.75 | 12.77 | 56.09 | 73.91 | |
| | F2LLM-v2-330M (distilled) | 334M | 23.28 | 13.38 | 30.23 | 26.46 | 34.90 | 28.48 | 2.96 | <u>5.93</u> | 13.13 | 12.43 | 61.12 | **92.12** | |
| | Tarka-Embedding-150M-V1 (distilled) | 156M | 21.76 | 10.96 | <u>35.03</u> | 25.96 | 58.86 | 27.94 | 2.01 | 4.98 | 6.14 | 11.91 | 56.28 | 58.30 | |
| | granite-embedding-english-r2 | 149M | 20.65 | 10.75 | 31.71 | 28.01 | 35.12 | 29.02 | 1.69 | 4.99 | 2.82 | 9.64 | 52.92 | 86.59 | |
| | F2LLM-v2-160M (distilled) | 159M | 19.62 | 9.47 | 27.76 | 22.25 | 43.16 | 23.75 | 1.46 | 4.86 | <u>14.48</u> | 7.45 | 49.72 | 84.50 | |
| | mxbai-embed-large-v1 | 335M | 19.50 | 10.53 | 23.32 | 28.99 | 47.99 | 28.45 | 2.33 | 3.39 | 1.51 | 12.24 | 43.79 | 55.57 | |
| | modernbert-embed-base | 149M | 16.04 | 10.38 | 30.08 | 28.55 | 22.82 | 28.93 | 1.19 | 3.41 | 6.66 | 5.82 | 41.31 | 58.58 | |
| | nomic-embed-text-v1.5 | 137M | 15.39 | 10.24 | 29.72 | 24.67 | 46.79 | 24.71 | 2.67 | 4.22 | 6.30 | 6.88 | 29.09 | 35.14 | |
| | bge-base-en-v1.5 | 109M | 14.59 | 9.66 | 10.99 | 26.64 | 13.76 | 25.70 | 0.94 | 1.42 | 3.37 | 8.90 | 46.47 | 46.86 | |
|
|
| ## 🏛️ Model Description |
|
|
| | | | |
| |---|---| |
| | Base model | [UltraBERT-150M](https://huggingface.co/dmis-lab/UltraBERT-150M) | |
| | Parameters | 151M | |
| | Embedding dimension | 768 | |
| | Pooling | CLS token | |
| | Similarity | cosine | |
| | Max sequence length | 32,768 tokens | |
| | Instruction format | `Instruct: {task}\nQuery: ` on queries | |
| | Language | English (text and code) | |
|
|
| ## ⚙️ Training |
|
|
| UltraBERT-Embed-150M is trained from [UltraBERT-150M](https://huggingface.co/dmis-lab/UltraBERT-150M) in two contrastive stages with Sentence Transformers. The first stage trains on about 500M weakly supervised text and code pairs with in-batch negatives only. The second stage fine-tunes on about 5.7M examples with mined hard negatives, with a task instruction on the query side and inputs up to 32K tokens so that long documents are seen whole. The released weights are the average of several checkpoints from the second stage. |
|
|
| Training data comes from public sources only, without distillation from a larger embedding model: |
|
|
| - Web question-answer and title-body pairs (PAQ, GooAQ, Yahoo Answers, Amazon QA, Reddit, StackOverflow, news, Wikipedia, S2ORC) |
| - Code pairs (CornStack, OCGI, xCodeEval) |
| - Retrieval datasets with hard negatives (MS MARCO, NQ, TriviaQA, HotpotQA, FEVER, SQuAD 2.0, MIRACL, MLDR, DBpedia, BioASQ, PubMedQA, MedQuAD) |
| - Sentence similarity (NLI, QQP, PAWS-X, synthetic STS) |
| - Classification and clustering (synthetic classification data from the Nemotron embedding dataset, Amazon polarity, toxicity datasets) |
|
|
| ## Citation |
|
|
| ```bibtex |
| @misc{park2026ultrabert, |
| title={UltraBERT: Beyond the Limit of BERT Pre-Training}, |
| author={Jungwoo Park and Taewhoo Lee and Jaewoo Kang}, |
| year={2026}, |
| eprint={2610.15028}, |
| archivePrefix={arXiv}, |
| primaryClass={cs.CL}, |
| url={https://arxiv.org/abs/2610.15028}, |
| } |
| ``` |
|
|
